Merge remote-tracking branch 'origin' into dependabot/go_modules/dock…

…er/sandbox-bundled/bootstrap/golang.org/x/net-0.7.0

Signed-off-by: Eduardo Apolinario <[email protected]>

.codespellrc

@@ -0,0 +1,6 @@
+[codespell]
+skip = .git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen
+# some strings with unicodes, constructs like [o]utput
+ignore-regex = ".*\\0[0-9][0-9].*"|json:"[^"]*"|\b[a-z]*\[[a-z]\][a-z]*\b
+# some ad-hoc variable names etc
+ignore-words-list = te,nd,querys,ser

.gitattributes

@@ -1 +1,5 @@
 docs/**/*html linguist-generated=true
+flyteidl/gen/** linguist-generated=true
+flyteidl/protos/**/*.rst linguist-generated=true
+flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true
+

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,38 +1,51 @@
-<!--
-Thank you for sending the PR! 
-Please fill the applicable details below
-Happy contributing!
--->
-
 ## Tracking issue
+_https://github.com/flyteorg/flyte/issues/<number>_
 
 <!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
 
 <!-- Remove this section if not applicable -->
 
 <!-- Example: Closes #31 -->
 
-## Describe your changes
+## Why are the changes needed?
+
+<!--
+Please clarify why the changes are needed. For instance,
+1. If you propose a new API, clarify the use case for a new API.
+2. If you fix a bug, you can clarify why it is a bug.
+-->
+
+## What changes were proposed in this pull request?
+
+<!--
+Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
+If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
+1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
+2. If there is design documentation, please add the link.
+-->
 
-<!-- List all the proposed changes in your PR -->
+## How was this patch tested?
 
-<!-- Mark all the applicable boxes. To mark the box as done follow the following conventions -->
 <!--
-[x] - Correct; marked as done
-[X] - Correct; marked as done
-[ ] - Not correct; marked as **not** done
+If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
+If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
+If tests were not added, please describe why they were not added and/or why it was difficult to add.
 -->
 
+### Setup process
+
+### Screenshots
+
 ## Check all the applicable boxes <!-- Follow the above conventions to check the box -->
 
 - [ ] I updated the documentation accordingly.
 - [ ] All new and existing tests passed.
 - [ ] All commits are signed-off.
 
-## Screenshots
+## Related PRs
 
-<!-- Add all the screenshots which support your changes -->
+<!-- Add related pull requests for reviewers to check -->
 
-## Note to reviewers
+## Docs link
 
-<!-- Add notes to reviewers if applicable -->
+<!-- Add documentation link built by CI jobs here, and specify the changed place -->

.github/workflows/checks.yml

@@ -0,0 +1,131 @@
+name: Components Checks
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+env:
+  GO_VERSION: "1.21"
+  PRIORITIES: "P0"
+jobs:
+  unpack-envvars:
+    runs-on: ubuntu-latest
+    outputs:
+      go-version: ${{ steps.step.outputs.go-version }}
+    steps:
+      - id: step
+        run: |
+          echo "go-version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT
+
+  lint:
+    name: Lint
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          # TODO(monorepo): Enable lint flytecopilot
+          # - flytecopilot
+          - flyteidl
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    uses: ./.github/workflows/lint.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+  unit-tests:
+    name: Unit Tests
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+  docker-build:
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+    name: Docker Build Images
+    uses: ./.github/workflows/component_docker_build.yml
+    with:
+       component: ${{ matrix.component }}
+
+  # TODO(monorepo): these tests are broken. They never test an actual change.
+  # endtoend:
+  #   name: End2End Test
+  #   needs: [ docker-build ]
+  #   uses: ./.github/workflows/end2end.yml
+  #   with:
+  #     # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id)
+  #     cache_key: ${{ needs.docker-build.outputs.cache_key }}
+  #     priorities: "P0"
+
+  integration:
+    name: Integration Test
+    needs:
+      - docker-build
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - flyteadmin
+    uses: ./.github/workflows/integration.yml
+    with:
+      component: ${{ matrix.component }}
+      cache_key: ${{ needs.docker-build.outputs.cache_key }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+
+  generate:
+    name: Check Go Generate
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+    uses: ./.github/workflows/go_generate.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+
+  build_docker_images:
+    name: Build Images
+    uses: ./.github/workflows/publish-images.yml
+    with:
+      push: false
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}

.github/workflows/codespell.yml

@@ -0,0 +1,28 @@
+---
+name: Codespell
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Codespell
+        uses: codespell-project/actions-codespell@v2
+        with:
+          skip: "*.pb,monodocs-environment.lock.yaml"

.github/workflows/component_docker_build.yml

@@ -0,0 +1,49 @@
+name: Build Docker Image
+
+on:
+  workflow_call:
+    inputs:
+      component:
+        required: true
+        type: string
+    outputs:
+      cache_key:
+        description: "Docker Cache key"
+        value: ${{ jobs.build_docker.outputs.cache_key }}
+jobs:
+  build_docker:
+    runs-on: ubuntu-latest
+    outputs:
+      cache_key: ${{ steps.cache_key.outputs.cache_key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-${{ inputs.component }}
+          key: /tmp/docker-images-${{ github.run_id }}-${{ inputs.component }}
+          restore-keys: |
+            /tmp/docker-images-
+      - name: Set cache key output
+        id: cache_key
+        run: |
+          echo ::set-output name=cache_key::/tmp/docker-images-${{ github.run_id }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        env:
+          # We are unable to leverage docker buildx here without changing the
+          # caching mechanism significantly. See here for the caching options
+          # available for docker buildx: https://docs.docker.com/engine/reference/commandline/buildx_build/#cache-from
+          # For now at least enable DOCKER_BUILDKIT for faster builds. Eventually we
+          # should rewrite this pipeline to use docker buildx with cache type=gha.
+          DOCKER_BUILDKIT: "1"
+        run: |
+          docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:builder --target builder --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} .
+          docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:latest --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} .
+
+      - name: Tag and cache docker image
+        run: |
+          mkdir -p /tmp/tmp/docker-images-${{ inputs.component }} && docker save ${{ github.repository_owner }}/${{ inputs.component }}:builder -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-builder-${{ inputs.component }}.tar && docker save ${{ github.repository_owner }}/${{ inputs.component }}:latest -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-${{ inputs.component }}.tar