Merge branch 'main' into main

.github/actions/upload_managed_plugin/action.yml

@@ -0,0 +1,86 @@
+name: upload-managed-plugin
+description: Upload binaries as rpk managed plugin
+inputs:
+  aws_access_key_id:
+    description: For accessing S3 bucket
+    required: true
+  aws_secret_access_key:
+    description: For accessing S3 bucket
+    required: true
+  aws_region:
+    description: For accessing S3 bucket
+    required: true
+  aws_s3_bucket:
+    description: S3 bucket to use
+    required: true
+  artifacts_file:
+    description: Path to goreleaser artifacts.json
+    required: true
+  metadata_file:
+    description: Path to goreleaser artifacts.json
+    required: true
+  project_root_dir:
+    description: Root dir of goreleaser project
+    required: true
+  plugin_name:
+    description: Should match the goreleaser build id for the binary  E.g. "connect"
+    required: true
+  goos:
+    description: CSV list of target OS's to filter on
+    required: true
+  goarch:
+    description: CSV list of target arch's to filter on
+    required: true
+  repo_hostname:
+    description: RPK Plugins repo hostname. E.g. rpk-plugins.redpanda.com
+    required: true
+  dry_run:
+    description: Dry run means skipping writes to S3 ("true" or "false")
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws_access_key_id }}
+        aws-secret-access-key: ${{ inputs.aws_secret_access_key }}
+        aws-region: ${{ inputs.aws_region }}
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: install deps
+      working-directory: resources/plugin_uploader
+      shell: bash
+      run: pip install -r requirements.txt
+
+    - name: Upload archives
+      working-directory: resources/plugin_uploader
+      shell: bash
+      run: |
+        DRY_RUN_FLAG=${{ inputs.dry_run != 'false' && '--dry-run' || '' }}
+        ./plugin_uploader.py upload-archives \
+          --artifacts-file=${{ inputs.artifacts_file }} \
+          --metadata-file=${{ inputs.metadata_file }} \
+          --project-root-dir=${{ inputs.project_root_dir }} \
+          --region=${{ inputs.aws_region }} \
+          --bucket=${{ inputs.aws_s3_bucket }} \
+          --plugin=${{ inputs.plugin_name }} \
+          --goos=${{ inputs.goos }} \
+          --goarch=${{ inputs.goarch }} \
+          $DRY_RUN_FLAG
+
+    - name: Upload manifest
+      working-directory: resources/plugin_uploader
+      shell: bash
+      run: |
+        DRY_RUN_FLAG=${{ inputs.dry_run != 'false' && '--dry-run' || '' }}
+        ./plugin_uploader.py upload-manifest \
+          --region=${{ inputs.aws_region }} \
+          --bucket=${{ inputs.aws_s3_bucket }} \
+          --plugin=${{ inputs.plugin_name }} \
+          --repo-hostname=${{ inputs.repo_hostname }} \
+          $DRY_RUN_FLAG

.github/workflows/release.yml

@@ -42,6 +42,29 @@ jobs:
 
   docker:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - flavor: default
+            latest: auto
+            suffix: ""
+            platform: linux/amd64,linux/arm64
+            file: ./resources/docker/Dockerfile
+          - flavor: cgo
+            latest: false
+            suffix: -cgo
+            platform: linux/amd64
+            file: ./resources/docker/Dockerfile.cgo
+          - flavor: cloud
+            latest: false
+            suffix: -cloud
+            platform: linux/amd64,linux/arm64
+            file: ./resources/docker/Dockerfile.cloud
+          - flavor: ai
+            latest: false
+            suffix: -ai
+            platform: linux/amd64,linux/arm64
+            file: ./resources/docker/Dockerfile.ai
     permissions:
       id-token: write
       packages: write
@@ -84,93 +107,25 @@ jobs:
       id: buildx
       uses: docker/setup-buildx-action@v3
 
-    - name: Docker meta CGO
-      id: docker_meta_cgo
-      uses: docker/metadata-action@v5
-      with:
-        images: |
-          redpandadata/connect
-        flavor: |
-          latest=auto
-          suffix=-cgo
-        tags: |
-          type=semver,suffix=-cgo,pattern={{version}}
-          type=semver,suffix=-cgo,pattern={{major}}.{{minor}}
-          type=semver,suffix=-cgo,pattern={{major}}
-
-    - name: Build and push CGO
-      uses: docker/build-push-action@v6
-      with:
-        context: ./
-        file: ./resources/docker/Dockerfile.cgo
-        push: true
-        tags: ${{ steps.docker_meta_cgo.outputs.tags }}
-
     - name: Docker meta
       id: docker_meta
       uses: docker/metadata-action@v5
       with:
         images: |
           redpandadata/connect
+        flavor: |
+          latest=${{ matrix.latest }}
+          suffix=${{ matrix.suffix }}
         tags: |
-          type=semver,pattern={{version}}
-          type=semver,pattern={{major}}.{{minor}}
-          type=semver,pattern={{major}}
+          type=semver,suffix=${{ matrix.suffix }},pattern={{version}}
+          type=semver,suffix=${{ matrix.suffix }},pattern={{major}}.{{minor}}
+          type=semver,suffix=${{ matrix.suffix }},pattern={{major}}
 
     - name: Build and push
       uses: docker/build-push-action@v6
       with:
         context: ./
-        file: ./resources/docker/Dockerfile
-        builder: ${{ steps.buildx.outputs.name }}
-        platforms: linux/amd64,linux/arm64
+        file: ${{ matrix.file }}
+        platforms: ${{ matrix.platform }}
         push: true
         tags: ${{ steps.docker_meta.outputs.tags }}
-
-    - name: Docker meta cloud
-      id: docker_meta_cloud
-      uses: docker/metadata-action@v5
-      with:
-        images: |
-          redpandadata/connect
-        flavor: |
-          latest=false
-          suffix=-cloud
-        tags: |
-          type=semver,suffix=-cloud,pattern={{version}}
-          type=semver,suffix=-cloud,pattern={{major}}.{{minor}}
-          type=semver,suffix=-cloud,pattern={{major}}
-
-    - name: Build and push cloud
-      uses: docker/build-push-action@v6
-      with:
-        context: ./
-        file: ./resources/docker/Dockerfile.cloud
-        builder: ${{ steps.buildx.outputs.name }}
-        platforms: linux/amd64,linux/arm64
-        push: true
-        tags: ${{ steps.docker_meta_cloud.outputs.tags }}
-
-    - name: Docker meta cloud AI
-      id: docker_meta_ai
-      uses: docker/metadata-action@v5
-      with:
-        images: |
-          redpandadata/connect
-        flavor: |
-          latest=false
-          suffix=-ai
-        tags: |
-          type=semver,suffix=-ai,pattern={{version}}
-          type=semver,suffix=-ai,pattern={{major}}.{{minor}}
-          type=semver,suffix=-ai,pattern={{major}}
-
-    - name: Build and push cloud
-      uses: docker/build-push-action@v6
-      with:
-        context: ./
-        file: ./resources/docker/Dockerfile.ai
-        builder: ${{ steps.buildx.outputs.name }}
-        platforms: linux/amd64,linux/arm64
-        push: true
-        tags: ${{ steps.docker_meta_ai.outputs.tags }}

.github/workflows/test_plugin_uploader.yml

@@ -0,0 +1,61 @@
+name: Test Plugin Uploader
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'resources/plugin_uploader/**'
+      - '.github/workflows/test_plugin_uploader.yml'
+  pull_request:
+    paths:
+      - 'resources/plugin_uploader/**'
+      - '.github/workflows/test_plugin_uploader.yml'
+
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - working-directory: resources/plugin_uploader
+        run: pip install -r requirements_test.txt
+
+      - working-directory: resources/plugin_uploader
+        run: pytest -vv .
+
+  ruff-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Lint with Ruff
+        working-directory: resources/plugin_uploader
+        run: |
+          pip install ruff==0.4.10
+          ruff check --output-format=github
+
+  pyright-type-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - working-directory: resources/plugin_uploader
+        run: pip install -r requirements_test.txt
+
+      - run: pip install pyright==1.1.378
+
+      - working-directory: resources/plugin_uploader
+        run: pyright

.github/workflows/upload_plugin.yml

@@ -0,0 +1,84 @@
+name: Upload rpk connect plugin to S3
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      # All runs triggered by tag will really push to S3.
+      # Take care when adding more patterns here.
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+'
+  pull_request:
+    # Keep CI snappy for unrelated PRs
+    paths:
+      - 'resources/plugin_uploader/**'
+      - '.github/workflows/upload_plugin.yml'
+      - '.github/actions/upload_managed_plugin/**'
+      - '.goreleaser.yml'
+  workflow_dispatch: {}
+
+env:
+  # Do dry run in most cases, UNLESS the triggering event was a "tag".
+  DRY_RUN: ${{ github.ref_type != 'tag' }}
+
+jobs:
+  upload_rpk_connect_plugin:
+    # Let's make this fast by using a beefy runner.
+    runs-on: ubuntu-latest-32
+    if: ${{ github.repository == 'redpanda-data/connect' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'redpanda-data/connect') }}
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.RP_AWS_CRED_REGION }}
+          role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }}
+
+      - name: Get secrets from AWS Secrets Manager (for read/writing S3-backed plugins repo)
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+          secret-ids: |
+            ,sdlc/prod/github/rpk_plugin_publisher
+          parse-json-secrets: true
+
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.22.x
+          check-latest: true
+
+      - name: Build binaries (dry run / snapshot mode)
+        if: ${{ env.DRY_RUN != 'false' }}
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          version: 1.26.2
+          args: build --snapshot
+
+      - name: Build binaries
+        # Only one way to not dry run - see 'false'. All other cases, conservatively assume --dry-run
+        if: ${{ env.DRY_RUN == 'false' }}
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          version: 1.26.2
+          args: build
+
+      - name: Upload connect plugin to S3
+        uses: ./.github/actions/upload_managed_plugin
+        with:
+          aws_access_key_id: ${{ env.RPK_PLUGIN_PUBLISHER_AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ env.RPK_PLUGIN_PUBLISHER_AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_s3_bucket: "rpk-plugins-repo"
+          project_root_dir: ${{ github.workspace }}
+          artifacts_file: ${{ github.workspace }}/target/dist/artifacts.json
+          metadata_file: ${{ github.workspace }}/target/dist/metadata.json
+          plugin_name: "connect"
+          goos: linux,darwin
+          goarch: amd64,arm64
+          repo_hostname: rpk-plugins.redpanda.com
+          dry_run: ${{ env.DRY_RUN != 'false' }}
+

.gitignore

@@ -9,3 +9,4 @@ release_notes.md
 .idea
 .vscode
 .op
+__pycache__