Merge pull request #2736 from cal-itp/python-ci

migrate CI to use python invoke
cal-itp · Jun 26, 2023 · 5409fb1 · 5409fb1
2 parents 9302cd5 + 1ab4613
commit 5409fb1
Show file tree

Hide file tree

Showing 45 changed files with 1,280 additions and 887 deletions.
diff --git a/.github/workflows/service-release-candidate.yml b/.github/workflows/service-release-candidate.yml
@@ -15,7 +15,7 @@ on:
 
 
 env:
-  GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }}
   BIO_RELEASE: 1.6.372
 
 
@@ -28,6 +28,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
+        token: ${{ secrets.GH_ACTIONS_TOKEN }}
 
     - name: 'Build release candidate branch'
       run: |

diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml
@@ -14,33 +14,31 @@ jobs:
       GKE_REGION: us-west1
       USE_GKE_GCLOUD_AUTH_PLUGIN: True
     steps:
-
+      # Setup
       - name: Check out repo
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-
       - uses: google-github-actions/setup-gcloud@v0
         with:
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           export_default_credentials: true
-
       - name: install auth plugin
         run: gcloud components install gke-gcloud-auth-plugin
-
       - uses: google-github-actions/get-gke-credentials@v1
         with:
           cluster_name: ${{ env.GKE_NAME }}
           location: ${{ env.GKE_REGION }}
+      - run: curl -sSL https://install.python-poetry.org | python -
 
-      - id: service-release
-        name: Run ci/workflows/service-release.sh
+      # Release to channel
+      - id: poetry-invoke
+        name: Run poetry invoke
         shell: bash
+        working-directory: ci
         run: |
-          # GITHUB_JOB is not populated until the job is running
-          git config user.name "Github Action $GITHUB_JOB"
-          git config user.email "$(whoami)@$(uname -n)"
           export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/}
-          printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL"
-          set -- "$GITHUB_WORKSPACE/ci/vars/project.env"
-          source "$GITHUB_WORKSPACE/ci/workflows/service-release.sh"
+          printf 'WORKFLOW: service-release-channel; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL"
+          poetry install
+          poetry run invoke secrets -f "./channels/$RELEASE_CHANNEL.yaml"
+          poetry run invoke release -f "./channels/$RELEASE_CHANNEL.yaml"
diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml
@@ -0,0 +1,62 @@
+name: Show diff for release channel
+
+on:
+  pull_request:
+    branches:
+      - 'releases/*'
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }}
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    env:
+      CLOUDSDK_CORE_PROJECT: cal-itp-data-infra
+      GKE_NAME: data-infra-apps
+      GKE_REGION: us-west1
+      USE_GKE_GCLOUD_AUTH_PLUGIN: True
+    steps:
+      # Setup
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+      - run: gcloud components install gke-gcloud-auth-plugin
+      - uses: google-github-actions/get-gke-credentials@v1
+        with:
+          cluster_name: ${{ env.GKE_NAME }}
+          location: ${{ env.GKE_REGION }}
+      - run: curl -sSL https://install.python-poetry.org | python -
+
+      # Diff and write back to PR
+      - id: diff
+        name: Run poetry invoke
+        shell: bash
+        working-directory: ci
+        run: |
+          export RELEASE_CHANNEL=${GITHUB_BASE_REF#releases/}
+          printf 'WORKFLOW: service-release-diff; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL"
+          poetry install
+          poetry run invoke diff -f "./channels/$RELEASE_CHANNEL.yaml" --outfile=diff.txt
+
+      - uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.number }}
+          comment-author: 'github-actions[bot]'
+          direction: last
+      - uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.number }}
+          body-file: "ci/diff.txt"
+          edit-mode: replace
diff --git a/ci/channels/prod.yaml b/ci/channels/prod.yaml
@@ -0,0 +1,91 @@
+calitp:
+  channel: prod
+  releases:
+    - name: airflow-jobs
+      driver: kustomize
+      kustomize_dir: kubernetes/apps/manifests/airflow-jobs
+      secrets:
+        - airflow-jobs_jobs-data
+    - name: grafana
+      driver: helm
+      namespace: monitoring-grafana
+      helm_name: grafana
+      helm_chart: kubernetes/apps/charts/grafana
+      helm_values:
+        - kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml
+      secrets:
+        - monitoring-grafana__grafana-initial-admin
+    - name: jupyterhub
+      driver: helm
+      namespace: jupyterhub
+      helm_name: jupyterhub
+      helm_chart: kubernetes/apps/charts/jupyterhub
+      secrets:
+        - jupyterhub_jupyterhub-gcloud-service-key
+        - jupyterhub_jupyterhub-github-config
+    - name: loki
+      driver: helm
+      namespace: monitoring-loki
+      helm_name: loki
+      helm_chart: kubernetes/apps/charts/loki
+    - name: metabase
+      driver: helm
+      namespace: metabase
+      helm_name: metabase
+      helm_chart: kubernetes/apps/charts/metabase
+      helm_values:
+        - kubernetes/apps/values/metabase.yaml
+      secrets:
+        - metabase_service_account_key
+    - name: postgresql-backup-grafana
+      driver: helm
+      namespace: grafana
+      helm_name: postgresql-backup
+      helm_chart: kubernetes/apps/charts/postgresql-backup
+      helm_values:
+        - kubernetes/apps/values/postgresql-backup-grafana.yaml
+      secrets:
+        - monitoring-grafana_database-backup
+        - monitoring-grafana_grafana-postgresql
+    - name: postgresql-backup-metabase
+      driver: helm
+      namespace: metabase
+      helm_name: postgresql-backup
+      helm_chart: kubernetes/apps/charts/postgresql-backup
+      helm_values:
+        - kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml
+      secrets:
+        - metabase_database-backup
+        - metabase_gcs-upload-svcacct
+    - name: postgresql-backup-sentry
+      driver: helm
+      namespace: sentry
+      helm_name: postgresql-backup
+      helm_chart: kubernetes/apps/charts/postgresql-backup
+      helm_values:
+        - kubernetes/apps/values/postgresql-backup-sentry.yaml
+      secrets:
+        - sentry_database-backup
+    - name: prometheus
+      driver: helm
+      namespace: monitoring-prometheus
+      helm_name: prometheus
+      helm_chart: kubernetes/apps/charts/prometheus
+      helm_values:
+        - kubernetes/apps/values/prometheus.yaml
+    - name: promtail
+      driver: helm
+      namespace: monitoring-loki
+      helm_name: promtail
+      helm_chart: kubernetes/apps/charts/promtail
+      helm_values:
+        - kubernetes/apps/values/promtail.yaml
+    - name: sftp-ingest-elavon
+      driver: kustomize
+      kustomize_dir: kubernetes/apps/overlays/prod-sftp-ingest-elavon
+      # TODO: sftp secrets
+    # TODO: Sentry
+#    - name: sentry
+#      secrets:
+#        - sentry_sentry-secret
+#        - sentry_sentry-sentry-postgresql
diff --git a/ci/channels/test.yaml b/ci/channels/test.yaml
@@ -0,0 +1,26 @@
+calitp:
+  channel: test
+  releases:
+    - name: archiver
+      driver: kustomize
+      kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test
+      secrets:
+        - gtfs-rt-v3-test_gtfs-feed-secrets
+    - name: metabase
+      driver: helm
+      namespace: metabase-test
+      helm_name: metabase-test
+      helm_chart: kubernetes/apps/charts/metabase
+      helm_values:
+        - kubernetes/apps/values/metabase-test.yaml
+    - name: postgresql-backup-metabase
+      driver: helm
+      namespace: metabase-test
+      helm_name: postgresql-backup
+      helm_chart: kubernetes/apps/charts/postgresql-backup
+      helm_values:
+        - kubernetes/apps/values/postgresql-backup-metabase.yaml
+        - kubernetes/apps/values/postgresql-backup-metabase-test.yaml
+      secrets:
+        - metabase-test_database-backup
+        - metabase-test_gcs-upload-svcacct
diff --git a/ci/invoke.yaml b/ci/invoke.yaml
@@ -0,0 +1,5 @@
+run:
+  echo: true
+  echo_format: "Executing: {command}" # without this, the echoed text is white which does not show on light themes
+calitp:
+  git_repo: .