Skip to content

Adding support for Amazon Athena #535

Adding support for Amazon Athena

Adding support for Amazon Athena #535

Workflow file for this run

name: CI test package
on:
workflow_dispatch:
# all PRs, important to note that `pull_request_target` workflows
# will run in the context of the target branch of a PR
pull_request_target:
env:
# These are configured in GitHub secrets
DBT_PROFILES_DIR: ${{ github.workspace }}/integration_test_project
GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
DBT_ENV_SECRET_DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
DBT_ENV_SECRET_DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
DBT_ENV_SECRET_GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
# Env var to test version
LAST_RELEASE_SUPPORTED_DBT_VERSION: 1_7_0 # A dbt version supported by both the last release and this one
# Env vars to test invocations model
DBT_CLOUD_PROJECT_ID: 123
DBT_CLOUD_JOB_ID: ABC
DBT_CLOUD_RUN_REASON: "String with 'quotes' !"
TEST_ENV_VAR_1: TEST_VALUE
TEST_ENV_VAR_NUMBER: 3
TEST_ENV_VAR_EMPTY: ""
TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG."
DBT_ENV_CUSTOM_ENV_FAVOURITE_DBT_PACKAGE: dbt_artifacts
jobs:
integration:
strategy:
fail-fast: false # Don't fail one DWH if the others fail
matrix:
warehouse: ["snowflake", "bigquery", "postgres"]
runs-on: ubuntu-latest
environment:
name: Approve Integration Tests
permissions:
contents: "read"
id-token: "write"
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Get latest release
uses: rez0n/actions-github-release@main
id: latest_release
env:
token: ${{ secrets.GITHUB_TOKEN }}
repository: ${{ github.repository }}
type: "stable"
- name: Checkout latest release
uses: actions/checkout@v3
with:
ref: ${{ steps.latest_release.outputs.release }}
- uses: actions/setup-python@v4
with:
python-version: "3.8.x"
architecture: "x64"
- name: Install tox
run: python3 -m pip install tox
- id: auth-on-release
if: ${{ matrix.warehouse == 'bigquery' }}
uses: google-github-actions/auth@v1
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Build tables for latest release
env:
DBT_VERSION: "noversion"
run: tox -e integration_${{ matrix.warehouse }}
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
# Need to get the auth file again
- id: auth-on-pr
if: ${{ matrix.warehouse == 'bigquery' }}
uses: google-github-actions/auth@v1
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Run Tests on PR
run: tox -e integration_${{ matrix.warehouse }}_${{ env.LAST_RELEASE_SUPPORTED_DBT_VERSION }}
single-run-different-versions:
needs: integration
strategy:
fail-fast: false # Don't fail one DWH if the others fail
matrix:
warehouse: ["snowflake", "bigquery", "postgres"]
# When supporting a new version, update the list here
version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0"]
runs-on: ubuntu-latest
environment:
name: Approve Integration Tests
permissions:
contents: "read"
id-token: "write"
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- uses: actions/setup-python@v4
with:
python-version: "3.8.x"
architecture: "x64"
- name: Install tox
run: python3 -m pip install tox
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
# Need to get the auth file again
- id: auth-on-pr
if: ${{ matrix.warehouse == 'bigquery' }}
uses: google-github-actions/auth@v1
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Run Tests on PR
env:
DBT_VERSION: ${{ matrix.version }}
run: tox -e integration_${{ matrix.warehouse }}_${{ matrix.version }}
# Databricks doesn't like the matrix strategy, so moving back to the old integration testing without versioning
# integration-databricks:
# runs-on: ubuntu-latest
# environment:
# name: Approve Integration Tests
# steps:
# - name: Checkout
# uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
# - uses: actions/setup-python@v4
# with:
# python-version: '3.8.x'
# architecture: 'x64'
# - name: Install tox
# run: python3 -m pip install tox
# - name: Run Databricks Tests
# env:
# DBT_VERSION: ''
# run: tox -e integration_databricks