From cb88ce82954dbe1d4b6cbe41860be9f56449cd69 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 22 Jul 2024 18:20:43 -0400 Subject: [PATCH 01/10] Update integration.yml with git_adapter_branch and name --- .github/scripts/update_dev_packages.sh | 17 ++++++++++ .github/workflows/integration.yml | 45 ++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 7 deletions(-) create mode 100755 .github/scripts/update_dev_packages.sh diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_packages.sh new file mode 100755 index 000000000..fb26c39b5 --- /dev/null +++ b/.github/scripts/update_dev_packages.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e +set -e + + +adapters_git_branch=$1 +core_git_branch=$2 +target_req_file="dev-requirements.txt" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git.*#egg=dbt-adapters|dbt-adapters.git@${adapters_git_branch}#egg=dbt-adapters|g" +if [[ "$OSTYPE" == darwin* ]]; then + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$adapters_req_sed_pattern" $target_req_file +else + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$adapters_req_sed_pattern" $target_req_file +fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1e058aa49..bb991de4b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,6 +10,8 @@ name: Adapter Integration Tests +run-name: ${{ inputs.name }} by @${{ github.actor }} + on: push: branches: @@ -24,10 +26,38 @@ on: workflow_dispatch: inputs: - dbt-core-branch: + dbt_core_branch: + description: "branch of dbt-core to use in dev-requirements.txt" + required: false + type: string + default: main + name: + description: "name to associate with run" + required: false + type: string + default: "Adapter Integration Tests" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: main + workflow_call: + inputs: + dbt_core_branch: description: "branch of dbt-core to use in dev-requirements.txt" required: false type: string + default: main + name: + description: "name to associate with run" + required: false + type: string + default: "Adapter Integration Tests" + dbt_adapters_branch: + description: "The branch of dbt-adapters to use" + type: string + required: false + default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -95,17 +125,18 @@ jobs: with: python-version: "3.11" + - name: Update Adapters and Core branches (update dev_requirements.txt) + if: ${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }} + run: | + ./.github/scripts/update_dev_packages.sh \ + ${{ inputs.dbt_adapters_branch }} \ + ${{ inputs.dbt_core_branch }} + - name: Install python dependencies run: | python -m pip install --user --upgrade pip python -m pip --version python -m pip install -r dagger/requirements.txt - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - name: Run tests for ${{ matrix.test }} run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }} From 515e1cee4025b12b07acfb003b0e1fddb7619d77 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 23 Jul 2024 16:49:05 -0400 Subject: [PATCH 02/10] Add adapter_branch --- .github/workflows/integration.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bb991de4b..85630056c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ name: Adapter Integration Tests -run-name: ${{ inputs.name }} by @${{ github.actor }} +run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ contains(github.event_name, 'workflow_') && inputs.adapter_branch || github.ref_name }} by @${{ github.actor }}" on: push: @@ -41,6 +41,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main workflow_call: inputs: dbt_core_branch: @@ -58,6 +63,11 @@ on: type: string required: false default: main + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -105,10 +115,19 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check out the repository (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' uses: actions/checkout@v4 with: persist-credentials: false + # We can't actually change branch until this is in main, because it + # won't find the update_dev_packages.sh script. + # ref: ${{ inputs.adapter_branch }} # explicitly checkout the branch for the PR, # this is necessary for the `pull_request` event From 3a3a3afc6bc57961f5c9ab82d3a4ed03621d708b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 10:52:34 -0400 Subject: [PATCH 03/10] Update update_dev_packages.sh --- .github/scripts/update_dev_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_packages.sh index fb26c39b5..7fdd47c35 100755 --- a/.github/scripts/update_dev_packages.sh +++ b/.github/scripts/update_dev_packages.sh @@ -6,7 +6,7 @@ adapters_git_branch=$1 core_git_branch=$2 target_req_file="dev-requirements.txt" core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" -adapters_req_sed_pattern="s|dbt-adapters.git.*#egg=dbt-adapters|dbt-adapters.git@${adapters_git_branch}#egg=dbt-adapters|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg sed -i "" "$core_req_sed_pattern" $target_req_file From 4c99ac08ac1ec4a96f6a4962f2f9c0ba5e676c35 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 16:58:16 -0400 Subject: [PATCH 04/10] cat dev-requirements.txt --- .github/workflows/integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 85630056c..d334decfb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -150,6 +150,7 @@ jobs: ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ ${{ inputs.dbt_core_branch }} + cat dev-requirements.txt - name: Install python dependencies run: | From b01a8b8b8f28588f65128623f97e5d4f7b88fee1 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 24 Jul 2024 17:48:35 -0400 Subject: [PATCH 05/10] Remove workflow_call --- .github/workflows/integration.yml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d334decfb..fd5da3426 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -46,28 +46,6 @@ on: type: string required: false default: main - workflow_call: - inputs: - dbt_core_branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - default: main - name: - description: "name to associate with run" - required: false - type: string - default: "Adapter Integration Tests" - dbt_adapters_branch: - description: "The branch of dbt-adapters to use" - type: string - required: false - default: main - adapter_branch: - description: "The branch of this adapter repository to use" - type: string - required: false - default: main # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -115,13 +93,13 @@ jobs: steps: - name: Check out the repository - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' + if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false - name: Check out the repository (workflow_dispatch) - if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v4 with: persist-credentials: false @@ -145,7 +123,7 @@ jobs: python-version: "3.11" - name: Update Adapters and Core branches (update dev_requirements.txt) - if: ${{ github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' }} + if: github.event_name == 'workflow_dispatch' run: | ./.github/scripts/update_dev_packages.sh \ ${{ inputs.dbt_adapters_branch }} \ From 7a0b44c9ff4fb49b1ebfe4bdece25ab0dba83aae Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 25 Jul 2024 17:05:13 -0400 Subject: [PATCH 06/10] Remove update_dbt_core_branch --- .github/scripts/update_dbt_core_branch.sh | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100755 .github/scripts/update_dbt_core_branch.sh diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index 1a5a5c2d7..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major From 13c23f01b702dd5ee61a9d291a5a6d7106488017 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 25 Jul 2024 17:09:36 -0400 Subject: [PATCH 07/10] use adapter_branch in repo checkout --- .github/workflows/integration.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fd5da3426..61c856c9f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -103,9 +103,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - # We can't actually change branch until this is in main, because it - # won't find the update_dev_packages.sh script. - # ref: ${{ inputs.adapter_branch }} + ref: ${{ inputs.adapter_branch }} # explicitly checkout the branch for the PR, # this is necessary for the `pull_request` event From 7496c45a2219e752e1ef5eb498dcc14549b557b0 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 11:22:50 -0400 Subject: [PATCH 08/10] Change script name and add dbt-common --- ...s.sh => update_dev_dependency_branches.sh} | 16 ++++++---- .github/workflows/integration.yml | 32 +++++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) rename .github/scripts/{update_dev_packages.sh => update_dev_dependency_branches.sh} (62%) diff --git a/.github/scripts/update_dev_packages.sh b/.github/scripts/update_dev_dependency_branches.sh similarity index 62% rename from .github/scripts/update_dev_packages.sh rename to .github/scripts/update_dev_dependency_branches.sh index 7fdd47c35..022df6a8a 100755 --- a/.github/scripts/update_dev_packages.sh +++ b/.github/scripts/update_dev_dependency_branches.sh @@ -2,16 +2,20 @@ set -e -adapters_git_branch=$1 -core_git_branch=$2 +dbt_adapters_branch=$1 +dbt_core_branch=$2 +dbt_common_branch=$3 target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${core_git_branch}#egg=dbt-core|g" -adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${adapters_git_branch}|g" +core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g" +adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g" +common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g" if [[ "$OSTYPE" == darwin* ]]; then # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file sed -i "" "$adapters_req_sed_pattern" $target_req_file + sed -i "" "$core_req_sed_pattern" $target_req_file + sed -i "" "$common_req_sed_pattern" $target_req_file else - sed -i "$core_req_sed_pattern" $target_req_file sed -i "$adapters_req_sed_pattern" $target_req_file + sed -i "$core_req_sed_pattern" $target_req_file + sed -i "$common_req_sed_pattern" $target_req_file fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 61c856c9f..111f44266 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ name: Adapter Integration Tests -run-name: "${{ contains(github.event_name, 'workflow_') && inputs.name || github.event_name }}: ${{ contains(github.event_name, 'workflow_') && inputs.adapter_branch || github.ref_name }} by @${{ github.actor }}" +run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}" on: push: @@ -26,26 +26,31 @@ on: workflow_dispatch: inputs: - dbt_core_branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - default: main name: - description: "name to associate with run" + description: "Name to associate with run (example: 'dbt-adapters-242')" required: false type: string default: "Adapter Integration Tests" + adapter_branch: + description: "The branch of this adapter repository to use" + type: string + required: false + default: main dbt_adapters_branch: description: "The branch of dbt-adapters to use" type: string required: false - default: main - adapter_branch: - description: "The branch of this adapter repository to use" + default: "main" + dbt_core_branch: + description: "The branch of dbt-core to use" type: string required: false - default: main + default: "main" + dbt_common_branch: + description: "The branch of dbt-common to use" + type: string + required: false + default: "main" # explicitly turn off permissions for `GITHUB_TOKEN` permissions: read-all @@ -123,9 +128,10 @@ jobs: - name: Update Adapters and Core branches (update dev_requirements.txt) if: github.event_name == 'workflow_dispatch' run: | - ./.github/scripts/update_dev_packages.sh \ + ./.github/scripts/update_dev_dependency_branches.sh \ ${{ inputs.dbt_adapters_branch }} \ - ${{ inputs.dbt_core_branch }} + ${{ inputs.dbt_core_branch }} \ + ${{ inputs.dbt_common_branch }} cat dev-requirements.txt - name: Install python dependencies From d2d60d295e9947e258ea4c382672fc94f2589893 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 13:51:39 -0400 Subject: [PATCH 09/10] Consistent quotes for defaults --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 111f44266..a77075ee6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -35,7 +35,7 @@ on: description: "The branch of this adapter repository to use" type: string required: false - default: main + default: "main" dbt_adapters_branch: description: "The branch of dbt-adapters to use" type: string From a067918910f9aaea55102422986318f1eacf3c23 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 29 Jul 2024 15:10:41 -0400 Subject: [PATCH 10/10] Tweak if for repository checkout --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a77075ee6..699d45391 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -97,8 +97,8 @@ jobs: DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' && github.event_name != 'workflow_dispatch' + - name: Check out the repository (push) + if: github.event_name == 'push' uses: actions/checkout@v4 with: persist-credentials: false