From eb1822220c37c81c077776527e22cd03ae1b43f0 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 25 Sep 2024 20:14:18 +0200 Subject: [PATCH 01/18] Added sql-server tests for dbt 1.3.0 to 1.8.0 --- tox.ini | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 50e79ce7..ae8be247 100644 --- a/tox.ini +++ b/tox.ini @@ -355,31 +355,54 @@ commands = [testenv:integration_sqlserver] changedir = integration_test_project -deps = dbt-sqlserver~=1.8.4 +deps = dbt-sqlserver~=1.8.0 commands = dbt clean dbt deps dbt build --target sqlserver -[testenv:integration_sqlserver_1_4_3] +[testenv:integration_sqlserver_1_3_0] changedir = integration_test_project -deps = dbt-sqlserver~=1.4.3 +deps = dbt-sqlserver~=1.3.0 commands = dbt clean dbt deps dbt build --target sqlserver -[testenv:integration_sqlserver_1_7_4] +[testenv:integration_sqlserver_1_4_0] changedir = integration_test_project -deps = dbt-sqlserver~=1.7.4 +deps = dbt-sqlserver~=1.4.0 commands = dbt clean dbt deps dbt build --target sqlserver -[testenv:integration_sqlserver_1_8_4] +[testenv:integration_sqlserver_1_5_0] changedir = integration_test_project -deps = dbt-sqlserver~=1.8.4 +deps = dbt-sqlserver~=1.5.0 +commands = + dbt clean + dbt deps + dbt build --target sqlserver + +[testenv:integration_sqlserver_1_6_0] +changedir = integration_test_project +deps = dbt-sqlserver~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target sqlserver +[testenv:integration_sqlserver_1_7_0] +changedir = integration_test_project +deps = dbt-sqlserver~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target sqlserver + +[testenv:integration_sqlserver_1_8_0] +changedir = integration_test_project +deps = dbt-sqlserver~=1.8.0 commands = dbt clean dbt deps From db5e1b18e06e4602f74484125b70aa4d8da6234b Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 25 Sep 2024 21:41:20 +0200 Subject: [PATCH 02/18] Removed test support for sqlserver-1.5.0 and 1.6.0. These versions don't exist. --- tox.ini | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tox.ini b/tox.ini index ae8be247..9a67e0b2 100644 --- a/tox.ini +++ b/tox.ini @@ -321,22 +321,6 @@ commands = dbt deps dbt build --target postgres -[testenv:integration_postgres_1_5_0] -changedir = integration_test_project -deps = dbt-postgres~=1.5.0 -commands = - dbt clean - dbt deps - dbt build --target postgres - -[testenv:integration_postgres_1_6_0] -changedir = integration_test_project -deps = dbt-postgres~=1.6.0 -commands = - dbt clean - dbt deps - dbt build --target postgres - [testenv:integration_postgres_1_7_0] changedir = integration_test_project deps = dbt-postgres~=1.7.0 From 9acfe07eb5b30159ef0ca6abf028078b6763a9a7 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 25 Sep 2024 21:51:18 +0200 Subject: [PATCH 03/18] Readded postgres tests. They were accidentally removed. --- tox.ini | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tox.ini b/tox.ini index 9a67e0b2..d085e43d 100644 --- a/tox.ini +++ b/tox.ini @@ -321,6 +321,22 @@ commands = dbt deps dbt build --target postgres +[testenv:integration_postgres_1_5_0] +changedir = integration_test_project +deps = dbt-postgres~=1.5.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_6_0] +changedir = integration_test_project +deps = dbt-postgres~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + [testenv:integration_postgres_1_7_0] changedir = integration_test_project deps = dbt-postgres~=1.7.0 @@ -361,21 +377,6 @@ commands = dbt deps dbt build --target sqlserver -[testenv:integration_sqlserver_1_5_0] -changedir = integration_test_project -deps = dbt-sqlserver~=1.5.0 -commands = - dbt clean - dbt deps - dbt build --target sqlserver - -[testenv:integration_sqlserver_1_6_0] -changedir = integration_test_project -deps = dbt-sqlserver~=1.6.0 -commands = - dbt clean - dbt deps - dbt build --target sqlserver [testenv:integration_sqlserver_1_7_0] changedir = integration_test_project deps = dbt-sqlserver~=1.7.0 From b8c2c7132fa751a7a9ea39420a08c6471e108bfa Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 25 Sep 2024 22:35:12 +0200 Subject: [PATCH 04/18] Squash merge workflow_tests into fix_test_package --- .github/workflows/ci_test_package.yml | 45 ++++++++++++++++++++- .github/workflows/main_test_package.yml | 52 +++++++++++++++++++++---- integration_test_project/profiles.yml | 2 +- 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 5eb359fd..c235dbf2 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -113,7 +113,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] + warehouse: ["snowflake", "bigquery", "postgres"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest @@ -193,3 +193,46 @@ jobs: # env: # DBT_VERSION: '' # run: tox -e integration_databricks + + integration-sqlserver: + strategy: + fail-fast: false # Don't fail one DWH if the others fail + matrix: + # When supporting a new version, update the list here + version: ["1_3_0", "1_4_0", "1_7_0", "1_8_0"] + runs-on: ubuntu-latest + environment: + name: Approve Integration Tests + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.8.x" + architecture: "x64" + - name: Install SQL Server + uses: Particular/install-sql-server-action@v1.2.0 + with: + connection-string-env-var: SQL_SERVER_CONNECTION_STRING + catalog: dbt_artifact_integrationtests + - name: Create DBT User + shell: pwsh + run: | + echo "Create dbt login with sysadmin" + sqlcmd -Q "CREATE LOGIN dbt WITH PASSWORD = '123', CHECK_POLICY = OFF, CHECK_EXPIRATION = OFF" -d "dbt_artifact_integrationtests" + sqlcmd -Q "ALTER SERVER ROLE sysadmin ADD MEMBER dbt" -d "dbt_artifact_integrationtests" + + - name: Install tox + run: python3 -m pip install tox + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + + - name: Run Tests on PR + env: + DBT_VERSION: ${{ matrix.version }} + run: tox -e integration_sqlserver_${{ matrix.version }} diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 1c55194e..c0407572 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,7 +34,7 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery", "postgres", "sqlserver"] + warehouse: ["snowflake", "bigquery", "postgres"] version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0", "1_7_0", "1_8_0"] runs-on: ubuntu-latest permissions: @@ -66,12 +66,6 @@ jobs: - name: Install tox run: python3 -m pip install tox - - name: Install SQL Server - run: docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=123" -p 1433:1433 -d mcr.microsoft.com/mssql/server:2022-latest - - - name: Install Microsoft ODBC - run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y - - id: auth if: ${{ matrix.warehouse == 'bigquery' }} uses: google-github-actions/auth@v1 @@ -102,3 +96,47 @@ jobs: # - name: Run Databricks Tests # run: tox -e integration_databricks + + integration-sqlserver: + strategy: + fail-fast: false # Don't fail one DWH if the others fail + matrix: + # When supporting a new version, update the list here + version: ["1_3_0", "1_4_0", "1_7_0", "1_8_0"] + runs-on: ubuntu-latest + environment: + name: Approve Integration Tests + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.8.x" + architecture: "x64" + - name: Install SQL Server + uses: Particular/install-sql-server-action@v1.2.0 + with: + connection-string-env-var: SQL_SERVER_CONNECTION_STRING + catalog: dbt_artifact_integrationtests + - name: Create DBT User + shell: pwsh + run: | + echo "Create dbt login with sysadmin" + sqlcmd -Q "CREATE LOGIN dbt WITH PASSWORD = '123', CHECK_POLICY = OFF, CHECK_EXPIRATION = OFF" -d "dbt_artifact_integrationtests" + sqlcmd -Q "ALTER SERVER ROLE sysadmin ADD MEMBER dbt" -d "dbt_artifact_integrationtests" + + - name: Install tox + run: python3 -m pip install tox + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + + - name: Run Tests on PR + env: + DBT_VERSION: ${{ matrix.version }} + run: tox -e integration_sqlserver_${{ matrix.version }} + diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index 33a1117a..c5188699 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -61,5 +61,5 @@ dbt_artifacts: schema: dbo windows_login: False trust_cert: True - user: sa + user: dbt password: "123" From 4a68ef82a6653da5323a66c5e50116c3058b14c3 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 25 Sep 2024 22:45:43 +0200 Subject: [PATCH 05/18] Added latest changes from own test branch --- .github/workflows/ci_test_package.yml | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index c235dbf2..a923d497 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -195,6 +195,46 @@ jobs: # run: tox -e integration_databricks integration-sqlserver: + strategy: + fail-fast: false # Don't fail one DWH if the others fail + runs-on: ubuntu-latest + environment: + name: Approve Integration Tests + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.8.x" + architecture: "x64" + - name: Install SQL Server + uses: Particular/install-sql-server-action@v1.2.0 + with: + connection-string-env-var: SQL_SERVER_CONNECTION_STRING + catalog: dbt_artifact_integrationtests + - name: Create DBT User + shell: pwsh + run: | + echo "Create dbt login with sysadmin" + sqlcmd -Q "CREATE LOGIN dbt WITH PASSWORD = '123', CHECK_POLICY = OFF, CHECK_EXPIRATION = OFF" -d "dbt_artifact_integrationtests" + sqlcmd -Q "ALTER SERVER ROLE sysadmin ADD MEMBER dbt" -d "dbt_artifact_integrationtests" + + - name: Install tox + run: python3 -m pip install tox + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR + + - name: Run Tests on PR + env: + DBT_VERSION: ${{ matrix.version }} + run: tox -e integration_sqlserver + + integration-sqlserver-single-run: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: From 2ef4c98b842fd9436116d6417fc6325e69f4cfb7 Mon Sep 17 00:00:00 2001 From: michelley-an <101600153+michelley-an@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:57:41 -0700 Subject: [PATCH 06/18] Update integration_test_project/profiles.yml --- integration_test_project/profiles.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index c5188699..18a6bd15 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -63,3 +63,4 @@ dbt_artifacts: trust_cert: True user: dbt password: "123" + encrypt: False From f1562954c73f0f102bdc81de71a4316b03dfb6c9 Mon Sep 17 00:00:00 2001 From: michelley-an <101600153+michelley-an@users.noreply.github.com> Date: Tue, 1 Oct 2024 21:06:14 -0700 Subject: [PATCH 07/18] Update integration_test_project/profiles.yml --- integration_test_project/profiles.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index 18a6bd15..c5188699 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -63,4 +63,3 @@ dbt_artifacts: trust_cert: True user: dbt password: "123" - encrypt: False From af2060464f40d88c63f149b4f064b2860190b59c Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 2 Oct 2024 20:56:58 +0200 Subject: [PATCH 08/18] Set correct postgres dbt version for 1_8_0 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d085e43d..2c0e595e 100644 --- a/tox.ini +++ b/tox.ini @@ -347,7 +347,7 @@ commands = [testenv:integration_postgres_1_8_0] changedir = integration_test_project -deps = dbt-postgres~=1.7.0 +deps = dbt-postgres~=1.8.0 commands = dbt clean dbt deps From 28fad23aa9742e9d5376f00b04f78f5da5b605d6 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 2 Oct 2024 21:05:51 +0200 Subject: [PATCH 09/18] Added dbt-core 1.8.0 requirement for postgres --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 2c0e595e..33215226 100644 --- a/tox.ini +++ b/tox.ini @@ -299,7 +299,7 @@ commands = [testenv:integration_postgres] changedir = integration_test_project -deps = dbt-postgres~=1.8.0 +deps = dbt-postgres~=1.8.0, dbt-core=~1.8.0 commands = dbt clean dbt deps @@ -347,7 +347,7 @@ commands = [testenv:integration_postgres_1_8_0] changedir = integration_test_project -deps = dbt-postgres~=1.8.0 +deps = dbt-postgres~=1.8.0, dbt-core~=1.8.0 commands = dbt clean dbt deps From 4ef20a884ba86033e9375b4a6c362b0e151fa041 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 2 Oct 2024 21:09:34 +0200 Subject: [PATCH 10/18] Different formatting --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 33215226..21f7a2b1 100644 --- a/tox.ini +++ b/tox.ini @@ -299,7 +299,9 @@ commands = [testenv:integration_postgres] changedir = integration_test_project -deps = dbt-postgres~=1.8.0, dbt-core=~1.8.0 +deps = + dbt-postgres~=1.8.0 + dbt-core=~1.8.0 commands = dbt clean dbt deps From 8a2fec3aa4a49f0fcfd9d07fe1b38e971ebd0e92 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 2 Oct 2024 21:14:34 +0200 Subject: [PATCH 11/18] Fixed typo --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 21f7a2b1..6faf721a 100644 --- a/tox.ini +++ b/tox.ini @@ -301,7 +301,7 @@ commands = changedir = integration_test_project deps = dbt-postgres~=1.8.0 - dbt-core=~1.8.0 + dbt-core~=1.8.0 commands = dbt clean dbt deps From ba658b3a6365a37a76672b88d85933eeb136be65 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 2 Oct 2024 21:23:14 +0200 Subject: [PATCH 12/18] Moved deps around --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6faf721a..4b6203cf 100644 --- a/tox.ini +++ b/tox.ini @@ -300,8 +300,8 @@ commands = [testenv:integration_postgres] changedir = integration_test_project deps = - dbt-postgres~=1.8.0 dbt-core~=1.8.0 + dbt-postgres~=1.8.0 commands = dbt clean dbt deps From b5e1321017707412acf2a6081ddcae78c28678f9 Mon Sep 17 00:00:00 2001 From: KijkEr Date: Wed, 2 Oct 2024 21:25:51 +0200 Subject: [PATCH 13/18] Set deps for postgres_1_8_0 --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4b6203cf..d865318c 100644 --- a/tox.ini +++ b/tox.ini @@ -349,7 +349,9 @@ commands = [testenv:integration_postgres_1_8_0] changedir = integration_test_project -deps = dbt-postgres~=1.8.0, dbt-core~=1.8.0 +deps = + dbt-core~=1.8.0 + dbt-postgres~=1.8.0 commands = dbt clean dbt deps From f859b4169a379d84f8dd53f273429b5314a912ba Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Fri, 4 Oct 2024 12:52:39 -0400 Subject: [PATCH 14/18] Update tox.ini to exclude 1.9 for postgres --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d865318c..a64f4de7 100644 --- a/tox.ini +++ b/tox.ini @@ -300,7 +300,7 @@ commands = [testenv:integration_postgres] changedir = integration_test_project deps = - dbt-core~=1.8.0 + dbt-core<1.9.0 dbt-postgres~=1.8.0 commands = dbt clean @@ -350,7 +350,7 @@ commands = [testenv:integration_postgres_1_8_0] changedir = integration_test_project deps = - dbt-core~=1.8.0 + dbt-core<1.9.0 dbt-postgres~=1.8.0 commands = dbt clean From 8e55b4c7dc651a47bfcf54c4cf428c87e707b6c2 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Fri, 4 Oct 2024 13:01:02 -0400 Subject: [PATCH 15/18] tox.ini to lock dbt-core version for dbt-postgres~=1.8.0 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a64f4de7..140288bd 100644 --- a/tox.ini +++ b/tox.ini @@ -300,7 +300,7 @@ commands = [testenv:integration_postgres] changedir = integration_test_project deps = - dbt-core<1.9.0 + dbt-core==1.8.7 dbt-postgres~=1.8.0 commands = dbt clean @@ -350,7 +350,7 @@ commands = [testenv:integration_postgres_1_8_0] changedir = integration_test_project deps = - dbt-core<1.9.0 + dbt-core==1.8.7 dbt-postgres~=1.8.0 commands = dbt clean From 31c7aec2d5dc5c6edcc4c48b76da69525d8d61cb Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Fri, 4 Oct 2024 13:41:15 -0400 Subject: [PATCH 16/18] force recreation on tox --- .github/workflows/ci_test_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index a923d497..7adbe8e9 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -90,7 +90,7 @@ jobs: - name: Build tables for latest release env: DBT_VERSION: "noversion" - run: tox -e integration_${{ matrix.warehouse }} + run: tox -r -e integration_${{ matrix.warehouse }} - name: Checkout uses: actions/checkout@v3 From 7b44f8c1a91765b998c814fc915c333a6c74c780 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Fri, 4 Oct 2024 13:47:29 -0400 Subject: [PATCH 17/18] Update ci_test_package.yml --- .github/workflows/ci_test_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 7adbe8e9..a923d497 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -90,7 +90,7 @@ jobs: - name: Build tables for latest release env: DBT_VERSION: "noversion" - run: tox -r -e integration_${{ matrix.warehouse }} + run: tox -e integration_${{ matrix.warehouse }} - name: Checkout uses: actions/checkout@v3 From db2f8b1edf090fe976004b67abb394bde7776540 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Fri, 4 Oct 2024 13:48:05 -0400 Subject: [PATCH 18/18] Update tox.ini --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 140288bd..d865318c 100644 --- a/tox.ini +++ b/tox.ini @@ -300,7 +300,7 @@ commands = [testenv:integration_postgres] changedir = integration_test_project deps = - dbt-core==1.8.7 + dbt-core~=1.8.0 dbt-postgres~=1.8.0 commands = dbt clean @@ -350,7 +350,7 @@ commands = [testenv:integration_postgres_1_8_0] changedir = integration_test_project deps = - dbt-core==1.8.7 + dbt-core~=1.8.0 dbt-postgres~=1.8.0 commands = dbt clean