diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml deleted file mode 100644 index 01cbc1cb..00000000 --- a/.github/actions/publish-pypi/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish PyPI - -inputs: - python-version: - description: Create an environment with the appropriate version of python and hatch installed - default: "3.11" - -runs: - using: composite - steps: - - name: Setup environment - uses: ./.github/actions/setup-environment - with: - python-version: ${{ inputs.python-version }} - - - name: Build artifacts - run: hatch build - shell: bash - - - name: Check artifacts - run: hatch run build:check-all - shell: bash - - - name: Publish artifacts to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml deleted file mode 100644 index 5bd6e341..00000000 --- a/.github/actions/publish-results/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish results - -inputs: - file-name: - description: File type for file name stub (e.g. "unit-tests") - required: true - python-version: - description: Create an environment with the appropriate version of python and hatch installed - required: true - source-file: - description: File to be uploaded - required: true - -runs: - using: composite - steps: - - name: Get timestamp - id: timestamp - run: echo "ts=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - shell: bash - - - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.file-name }}_${{ inputs.python-version }}-${{ steps.timestamp.outputs.ts }}.csv - path: ${{ inputs.source-file }} diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml deleted file mode 100644 index 6227c698..00000000 --- a/.github/actions/setup-environment/action.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Setup `hatch` - -inputs: - python-version: - description: Create an environment with the appropriate version of python and hatch installed - required: true - -runs: - using: composite - steps: - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python-version }} - - - name: Install hatch - run: python -m pip install hatch - shell: bash diff --git a/.github/actions/setup-postgres-linux/action.yml b/.github/actions/setup-postgres-linux/action.yml deleted file mode 100644 index 1c8fc772..00000000 --- a/.github/actions/setup-postgres-linux/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: "Set up postgres (linux)" -description: "Set up postgres service on linux vm for dbt integration tests" -runs: - using: "composite" - steps: - - shell: bash - run: | - sudo systemctl start postgresql.service - pg_isready - sudo -u postgres bash ${{ github.action_path }}/setup_db.sh diff --git a/.github/actions/setup-postgres-linux/setup_db.sh b/.github/actions/setup-postgres-linux/setup_db.sh deleted file mode 100755 index de59bf0f..00000000 --- a/.github/actions/setup-postgres-linux/setup_db.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -x -env | grep '^PG' - -# If you want to run this script for your own postgresql (run with -# docker-compose) it will look like this: -# PGHOST=127.0.0.1 PGUSER=root PGPASSWORD=password PGDATABASE=postgres \ -PGUSER="${PGUSER:-postgres}" -export PGUSER -PGPORT="${PGPORT:-5432}" -export PGPORT -PGHOST="${PGHOST:-localhost}" - -function connect_circle() { - # try to handle circleci/docker oddness - let rc=1 - while [[ $rc -eq 1 ]]; do - nc -z ${PGHOST} ${PGPORT} - let rc=$? - done - if [[ $rc -ne 0 ]]; then - echo "Fatal: Could not connect to $PGHOST" - exit 1 - fi -} - -# appveyor doesn't have 'nc', but it also doesn't have these issues -if [[ -n $CIRCLECI ]]; then - connect_circle -fi - -for i in {1..10}; do - if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then - break - fi - - echo "Waiting for postgres to be ready..." - sleep 2; -done; - -createdb dbt -psql -c "CREATE ROLE root WITH PASSWORD 'password';" -psql -c "ALTER ROLE root WITH LOGIN;" -psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" - -psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" -psql -c "ALTER ROLE noaccess WITH LOGIN;" -psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" -psql -c "CREATE ROLE dbt_test_user_1;" -psql -c "CREATE ROLE dbt_test_user_2;" -psql -c "CREATE ROLE dbt_test_user_3;" - -psql -c 'CREATE DATABASE "dbtMixedCase";' -psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;' - -set +x diff --git a/.github/actions/setup-postgres-macos/action.yml b/.github/actions/setup-postgres-macos/action.yml deleted file mode 100644 index af9a9fe1..00000000 --- a/.github/actions/setup-postgres-macos/action.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: "Set up postgres (macos)" -description: "Set up postgres service on macos vm for dbt integration tests" -runs: - using: "composite" - steps: - - shell: bash - run: | - brew services start postgresql - echo "Check PostgreSQL service is running" - i=10 - COMMAND='pg_isready' - while [ $i -gt -1 ]; do - if [ $i == 0 ]; then - echo "PostgreSQL service not ready, all attempts exhausted" - exit 1 - fi - echo "Check PostgreSQL service status" - eval $COMMAND && break - echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i" - sleep 10 - ((i--)) - done - createuser -s postgres - bash ${{ github.action_path }}/setup_db.sh diff --git a/.github/actions/setup-postgres-macos/setup_db.sh b/.github/actions/setup-postgres-macos/setup_db.sh deleted file mode 100755 index de59bf0f..00000000 --- a/.github/actions/setup-postgres-macos/setup_db.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -x -env | grep '^PG' - -# If you want to run this script for your own postgresql (run with -# docker-compose) it will look like this: -# PGHOST=127.0.0.1 PGUSER=root PGPASSWORD=password PGDATABASE=postgres \ -PGUSER="${PGUSER:-postgres}" -export PGUSER -PGPORT="${PGPORT:-5432}" -export PGPORT -PGHOST="${PGHOST:-localhost}" - -function connect_circle() { - # try to handle circleci/docker oddness - let rc=1 - while [[ $rc -eq 1 ]]; do - nc -z ${PGHOST} ${PGPORT} - let rc=$? - done - if [[ $rc -ne 0 ]]; then - echo "Fatal: Could not connect to $PGHOST" - exit 1 - fi -} - -# appveyor doesn't have 'nc', but it also doesn't have these issues -if [[ -n $CIRCLECI ]]; then - connect_circle -fi - -for i in {1..10}; do - if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then - break - fi - - echo "Waiting for postgres to be ready..." - sleep 2; -done; - -createdb dbt -psql -c "CREATE ROLE root WITH PASSWORD 'password';" -psql -c "ALTER ROLE root WITH LOGIN;" -psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" - -psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" -psql -c "ALTER ROLE noaccess WITH LOGIN;" -psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" -psql -c "CREATE ROLE dbt_test_user_1;" -psql -c "CREATE ROLE dbt_test_user_2;" -psql -c "CREATE ROLE dbt_test_user_3;" - -psql -c 'CREATE DATABASE "dbtMixedCase";' -psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;' - -set +x diff --git a/.github/actions/setup-postgres-windows/action.yml b/.github/actions/setup-postgres-windows/action.yml deleted file mode 100644 index 419b5e26..00000000 --- a/.github/actions/setup-postgres-windows/action.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: "Set up postgres (windows)" -description: "Set up postgres service on windows vm for dbt integration tests" -runs: - using: "composite" - steps: - - shell: pwsh - run: | - $pgService = Get-Service -Name postgresql* - Set-Service -InputObject $pgService -Status running -StartupType automatic - Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru - $env:Path += ";$env:PGBIN" - bash ${{ github.action_path }}/setup_db.sh diff --git a/.github/actions/setup-postgres-windows/setup_db.sh b/.github/actions/setup-postgres-windows/setup_db.sh deleted file mode 100755 index de59bf0f..00000000 --- a/.github/actions/setup-postgres-windows/setup_db.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -x -env | grep '^PG' - -# If you want to run this script for your own postgresql (run with -# docker-compose) it will look like this: -# PGHOST=127.0.0.1 PGUSER=root PGPASSWORD=password PGDATABASE=postgres \ -PGUSER="${PGUSER:-postgres}" -export PGUSER -PGPORT="${PGPORT:-5432}" -export PGPORT -PGHOST="${PGHOST:-localhost}" - -function connect_circle() { - # try to handle circleci/docker oddness - let rc=1 - while [[ $rc -eq 1 ]]; do - nc -z ${PGHOST} ${PGPORT} - let rc=$? - done - if [[ $rc -ne 0 ]]; then - echo "Fatal: Could not connect to $PGHOST" - exit 1 - fi -} - -# appveyor doesn't have 'nc', but it also doesn't have these issues -if [[ -n $CIRCLECI ]]; then - connect_circle -fi - -for i in {1..10}; do - if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then - break - fi - - echo "Waiting for postgres to be ready..." - sleep 2; -done; - -createdb dbt -psql -c "CREATE ROLE root WITH PASSWORD 'password';" -psql -c "ALTER ROLE root WITH LOGIN;" -psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" - -psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" -psql -c "ALTER ROLE noaccess WITH LOGIN;" -psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" -psql -c "CREATE ROLE dbt_test_user_1;" -psql -c "CREATE ROLE dbt_test_user_2;" -psql -c "CREATE ROLE dbt_test_user_3;" - -psql -c 'CREATE DATABASE "dbtMixedCase";' -psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;' - -set +x diff --git a/.github/workflows/lint.yml b/.github/workflows/code-quality.yml similarity index 80% rename from .github/workflows/lint.yml rename to .github/workflows/code-quality.yml index 2b336a43..5079161f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/code-quality.yml @@ -1,4 +1,4 @@ -name: Lint +name: Code Quality on: push: @@ -16,7 +16,7 @@ concurrency: cancel-in-progress: true jobs: - lint: + code-quality: name: Python 3.8 runs-on: ubuntu-latest @@ -26,15 +26,11 @@ jobs: with: persist-credentials: false - - name: Setup environment - uses: ./.github/actions/setup-environment - with: - python-version: "3.8" + - name: Setup `hatch` + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: Run linters run: hatch run lint:all - shell: bash - name: Run typechecks run: hatch run typecheck:all - shell: bash diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ec1f6e5f..af04542d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -16,7 +16,7 @@ concurrency: cancel-in-progress: true jobs: - unit: + integration: name: Python ${{ matrix.python-version }} runs-on: ubuntu-latest @@ -25,35 +25,50 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Check out repository uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Setup environment - uses: ./.github/actions/setup-environment + - name: Setup postgres + shell: bash + run: psql -f ./scripts/setup_test_database.sql + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: postgres + + - name: Setup `hatch` + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main with: python-version: ${{ matrix.python-version }} - - name: Set up postgres (linux) - if: runner.os == 'Linux' - uses: ./.github/actions/setup-postgres-linux - - - name: Set up postgres (macos) - if: runner.os == 'macOS' - uses: ./.github/actions/setup-postgres-macos - - - name: Set up postgres (windows) - if: runner.os == 'Windows' - uses: ./.github/actions/setup-postgres-windows - - name: Run integration tests run: hatch run integration-tests:all - shell: bash + env: + POSTGRES_TEST_HOST: localhost + POSTGRES_TEST_PORT: 5432 + POSTGRES_TEST_USER: root + POSTGRES_TEST_PASS: password + POSTGRES_TEST_DATABASE: dbt + POSTGRES_TEST_THREADS: 4 - name: Publish results - uses: ./.github/actions/publish-results + uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main if: always() with: source-file: "results.csv" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b92ee91..ccf805f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,23 +13,18 @@ on: permissions: read-all -defaults: - run: - shell: bash - # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}-${{ inputs.deploy-to }} cancel-in-progress: true jobs: - release: name: PyPI - ${{ inputs.deploy-to }} runs-on: ubuntu-latest environment: name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_URL }} + url: ${{ vars.PYPI_PROJECT_URL }} permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing @@ -39,7 +34,13 @@ jobs: with: persist-credentials: false + - name: Setup `hatch` + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: Build `dbt-postgres` + uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main + - name: Publish to PyPI - uses: ./.github/actions/publish-pypi + uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main with: - python-version: "3.11" + pypi-repository-url: ${{ vars.PYPI_REPOSITORY_URL }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f8a39ea8..ac42e9ac 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -31,8 +31,8 @@ jobs: with: persist-credentials: false - - name: Setup environment - uses: ./.github/actions/setup-environment + - name: Setup `hatch` + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main with: python-version: ${{ matrix.python-version }} @@ -41,7 +41,7 @@ jobs: shell: bash - name: Publish results - uses: ./.github/actions/publish-results + uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main if: always() with: source-file: "results.csv" diff --git a/pyproject.toml b/pyproject.toml index 9f73d36e..c9ad0e50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ features = [ detached = true features = ["lint"] [tool.hatch.envs.lint.scripts] -all = ["- black", "- flake8", ] +all = ["black", "flake8"] black = "python -m black ." flake8 = "python -m flake8 ." diff --git a/scripts/setup_test_database.sql b/scripts/setup_test_database.sql new file mode 100644 index 00000000..8da2c0be --- /dev/null +++ b/scripts/setup_test_database.sql @@ -0,0 +1,16 @@ +CREATE DATABASE dbt; + +CREATE ROLE root WITH PASSWORD 'password'; +ALTER ROLE root WITH LOGIN; +GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION; + +CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER; +ALTER ROLE noaccess WITH LOGIN; +GRANT CONNECT ON DATABASE dbt TO noaccess; + +CREATE ROLE dbt_test_user_1; +CREATE ROLE dbt_test_user_2; +CREATE ROLE dbt_test_user_3; + +CREATE DATABASE "dbtMixedCase"; +GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION; diff --git a/tests/functional/minimal_cli/test_minimal_cli.py b/tests/functional/minimal_cli/test_minimal_cli.py index 64bcb1f1..1fccbbd0 100644 --- a/tests/functional/minimal_cli/test_minimal_cli.py +++ b/tests/functional/minimal_cli/test_minimal_cli.py @@ -30,16 +30,6 @@ def test_deps(self, runner, project): assert "1.0.0" in result.output -class TestLS(BaseConfigProject): - def test_ls(self, runner, project): - runner.invoke(cli, ["deps"]) - ls_result = runner.invoke(cli, ["ls"]) - assert "1 seed" in ls_result.output - assert "1 model" in ls_result.output - assert "5 data tests" in ls_result.output - assert "1 snapshot" in ls_result.output - - class TestBuild(BaseConfigProject): def test_build(self, runner, project): runner.invoke(cli, ["deps"]) diff --git a/tests/functional/schema_tests/test_schema_v2_tests.py b/tests/functional/schema_tests/test_schema_v2_tests.py index 79771c50..a268f096 100644 --- a/tests/functional/schema_tests/test_schema_v2_tests.py +++ b/tests/functional/schema_tests/test_schema_v2_tests.py @@ -892,24 +892,6 @@ def test_collision_test_names_get_hash( assert test_results[1].node.unique_id in expected_unique_ids -class TestGenericTestsCollide: - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": dupe_generic_tests_collide__schema_yml, - "model_a.sql": dupe_generic_tests_collide__model_a, - } - - def test_generic_test_collision( - self, - project, - ): - """These tests collide, since only the configs differ""" - with pytest.raises(DuplicateResourceNameError) as exc: - run_dbt() - assert "dbt found two tests with the name" in str(exc.value) - - class TestGenericTestsConfigCustomMacros: @pytest.fixture(scope="class") def models(self): diff --git a/tests/functional/test_experimental_parser.py b/tests/functional/test_experimental_parser.py index 18ee8525..9064ee4f 100644 --- a/tests/functional/test_experimental_parser.py +++ b/tests/functional/test_experimental_parser.py @@ -1,7 +1,7 @@ import os +from dbt.context.providers import RefArgs from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.graph.nodes import RefArgs import pytest from tests.functional.utils import run_dbt, run_dbt_and_capture diff --git a/tests/functional/test_init.py b/tests/functional/test_init.py index 43f0fcc2..1c3133c8 100644 --- a/tests/functional/test_init.py +++ b/tests/functional/test_init.py @@ -420,55 +420,6 @@ def setup(self, project): os.remove(os.path.join(project.project_root, "dbt_project.yml")) -class TestInitInvalidProjectNameCLI(TestInitOutsideOfProjectBase): - @patch("dbt.task.init._get_adapter_plugin_names") - @patch("click.confirm") - @patch("click.prompt") - def test_init_invalid_project_name_cli( - self, mock_prompt, mock_confirm, mock_get_adapter, project_name, project - ): - manager = Mock() - manager.attach_mock(mock_prompt, "prompt") - manager.attach_mock(mock_confirm, "confirm") - - invalid_name = "name-with-hyphen" - valid_name = project_name - manager.prompt.side_effect = [valid_name] - mock_get_adapter.return_value = [project.adapter.type()] - - run_dbt(["init", invalid_name, "--skip-profile-setup"]) - manager.assert_has_calls( - [ - call.prompt("Enter a name for your project (letters, digits, underscore)"), - ] - ) - - -class TestInitInvalidProjectNamePrompt(TestInitOutsideOfProjectBase): - @patch("dbt.task.init._get_adapter_plugin_names") - @patch("click.confirm") - @patch("click.prompt") - def test_init_invalid_project_name_prompt( - self, mock_prompt, mock_confirm, mock_get_adapter, project_name, project - ): - manager = Mock() - manager.attach_mock(mock_prompt, "prompt") - manager.attach_mock(mock_confirm, "confirm") - - invalid_name = "name-with-hyphen" - valid_name = project_name - manager.prompt.side_effect = [invalid_name, valid_name] - mock_get_adapter.return_value = [project.adapter.type()] - - run_dbt(["init", "--skip-profile-setup"]) - manager.assert_has_calls( - [ - call.prompt("Enter a name for your project (letters, digits, underscore)"), - call.prompt("Enter a name for your project (letters, digits, underscore)"), - ] - ) - - class TestInitInsideProjectAndSkipProfileSetup(TestInitInsideOfProjectBase): @patch("dbt.task.init._get_adapter_plugin_names") @patch("click.confirm")