From 8d5cef357f9ea22a5a7362069452c610c27a5efa Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 10 Dec 2024 19:03:14 -0500 Subject: [PATCH] ADAP-1125: Update the repo structure to migrate to a monorepo (#764) Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .github/workflows/_integration-tests.yml | 81 ++++++++++++ .github/workflows/code-quality.yml | 39 ------ .github/workflows/integration-tests.yml | 64 ---------- .github/workflows/publish-internal.yml | 102 --------------- .github/workflows/publish-pypi.yml | 85 ------------- .github/workflows/publish.yml | 113 ++++++++--------- .github/workflows/pull-request-checks.yml | 118 +++++++++++------- .github/workflows/unit-tests.yml | 54 -------- dbt-athena-community/hatch.toml | 56 +++++++++ dbt-athena-community/pyproject.toml | 66 +--------- dbt-athena/.changes/0.0.0.md | 5 + dbt-athena/.changes/header.tpl.md | 7 ++ dbt-athena/.changes/unreleased/.gitkeep | 0 dbt-athena/.changie.yaml | 144 ++++++++++++++++++++++ dbt-athena/CHANGELOG.md | 0 dbt-athena/CONTRIBUTING.md | 0 dbt-athena/hatch.toml | 57 +++++++++ dbt-athena/pyproject.toml | 69 ++--------- dbt-athena/tests/functional/__init__.py | 0 19 files changed, 496 insertions(+), 564 deletions(-) create mode 100644 .github/workflows/_integration-tests.yml delete mode 100644 .github/workflows/code-quality.yml delete mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/publish-internal.yml delete mode 100644 .github/workflows/publish-pypi.yml delete mode 100644 .github/workflows/unit-tests.yml create mode 100644 dbt-athena-community/hatch.toml create mode 100644 dbt-athena/.changes/0.0.0.md create mode 100644 dbt-athena/.changes/header.tpl.md create mode 100644 dbt-athena/.changes/unreleased/.gitkeep create mode 100644 dbt-athena/.changie.yaml create mode 100644 dbt-athena/CHANGELOG.md create mode 100644 dbt-athena/CONTRIBUTING.md create mode 100644 dbt-athena/hatch.toml create mode 100644 dbt-athena/tests/functional/__init__.py diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml new file mode 100644 index 00000000..943fa7d8 --- /dev/null +++ b/.github/workflows/_integration-tests.yml @@ -0,0 +1,81 @@ +name: "Integration tests" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to test" + type: string + default: "dbt-athena" + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-athena" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: string + default: ${{ vars.DEFAULT_PYTHON_VERSION }} + workflow_dispatch: + inputs: + package: + description: "Choose the package to test" + type: choice + options: ["dbt-athena", "dbt-athena-community"] + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-athena" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: choice + options: ["3.9", "3.10", "3.11", "3.12"] + +permissions: + id-token: write + contents: read + +env: + DBT_TEST_ATHENA_S3_STAGING_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/staging/ + DBT_TEST_ATHENA_S3_TMP_TABLE_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/tmp_tables/ + DBT_TEST_ATHENA_REGION_NAME: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} + DBT_TEST_ATHENA_DATABASE: awsdatacatalog + DBT_TEST_ATHENA_SCHEMA: dbt-tests + DBT_TEST_ATHENA_WORK_GROUP: athena-dbt-tests + DBT_TEST_ATHENA_THREADS: 16 + DBT_TEST_ATHENA_POLL_INTERVAL: 0.5 + DBT_TEST_ATHENA_NUM_RETRIES: 3 + +jobs: + integration-tests: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: pypa/hatch@install + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }} + aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} + - run: hatch run integration-tests + working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 425940da..00000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Code quality" - -on: - workflow_call: - inputs: - branch: - description: "Choose the branch to check" - type: string - default: "main" - repository: - description: "Choose the repository to check, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - workflow_dispatch: - inputs: - branch: - description: "Choose the branch to check" - type: string - default: "main" - repository: - description: "Choose the repository to check, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - -permissions: - contents: read - -jobs: - code-quality: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - repository: ${{ inputs.repository }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 33ab6073..00000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Integration tests" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to test" - type: string - default: "dbt-athena" - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - workflow_dispatch: - inputs: - package: - description: "Choose the package to test" - type: choice - options: ["dbt-athena", "dbt-athena-community"] - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - -permissions: - id-token: write - contents: read - -jobs: - integration-tests: - runs-on: ubuntu-latest - env: - DBT_TEST_ATHENA_S3_STAGING_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/staging/ - DBT_TEST_ATHENA_S3_TMP_TABLE_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/tmp_tables/ - DBT_TEST_ATHENA_REGION_NAME: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} - DBT_TEST_ATHENA_DATABASE: awsdatacatalog - DBT_TEST_ATHENA_SCHEMA: dbt-tests - DBT_TEST_ATHENA_WORK_GROUP: athena-dbt-tests - DBT_TEST_ATHENA_THREADS: 16 - DBT_TEST_ATHENA_POLL_INTERVAL: 0.5 - DBT_TEST_ATHENA_NUM_RETRIES: 3 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - repository: ${{ inputs.repository }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pypa/hatch@install - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }} - aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} - - run: hatch run integration-tests - working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/publish-internal.yml b/.github/workflows/publish-internal.yml deleted file mode 100644 index c7170ba8..00000000 --- a/.github/workflows/publish-internal.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: "Publish Internally" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "prod" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - workflow_dispatch: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "test" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - -defaults: - run: - shell: bash - -jobs: - publish: - runs-on: ubuntu-latest - environment: - name: ${{ inputs.deploy-to }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pypa/hatch@install - - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ vars.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - id: package - run: | - # strip the pre-release off to find all iterations of this patch - hatch version release - echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} - - id: published - run: | - versions_published="$(aws codeartifact list-package-versions \ - --domain ${{ vars.AWS_DOMAIN }} \ - --repository ${{ vars.AWS_REPOSITORY }} \ - --format pypi \ - --package ${{ inputs.package }} \ - --output json \ - --query 'versions[*].version' | jq -r '.[]' | grep "^${{ steps.package.outputs.version }}" || true )" # suppress pipefail only here - echo "versions=$(echo "${versions_published[*]}"| tr '\n' ',')" >> $GITHUB_OUTPUT - - id: next - uses: dbt-labs/dbt-release/.github/actions/next-cloud-release-version@main - with: - version_number: ${{ steps.package.outputs.version }} - versions_published: ${{ steps.published.outputs.versions }} - - name: "Update version to internal PyPI format" - run: | - VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) - tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/athena/__version__.py - working-directory: ./${{ inputs.package }} - - name: "Remove dbt-core from build requirements" - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml - working-directory: ./${{ inputs.package }} - - run: | - export HATCH_INDEX_USER=${{ secrets.AWS_USER }} - - export HATCH_INDEX_AUTH=$(aws codeartifact get-authorization-token \ - --domain ${{ vars.AWS_DOMAIN }} \ - --output text \ - --query authorizationToken) - - export HATCH_INDEX_REPO=$(aws codeartifact get-repository-endpoint \ - --domain ${{ vars.AWS_DOMAIN }} \ - --repository ${{ vars.AWS_REPOSITORY }} \ - --format pypi \ - --output text \ - --query repositoryEndpoint) - - hatch build --clean - hatch run build:check-all - hatch publish - working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml deleted file mode 100644 index 379d8b90..00000000 --- a/.github/workflows/publish-pypi.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: "Publish to PyPI" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "prod" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - workflow_dispatch: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "test" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - -permissions: - contents: read - -defaults: - run: - shell: bash - -jobs: - publish: - runs-on: ubuntu-latest - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }} - permissions: - # this permission is required for trusted publishing - # see https://github.com/marketplace/actions/pypi-publish - id-token: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pypa/hatch@install - # hatch will build using test PyPI first and fall back to prod PyPI when deploying to test - # this is done via environment variables in the test environment in GitHub - - run: hatch build && hatch run build:check-all - working-directory: ./${{ inputs.package }} - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - packages-dir: ${{ inputs.package }}/dist/ - - verify: - runs-on: ubuntu-latest - needs: publish - # check the correct index - environment: - name: ${{ inputs.deploy-to }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - id: version - run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} - - uses: nick-fields/retry@v3 - with: - timeout_seconds: 10 - retry_wait_seconds: 10 - max_attempts: 15 # 5 minutes: (10s timeout + 10s delay) * 15 attempts - command: wget ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }}/${{ steps.version.outputs.version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5fa44b21..5066f1fb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,67 +1,70 @@ name: "Publish" on: - workflow_dispatch: - inputs: - deploy-to: - description: "Choose whether to deploy to test or prod" - type: environment - default: "prod" - branch: - description: "Choose the branch to release from" - type: string - default: "main" - pypi-internal: - description: "Publish Internally" - type: boolean - default: true - pypi-public: - description: "Publish to PyPI" - type: boolean - default: false + workflow_dispatch: + inputs: + deploy-to: + description: "Choose whether to deploy to test or prod" + type: environment + default: "prod" + branch: + description: "Choose the branch to release from" + type: string + default: "main" + pypi-internal: + description: "Publish Internally" + type: boolean + default: true + pypi-public: + description: "Publish to PyPI" + type: boolean + default: false # don't attempt to release the same target in parallel concurrency: - group: ${{ github.workflow }}-${{ inputs.deploy-to }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ inputs.deploy-to }} + cancel-in-progress: true jobs: - unit-tests: - uses: ./.github/workflows/unit-tests.yml - with: - branch: ${{ inputs.branch }} + unit-tests: + uses: dbt-labs/dbt-adapters/.github/workflows/_unit-tests.yml@main + with: + package: "dbt-athena" + branch: ${{ inputs.branch }} - integration-tests: - uses: ./.github/workflows/integration-tests.yml - with: - branch: ${{ inputs.branch }} - repository: ${{ github.repository }} - secrets: inherit + integration-tests: + uses: ./.github/workflows/_integration-tests.yml + with: + branch: ${{ inputs.branch }} + repository: ${{ github.repository }} + secrets: inherit - publish-internal: - if: ${{ inputs.pypi-internal == true }} - needs: [unit-tests, integration-tests] - uses: ./.github/workflows/publish-internal.yml - with: - deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} - secrets: inherit + publish-internal: + if: ${{ inputs.pypi-internal == true }} + needs: [unit-tests, integration-tests] + uses: dbt-labs/dbt-adapters/.github/workflows/_publish-internal.yml@main + with: + package: "dbt-athena" + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} + secrets: inherit - publish-pypi: - if: ${{ inputs.pypi-public == true }} - needs: [unit-tests, integration-tests] - uses: ./.github/workflows/publish-pypi.yml - with: - deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + publish-pypi: + if: ${{ inputs.pypi-public == true }} + needs: [unit-tests, integration-tests] + uses: dbt-labs/dbt-adapters/.github/workflows/_publish-pypi.yml@main + with: + package: "dbt-athena" + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} - publish-pypi-dbt-athena-community: - if: ${{ inputs.pypi-public == true }} - # dbt-athena-community is hard pinned to dbt-athena to ensure they are the same - # this means we need to finish publishing dbt-athena before starting to build dbt-athena-community - needs: publish-pypi - uses: ./.github/workflows/publish-pypi.yml - with: - package: "dbt-athena-community" - deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + publish-pypi-dbt-athena-community: + if: ${{ inputs.pypi-public == true }} + # dbt-athena-community is hard pinned to dbt-athena to ensure they are the same + # this means we need to finish publishing dbt-athena before starting to build dbt-athena-community + needs: publish-pypi + uses: dbt-labs/dbt-adapters/.github/workflows/_publish-pypi.yml@main + with: + package: "dbt-athena-community" + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index f977dfc1..5c59171f 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,50 +1,84 @@ name: "Pull request checks" on: - pull_request_target: - types: [opened, reopened, synchronize] + pull_request_target: + types: [opened, reopened, synchronize, labeled, unlabeled] # only run this once per PR at a time concurrency: - group: ${{ github.workflow }}-${{ github.event.number }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.number }} + cancel-in-progress: true jobs: - code-quality: - uses: ./.github/workflows/code-quality.yml - with: - branch: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - - unit-tests: - uses: ./.github/workflows/unit-tests.yml - with: - package: ${{ matrix.package }} - branch: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - strategy: - matrix: - package: ["dbt-athena", "dbt-athena-community"] - - integration-tests: - uses: ./.github/workflows/integration-tests.yml - with: - # integration test runs can't run in parallel for now, so only run dbt-athena - # both will run post merge and unit tests run for both, so this is sufficient - branch: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - secrets: inherit - - # This job does nothing and is only used for branch protection - results: - name: "Pull request checks" - if: always() - needs: - - code-quality - - unit-tests - - integration-tests - runs-on: ubuntu-latest - steps: - - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} + changelog-entry: + uses: dbt-labs/dbt-adapters/.github/workflows/_changelog-entry-check.yml@main + with: + package: "dbt-athena" + pull-request: ${{ github.event.pull_request.number }} + + code-quality: + uses: dbt-labs/dbt-adapters/.github/workflows/_code-quality.yml@main + with: + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' + skip_label: 'Skip Changelog' + secrets: inherit + + verify-builds: + uses: dbt-labs/dbt-adapters/.github/workflows/_verify-build.yml@main + strategy: + matrix: + package: ["dbt-athena", "dbt-athena-community"] + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit-tests: + uses: dbt-labs/dbt-adapters/.github/workflows/_unit-tests.yml@main + strategy: + matrix: + package: ["dbt-athena", "dbt-athena-community"] + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + integration-tests: + uses: ./.github/workflows/_integration-tests.yml + strategy: + matrix: + package: ["dbt-athena", "dbt-athena-community"] + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + secrets: inherit + + # This job does nothing and is only used for branch protection + results: + name: "Pull request checks" + if: always() + needs: [code-quality, changelog, verify-builds, unit-tests, integration-tests] + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 71e5559d..00000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: "Unit tests" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to test" - type: string - default: "dbt-athena" - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - workflow_dispatch: - inputs: - package: - description: "Choose the package to test" - type: choice - options: ["dbt-athena", "dbt-athena-community"] - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - -permissions: - contents: read - -jobs: - unit-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - repository: ${{ inputs.repository }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - uses: pypa/hatch@install - - run: hatch run unit-tests - shell: bash - working-directory: ./${{ inputs.package }} diff --git a/dbt-athena-community/hatch.toml b/dbt-athena-community/hatch.toml new file mode 100644 index 00000000..f1ef474c --- /dev/null +++ b/dbt-athena-community/hatch.toml @@ -0,0 +1,56 @@ +[build.targets.sdist] +include = ["src/dbt"] + +[build.targets.wheel] +packages = ["src/dbt"] + +[envs.default] +# the only build dependency is dbt-athena, which will never be published when running this +# because the versions need to be identical +detached = true +dependencies = [ + "dbt-athena @ {root:uri}/../dbt-athena", + "dbt-tests-adapter~=1.9.2", + "isort~=5.13", + "moto~=5.0.13", + "pre-commit~=3.5", + "pyparsing~=3.1.4", + "pytest~=8.3", + "pytest-cov~=5.0", + "pytest-dotenv~=0.5", + "pytest-xdist~=3.6", +] +[envs.default.scripts] +setup = [ + "pre-commit install", + "cp -n ../dbt-athena/test.env.example test.env", +] +code-quality = "pre-commit run --all-files" +unit-tests = "pytest --cov=dbt --cov-report=html:htmlcov {args:../dbt-athena/tests/unit}" +integration-tests = "python -m pytest -n auto {args:../dbt-athena/tests/functional}" +all-tests = ["unit-tests", "integration-tests"] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_athena_community-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-athena-community", + "pip freeze | grep dbt-athena", +] +check-sdist = [ + "twine check dist/*", + "find ./dist/dbt_athena_community-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-athena-community", + "pip freeze | grep dbt-athena", +] diff --git a/dbt-athena-community/pyproject.toml b/dbt-athena-community/pyproject.toml index a6f26907..b5de59bb 100644 --- a/dbt-athena-community/pyproject.toml +++ b/dbt-athena-community/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "dbt-athena-community" description = "The athena adapter plugin for dbt (data build tool)" @@ -30,69 +34,7 @@ Documentation = "https://docs.getdbt.com" Repository = "https://github.com/dbt-labs/dbt-athena.git#subdirectory=dbt-athena" Issues = "https://github.com/dbt-labs/dbt-athena/issues" -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.sdist] -include = ["src/dbt"] - -[tool.hatch.build.targets.wheel] -packages = ["src/dbt"] - -[tool.hatch.envs.default] -# the only build dependency is dbt-athena, which will never be published when running this -# because the versions need to be identical -detached = true -dependencies = [ - "dbt-athena @ {root:uri}/../dbt-athena", - "dbt-tests-adapter~=1.9.2", - "isort~=5.13", - "moto~=5.0.13", - "pre-commit~=3.5", - "pyparsing~=3.1.4", - "pytest~=8.3", - "pytest-cov~=5.0", - "pytest-dotenv~=0.5", - "pytest-xdist~=3.6", -] -[tool.hatch.envs.default.scripts] -setup = [ - "pre-commit install", - "cp -n ../dbt-athena/test.env.example test.env", -] -code-quality = "pre-commit run --all-files" -unit-tests = "pytest --cov=dbt --cov-report=html:htmlcov {args:../dbt-athena/tests/unit}" -integration-tests = "python -m pytest -n auto {args:../dbt-athena/tests/functional}" -all-tests = ["unit-tests", "integration-tests"] - -[tool.hatch.envs.build] -detached = true -dependencies = [ - "wheel", - "twine", - "check-wheel-contents", -] -[tool.hatch.envs.build.scripts] -check-all = [ - "- check-wheel", - "- check-sdist", -] -check-wheel = [ - "check-wheel-contents dist/*.whl --ignore W007,W008", - "find ./dist/dbt_athena_community-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", - "pip freeze | grep dbt-athena-community", - "pip freeze | grep dbt-athena", -] -check-sdist = [ - "twine check dist/*", - "find ./dist/dbt_athena_community-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", - "pip freeze | grep dbt-athena-community", - "pip freeze | grep dbt-athena", -] - [tool.pytest] -env_files = ["../dbt-athena/test.env"] testpaths = [ "../dbt-athena/tests/unit", "../dbt-athena/tests/functional", diff --git a/dbt-athena/.changes/0.0.0.md b/dbt-athena/.changes/0.0.0.md new file mode 100644 index 00000000..3e0643e7 --- /dev/null +++ b/dbt-athena/.changes/0.0.0.md @@ -0,0 +1,5 @@ +# Previous Releases + +For information on prior major and minor releases, see their changelogs: + +- [1.8](https://github.com/dbt-labs/dbt-athena/blob/main/CHANGELOG.md) diff --git a/dbt-athena/.changes/header.tpl.md b/dbt-athena/.changes/header.tpl.md new file mode 100644 index 00000000..44118343 --- /dev/null +++ b/dbt-athena/.changes/header.tpl.md @@ -0,0 +1,7 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), +and is generated by [Changie](https://github.com/miniscruff/changie). diff --git a/dbt-athena/.changes/unreleased/.gitkeep b/dbt-athena/.changes/unreleased/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/dbt-athena/.changie.yaml b/dbt-athena/.changie.yaml new file mode 100644 index 00000000..649b8763 --- /dev/null +++ b/dbt-athena/.changie.yaml @@ -0,0 +1,144 @@ +changesDir: .changes +unreleasedDir: unreleased +headerPath: header.tpl.md +versionHeaderPath: "" +changelogPath: CHANGELOG.md +versionExt: md +envPrefix: "CHANGIE_" +versionFormat: '## dbt-athena {{.Version}} - {{.Time.Format "January 02, 2006"}}' +kindFormat: '### {{.Kind}}' +changeFormat: |- + {{- $IssueList := list }} + {{- $changes := splitList " " $.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-athena/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) +kinds: +- label: Breaking Changes +- label: Features +- label: Fixes +- label: Under the Hood +- label: Dependencies + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-athena/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 +- label: Security + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-athena/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 +newlines: + afterChangelogHeader: 1 + afterKind: 1 + afterChangelogVersion: 1 + beforeKind: 1 + endOfVersion: 1 +custom: +- key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 +- key: Issue + label: GitHub Issue Number (separated by a single space if multiple) + type: string + minLength: 1 +footerFormat: | + {{- /* We only want to include non-dbt contributors, so build a list of Core and Adapters maintainers for exclusion */ -}} + {{- $maintainers := list -}} + {{- $core_team := splitList " " .Env.CORE_TEAM -}} + {{- range $team_member := $core_team -}} + {{- /* ensure all names in this list are all lowercase for later matching purposes */ -}} + {{- $team_member_lower := lower $team_member -}} + {{- $maintainers = append $maintainers $team_member_lower -}} + {{- end -}} + + {{- /* Ensure we always skip dependabot */ -}} + {{- $maintainers = append $maintainers "dependabot[bot]" -}} + + {{- /* Build the list of contributors along with their PRs */ -}} + {{- $contributorDict := dict -}} + {{- range $change := .Changes -}} + {{- /* PRs can have multiple authors */ -}} + {{- $authorList := splitList " " $change.Custom.Author -}} + + {{- /* Loop through all non-dbt authors for this changelog */ -}} + {{- range $author := $authorList -}} + {{- $authorLower := lower $author -}} + {{- if not (has $authorLower $maintainers) -}} + + {{- $changeList := splitList " " $change.Custom.Author -}} + {{- $IssueList := list -}} + {{- $changeLink := $change.Kind -}} + + {{- /* Build the issue link */ -}} + {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") -}} + {{- $changes := splitList " " $change.Custom.PR -}} + {{- range $issueNbr := $changes -}} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-athena/pull/nbr)" | replace "nbr" $issueNbr -}} + {{- $IssueList = append $IssueList $changeLink -}} + {{- end -}} + + {{- else -}} + {{- $changes := splitList " " $change.Custom.Issue -}} + {{- range $issueNbr := $changes -}} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-athena/issues/nbr)" | replace "nbr" $issueNbr -}} + {{- $IssueList = append $IssueList $changeLink -}} + {{- end -}} + + {{- end -}} + + {{- /* If this contributor has other changes associated with them already, add this issue to the list */ -}} + {{- if hasKey $contributorDict $author -}} + {{- $contributionList := get $contributorDict $author -}} + {{- $contributionList = concat $contributionList $IssueList -}} + {{- $contributorDict := set $contributorDict $author $contributionList -}} + + {{- /* Otherwise create a new entry for this contributor */ -}} + {{- else -}} + {{- $contributionList := $IssueList -}} + {{- $contributorDict := set $contributorDict $author $contributionList -}} + + {{- end -}} + + {{- end -}} + {{- end -}} + {{- end -}} + + {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */ -}} + {{- if $contributorDict }} + ### Contributors + {{- range $k,$v := $contributorDict }} + - [@{{ $k }}](https://github.com/{{ $k }}) ({{ range $index, $element := $v }}{{ if $index }}, {{ end }}{{ $element }}{{ end }}) + {{- end }} + {{- end }} diff --git a/dbt-athena/CHANGELOG.md b/dbt-athena/CHANGELOG.md new file mode 100644 index 00000000..e69de29b diff --git a/dbt-athena/CONTRIBUTING.md b/dbt-athena/CONTRIBUTING.md new file mode 100644 index 00000000..e69de29b diff --git a/dbt-athena/hatch.toml b/dbt-athena/hatch.toml new file mode 100644 index 00000000..8f0e1263 --- /dev/null +++ b/dbt-athena/hatch.toml @@ -0,0 +1,57 @@ +[version] +path = "src/dbt/adapters/athena/__version__.py" + +[build.targets.sdist] +packages = ["src/dbt/adapters", "src/dbt/include"] +sources = ["src"] + +[build.targets.wheel] +packages = ["src/dbt/adapters", "src/dbt/include"] +sources = ["src"] + +[envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "moto~=5.0.13", + "pre-commit~=3.5", + "pyparsing~=3.1.4", + "pytest~=8.3", + "pytest-cov~=5.0", + "pytest-dotenv~=0.5", + "pytest-xdist~=3.6", +] +[envs.default.scripts] +setup = [ + "pre-commit install", + "cp -n test.env.example test.env", +] +code-quality = "pre-commit run --all-files" +unit-tests = "pytest --cov=dbt --cov-report=html:htmlcov {args:tests/unit}" +integration-tests = "python -m pytest -n auto {args:tests/functional}" +all-tests = ["unit-tests", "integration-tests"] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_athena-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-athena", +] +check-sdist = [ + "twine check dist/*", + "find ./dist/dbt_athena-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-athena", +] diff --git a/dbt-athena/pyproject.toml b/dbt-athena/pyproject.toml index 1353ccda..69ec0018 100644 --- a/dbt-athena/pyproject.toml +++ b/dbt-athena/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] dynamic = ["version"] name = "dbt-athena" @@ -39,69 +43,12 @@ Homepage = "https://github.com/dbt-labs/dbt-athena/dbt-athena" Documentation = "https://docs.getdbt.com" Repository = "https://github.com/dbt-labs/dbt-athena.git#subdirectory=dbt-athena" Issues = "https://github.com/dbt-labs/dbt-athena/issues" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.sdist] -include = ["src/dbt"] - -[tool.hatch.build.targets.wheel] -packages = ["src/dbt"] - -[tool.hatch.version] -path = "src/dbt/adapters/athena/__version__.py" - -[tool.hatch.envs.default] -dependencies = [ - "dbt-tests-adapter~=1.9.2", - "moto~=5.0.13", - "pre-commit~=3.5", - "pyparsing~=3.1.4", - "pytest~=8.3", - "pytest-cov~=5.0", - "pytest-dotenv~=0.5", - "pytest-xdist~=3.6", -] -[tool.hatch.envs.default.scripts] -setup = [ - "pre-commit install", - "cp -n test.env.example test.env", -] -code-quality = "pre-commit run --all-files" -unit-tests = "pytest --cov=dbt --cov-report=html:htmlcov {args:tests/unit}" -integration-tests = "python -m pytest -n auto {args:tests/functional}" -all-tests = ["unit-tests", "integration-tests"] - -[tool.hatch.envs.build] -detached = true -dependencies = [ - "wheel", - "twine", - "check-wheel-contents", -] -[tool.hatch.envs.build.scripts] -check-all = [ - "- check-wheel", - "- check-sdist", -] -check-wheel = [ - "check-wheel-contents dist/*.whl --ignore W007,W008", - "find ./dist/dbt_athena-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", - "pip freeze | grep dbt-athena", -] -check-sdist = [ - "twine check dist/*", - "find ./dist/dbt_athena-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", - "pip freeze | grep dbt-athena", -] +Changelog = "https://github.com/dbt-labs/dbt-athena/blob/main/dbt-athena/CHANGELOG.md" [tool.pytest] -testpaths = [ - "tests/unit", - "tests/functional", -] +testpaths = ["tests/unit", "tests/functional"] +color = true +csv = "results.csv" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-athena/tests/functional/__init__.py b/dbt-athena/tests/functional/__init__.py new file mode 100644 index 00000000..e69de29b