diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..f0f9eb47 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,56 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "dbt-databricks", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye", + "features": { + "ghcr.io/devcontainers-extra/features/hatch:2": {} + }, + "postStartCommand": "./.devcontainer/initialize.sh", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "charliermarsh.ruff", + "ms-python.mypy-type-checker" + ], + "settings": { + "editor.formatOnSave": true, + "editor.formatOnType": true, + "editor.rulers": [ + 100 + ], + "files.exclude": { + ".hatch": true, + ".mypy_cache": true, + ".pytest_cache": true, + "**/__pycache__": true + }, + "python.analysis.diagnosticMode": "workspace", + "python.analysis.enablePytestSupport": true, + "python.defaultInterpreterPath": ".hatch/bin/python", + "python.envFile": "test.env", + "python.testing.pytestEnabled": true, + "python.languageServer": "Pylance", + "ruff.fixAll": true, + "ruff.format.args": [ + "--config=pyproject.toml" + ], + "ruff.lint.args": [ + "--config=pyproject.toml" + ], + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.codeActionsOnSave": { + "source.organizeImports": "always", + "source.fixAll": "always" + } + }, + } + } + }, + "containerEnv": { + "WORKSPACE_FOLDER": "${containerWorkspaceFolder}" + } +} \ No newline at end of file diff --git a/.devcontainer/initialize.sh b/.devcontainer/initialize.sh new file mode 100755 index 00000000..ec38c1c4 --- /dev/null +++ b/.devcontainer/initialize.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -x + +# Create the Hatch environment +hatch env create + +# Install pre-commit +(hatch run setup-precommit && pre-commit gc) & + +# Set up test.env file if it does not already exist +if test -f ${WORKSPACE_FOLDER}/test.env; then + cp ${WORKSPACE_FOLDER}/test.env.example ${WORKSPACE_FOLDER}/test.env +fi + diff --git a/.gitignore b/.gitignore index 0725a26b..5cf4a003 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ logs/ *.sublime* .python-version .hatch +databricks_demo diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD index 46c86437..79b246c7 100644 --- a/CONTRIBUTING.MD +++ b/CONTRIBUTING.MD @@ -50,11 +50,7 @@ We follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) with one exception: ### Linting -This project uses [Black](https://pypi.org/project/black/), [flake8](https://flake8.pycqa.org/en/latest/), and [mypy](https://www.mypy-lang.org/) for linting and static type checks. Run all three with the `linter` command and commit before opening your pull request. - -``` -tox -e linter -``` +This project uses [Ruff](https://docs.astral.sh/ruff/) and [mypy](https://www.mypy-lang.org/) for linting and static type checks. To simplify reviews you can commit any format changes in a separate commit. @@ -118,21 +114,21 @@ If you set your `user.name` and `user.email` git configs, you can sign your comm Unit tests do not require a Databricks account. Please confirm that your pull request passes our unit test suite before opening a pull request. ```bash -tox -e unit +hatch run unit ``` ## Functional Tests Functional tests require a Databricks account with access to a workspace containing three specific compute resources as detailed below. -The `tox` commands to run each set of these tests appear below: +The `hatch` commands to run each set of these tests appear below: -| Compute Type | Unity Catalog | Command | -| ------------------- | ------------- | ----------------------------------------------- | -| SQL Warehouse | Yes | `tox -e integration-databricks-uc-sql-endpoint` | -| All Purpose Cluster | Yes | `tox -e integration-databricks-uc-cluster` | -| All Purpose | No | `tox -e integration-databricks-cluster` | +| Compute Type | Unity Catalog | Command | +| ------------------- | ------------- | -------------------------- | +| SQL Warehouse | Yes | `hatch run sqlw-e2e` | +| All Purpose Cluster | Yes | `hatch run uc-cluster-e2e` | +| All Purpose | No | `hatch run cluster-e2e` | -These tests are configured with environment variables that `tox` reads from a file called [test.env](/test.env.example) which you can copy from the example: +These tests are configured with environment variables that `hatch` reads from a file called [test.env](/test.env.example) which you can copy from the example: ```sh cp test.env.example test.env