Everything but the docs

databricks · Nov 22, 2024 · 3c36cae · 3c36cae
1 parent 8fc6973
commit 3c36cae
Show file tree

Hide file tree

Showing 42 changed files with 260 additions and 362 deletions.
diff --git a/.coverage b/.coverage
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -26,20 +26,24 @@ jobs:
       TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.9"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
+
+      - name: Install Hatch
         id: install-dependencies
-        run: pip install tox
-      - name: Run integration-uc-databricks-cluster
-        run: [email protected] DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET tox -e integration-databricks-uc-cluster
+        uses: pypa/hatch@install
+
+      - name: Run UC Cluster Functional Tests
+        run: [email protected] DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch run -i py=3.9 uc-cluster-e2e
 
   run-tox-tests-uc-sql:
     runs-on: ubuntu-latest
@@ -54,20 +58,24 @@ jobs:
       TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.9"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
+
+      - name: Install Hatch
         id: install-dependencies
-        run: pip install tox
-      - name: Run integration-databricks-uc-sql-endpoint
-        run: [email protected] DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET tox -e integration-databricks-uc-sql-endpoint
+        uses: pypa/hatch@install
+
+      - name: Run Sql Endpoint Functional Tests
+        run: [email protected] DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch run -i py=3.9 sqlw-e2e
 
   run-tox-tests-non-uc:
     runs-on: ubuntu-latest
@@ -79,17 +87,21 @@ jobs:
       DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.9"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
+
+      - name: Install Hatch
         id: install-dependencies
-        run: pip install tox
-      - name: Run integration-databricks-cluster
-        run: [email protected] DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET tox -e integration-databricks-cluster
+        uses: pypa/hatch@install
+
+      - name: Run Cluster Functional Tests
+        run: [email protected] DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch run -i py=3.9 cluster-e2e
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -38,44 +38,32 @@ concurrency:
 
 defaults:
   run:
-    shell: bash
+    shell: zsh
 
 jobs:
   code-quality:
-    name: ${{ matrix.toxenv }}
+    name: Code Quality
 
     runs-on: ubuntu-latest
     timeout-minutes: 10
 
     strategy:
       fail-fast: false
-      matrix:
-        toxenv: [linter]
-
-    env:
-      TOXENV: ${{ matrix.toxenv }}
-      PYTEST_ADDOPTS: "-v --color=yes"
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4.3.0
+        uses: actions/setup-python@v5
         with:
           python-version: "3.9"
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install tox
-          tox --version
+      - name: Install Hatch
+        uses: pypa/hatch@install
 
-      - name: Run tox
-        run: tox
+      - name: Run Code Quality
+        run: hatch run -i py=3.9 code-quality
 
   unit:
     name: unit test / python ${{ matrix.python-version }}
@@ -88,40 +76,20 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
-    env:
-      TOXENV: "unit"
-      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
-
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
+        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4.3.0
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install tox
-          tox --version
-      - name: Run tox
-        run: tox
-
-      - name: Get current date
-        if: always()
-        id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%dT%H_%M_%S')" #no colons allowed for artifacts
+      - name: Install Hatch
+        uses: pypa/hatch@install
 
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
-          path: unit_results.csv
+      - name: Run Unit Tests
+        run: hatch run -i py=${{ matrix.python-version }} unit
 
   build:
     name: build packages
@@ -133,20 +101,18 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4.3.0
+        uses: actions/setup-python@v5
         with:
           python-version: "3.9"
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
-          python -m pip --version
+      - name: Install Hatch
+        uses: pypa/hatch@install
+
       - name: Build distributions
-        run: ./scripts/build-dist.sh
+        run: hatch build
 
       - name: Show distributions
         run: ls -lh dist/
@@ -165,7 +131,7 @@ jobs:
           if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi
           echo "::set-output name=is_alpha::$is_alpha"
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: dist
           path: dist/
@@ -187,16 +153,14 @@ jobs:
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4.3.0
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --upgrade wheel
-          python -m pip --version
-      - uses: actions/download-artifact@v3
+      - name: Install Hatch
+        uses: pypa/hatch@install
+
+      - uses: actions/download-artifact@v4
         with:
           name: dist
           path: dist/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.8.0
+    hooks:
+      # Run the linter.
+      - id: ruff
+      # Run the formatter.
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.13.0
+    hooks:
+      - id: mypy
+        additional_dependencies: [types-requests]
diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py
@@ -1 +1 @@
-version: str = "1.9.0b1"
+version = "1.9.0b1"
diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py
@@ -109,7 +109,7 @@
 @dataclass
 class DatabricksConfig(AdapterConfig):
     file_format: str = "delta"
-    table_format: TableFormat = TableFormat.DEFAULT
+    table_format: str = TableFormat.DEFAULT
     location_root: Optional[str] = None
     include_full_name_in_path: bool = False
     partition_by: Optional[Union[list[str], str]] = None
@@ -764,7 +764,7 @@ class RelationAPIBase(ABC, Generic[DatabricksRelationConfig]):
     For the most part, these are just namespaces to group related methods together.
     """
 
-    relation_type: ClassVar[DatabricksRelationType]
+    relation_type: ClassVar[str]
 
     @classmethod
     @abstractmethod

diff --git a/dbt/adapters/databricks/logging.py b/dbt/adapters/databricks/logging.py
@@ -1,18 +1,18 @@
-import logging
 import os
+from logging import Handler, LogRecord, getLogger
 from typing import Union
 
 from dbt.adapters.events.logging import AdapterLogger
 
 logger = AdapterLogger("Databricks")
 
 
-class DbtCoreHandler(logging.Handler):
+class DbtCoreHandler(Handler):
     def __init__(self, level: Union[str, int], dbt_logger: AdapterLogger):
         super().__init__(level=level)
         self.logger = dbt_logger
 
-    def emit(self, record: logging.LogRecord) -> None:
+    def emit(self, record: LogRecord) -> None:
         # record.levelname will be debug, info, warning, error, or critical
         # these map 1-to-1 with methods of the AdapterLogger
         log_func = getattr(self.logger, record.levelname.lower())
@@ -21,7 +21,7 @@ def emit(self, record: logging.LogRecord) -> None:
 
 dbt_adapter_logger = AdapterLogger("databricks-sql-connector")
 
-pysql_logger = logging.getLogger("databricks.sql")
+pysql_logger = getLogger("databricks.sql")
 pysql_logger_level = os.environ.get("DBT_DATABRICKS_CONNECTOR_LOG_LEVEL", "WARN").upper()
 pysql_logger.setLevel(pysql_logger_level)
 

diff --git a/dbt/adapters/databricks/python_models/python_submissions.py b/dbt/adapters/databricks/python_models/python_submissions.py
@@ -1,8 +1,8 @@
 from abc import ABC, abstractmethod
 from typing import Any, Optional
 
-from attr import dataclass
 from dbt_common.exceptions import DbtRuntimeError
+from pydantic import BaseModel
 from typing_extensions import override
 
 from dbt.adapters.base import PythonJobHelper
@@ -104,8 +104,7 @@ def upload(self, compiled_code: str) -> str:
         return file_path
 
 
-@dataclass(frozen=True)
-class PythonJobDetails:
+class PythonJobDetails(BaseModel):
     """Details required to submit a Python job run to Databricks."""
 
     run_name: str
@@ -224,7 +223,9 @@ def compile(self, path: str) -> PythonJobDetails:
         if access_control_list:
             job_spec["access_control_list"] = access_control_list
 
-        return PythonJobDetails(self.run_name, job_spec, additional_job_config)
+        return PythonJobDetails(
+            run_name=self.run_name, job_spec=job_spec, additional_job_config=additional_job_config
+        )
 
 
 class PythonNotebookSubmitter(PythonSubmitter):

diff --git a/dev-requirements.txt b/dev-requirements.txt
diff --git a/mypy.ini b/mypy.ini