From ad5ea160702e0af575b7ae126aea51756477a070 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 18 Apr 2023 12:34:26 -0700 Subject: [PATCH 01/18] Version bumps --- dbt/adapters/duckdb/__version__.py | 2 +- dev-requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/duckdb/__version__.py b/dbt/adapters/duckdb/__version__.py index 6abaa204..fa6c5a1a 100644 --- a/dbt/adapters/duckdb/__version__.py +++ b/dbt/adapters/duckdb/__version__.py @@ -1 +1 @@ -version = "1.4.1" +version = "1.5.0rc1" diff --git a/dev-requirements.txt b/dev-requirements.txt index 992ac7ea..bf248e99 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter -dbt-tests-adapter==1.4.5 +dbt-tests-adapter==1.5.0rc1 boto3 mypy-boto3-glue diff --git a/setup.py b/setup.py index fb3e8001..cbf439f0 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def _dbt_duckdb_version(): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.4.0", + "dbt-core~=1.5.0rc1", "duckdb>=0.5.0", ], extras_require={ From 71e0cb98fa45fc3d5692b19844640ceb9cdece43 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 18 Apr 2023 12:54:25 -0700 Subject: [PATCH 02/18] Various updates so tests will work against 1.5.0 --- tests/functional/adapter/test_python_model.py | 2 ++ tests/unit/test_duckdb_adapter.py | 8 ++++---- tests/unit/test_external_utils.py | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 2c4f163e..ed373fe7 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -30,6 +30,7 @@ def models(self): return { "schema.yml": schema_yml, "my_sql_model.sql": basic_sql, + "my_versioned_sql_model_v1.sql": basic_sql, "my_python_model.py": basic_python_template.format(extension=""), "second_sql_model.sql": second_sql, } @@ -41,6 +42,7 @@ def models(self): return { "schema.yml": schema_yml, "my_sql_model.sql": basic_sql, + "my_versioned_sql_model_v1.sql": basic_sql, "my_python_model.py": basic_python_template.format(extension=".df()"), "second_sql_model.sql": second_sql, } diff --git a/tests/unit/test_duckdb_adapter.py b/tests/unit/test_duckdb_adapter.py index 49e6abe4..690949fd 100644 --- a/tests/unit/test_duckdb_adapter.py +++ b/tests/unit/test_duckdb_adapter.py @@ -1,7 +1,8 @@ import unittest +from argparse import Namespace from unittest import mock -import dbt.flags as flags +from dbt.flags import set_from_args from dbt.adapters.duckdb import DuckDBAdapter from dbt.adapters.duckdb.connections import DuckDBConnectionManager from tests.unit.utils import config_from_parts_or_dicts, mock_connection @@ -9,8 +10,7 @@ class TestDuckDBAdapter(unittest.TestCase): def setUp(self): - pass - flags.STRICT_MODE = True + set_from_args(Namespace(STRICT_MODE=True), {}) profile_cfg = { "outputs": { @@ -34,7 +34,7 @@ def setUp(self): "config-version": 2, } - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg, cli_vars={}) self._adapter = None @property diff --git a/tests/unit/test_external_utils.py b/tests/unit/test_external_utils.py index 627059ad..3d86532d 100644 --- a/tests/unit/test_external_utils.py +++ b/tests/unit/test_external_utils.py @@ -1,13 +1,13 @@ import unittest +from argparse import Namespace -import dbt.flags as flags +from dbt.flags import set_from_args from dbt.adapters.duckdb import DuckDBAdapter from tests.unit.utils import config_from_parts_or_dicts, mock_connection class TestExternalUtils(unittest.TestCase): def setUp(self): - pass - flags.STRICT_MODE = True + set_from_args(Namespace(STRICT_MODE=True), {}) profile_cfg = { "outputs": { @@ -27,7 +27,7 @@ def setUp(self): "config-version": 2, } - self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) + self.config = config_from_parts_or_dicts(project_cfg, profile_cfg, cli_vars={}) self._adapter = None @property From 74caf4ca5a3f5b70fce3b3ccca8069fbc0b7d2f5 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Wed, 19 Apr 2023 20:51:32 -0700 Subject: [PATCH 03/18] Add in some of the new test functionality and restructure the directory a bit to prep for some extensions --- tests/functional/adapter/aliases/test_aliases.py | 11 +++++++++++ .../adapter/simple_seed/test_simple_seed.py | 16 ++++++++++++++++ .../test_store_test_failures.py | 7 +++++++ tests/functional/adapter/test_caching.py | 13 +++++++++++++ .../adapter/test_changing_relation_type.py | 5 +++++ tests/functional/adapter/test_simple_snapshot.py | 10 ++++++++++ .../functional/adapter/{ => utils}/test_utils.py | 0 7 files changed, 62 insertions(+) create mode 100644 tests/functional/adapter/aliases/test_aliases.py create mode 100644 tests/functional/adapter/simple_seed/test_simple_seed.py create mode 100644 tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py create mode 100644 tests/functional/adapter/test_caching.py create mode 100644 tests/functional/adapter/test_changing_relation_type.py create mode 100644 tests/functional/adapter/test_simple_snapshot.py rename tests/functional/adapter/{ => utils}/test_utils.py (100%) diff --git a/tests/functional/adapter/aliases/test_aliases.py b/tests/functional/adapter/aliases/test_aliases.py new file mode 100644 index 00000000..4a9d56a6 --- /dev/null +++ b/tests/functional/adapter/aliases/test_aliases.py @@ -0,0 +1,11 @@ +import pytest +from dbt.tests.adapter.aliases.test_aliases import BaseAliases, BaseAliasErrors, BaseSameAliasDifferentSchemas + +class TestAliasesDuckDB(BaseAliases): + pass + +class TestAliasesErrorDuckDB(BaseAliasErrors): + pass + +class BaseSameALiasDifferentSchemasDuckDB(BaseSameAliasDifferentSchemas): + pass diff --git a/tests/functional/adapter/simple_seed/test_simple_seed.py b/tests/functional/adapter/simple_seed/test_simple_seed.py new file mode 100644 index 00000000..1c30b995 --- /dev/null +++ b/tests/functional/adapter/simple_seed/test_simple_seed.py @@ -0,0 +1,16 @@ +import pytest + +from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase +from dbt.tests.util import run_dbt + + +class DuckDBTestSimpleBigSeedBatched(SeedConfigBase): + @pytest.fixture(scope="class") + def seeds(self): + seed_data = ["seed_id"] + seed_data.extend([str(i) for i in range(20_000)]) + return {"big_batched_seed.csv": "\n".join(seed_data)} + + def test_big_batched_seed(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 1 diff --git a/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py b/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py new file mode 100644 index 00000000..9ea6ec86 --- /dev/null +++ b/tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( + TestStoreTestFailures, +) + + +class DuckDBTestStoreTestFailures(TestStoreTestFailures): + pass diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py new file mode 100644 index 00000000..47a18f57 --- /dev/null +++ b/tests/functional/adapter/test_caching.py @@ -0,0 +1,13 @@ +from dbt.tests.adapter.caching.test_caching import ( + BaseCachingLowercaseModel, + BaseCachingUppercaseModel, + BaseCachingSelectedSchemaOnly, +) + + +class TestCachingLowerCaseModelDuckDB(BaseCachingLowercaseModel): + pass + + +class TestCachingSelectedSchemaOnlyDuckDB(BaseCachingSelectedSchemaOnly): + pass \ No newline at end of file diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py new file mode 100644 index 00000000..7c2b1cc2 --- /dev/null +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator + + +class TestChangeRelationTypesDuckDB(BaseChangeRelationTypeValidator): + pass \ No newline at end of file diff --git a/tests/functional/adapter/test_simple_snapshot.py b/tests/functional/adapter/test_simple_snapshot.py new file mode 100644 index 00000000..74da97d3 --- /dev/null +++ b/tests/functional/adapter/test_simple_snapshot.py @@ -0,0 +1,10 @@ +import pytest +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSnapshotCheck, BaseSimpleSnapshot + + +@pytest.mark.skip +class TestSimpleSnapshotDuckDB(BaseSimpleSnapshot): + pass + +class TestSnapshotCheckDuckDB(BaseSnapshotCheck): + pass \ No newline at end of file diff --git a/tests/functional/adapter/test_utils.py b/tests/functional/adapter/utils/test_utils.py similarity index 100% rename from tests/functional/adapter/test_utils.py rename to tests/functional/adapter/utils/test_utils.py From 44113bb12610e7f5076f195bc6f7dd10c1448b2b Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Wed, 19 Apr 2023 21:05:26 -0700 Subject: [PATCH 04/18] Update the snapshot merge SQL to sync it up with the Postgres impl; re-enable the tests --- dbt/include/duckdb/macros/snapshot_merge.sql | 19 ++++++++----------- tests/functional/adapter/test_caching.py | 3 +-- .../adapter/test_simple_snapshot.py | 9 ++++++--- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/dbt/include/duckdb/macros/snapshot_merge.sql b/dbt/include/duckdb/macros/snapshot_merge.sql index 9fc23a41..a1883517 100644 --- a/dbt/include/duckdb/macros/snapshot_merge.sql +++ b/dbt/include/duckdb/macros/snapshot_merge.sql @@ -1,21 +1,18 @@ {% macro duckdb__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - {% set insert_sql %} + update {{ target }} + set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to + from {{ source }} as DBT_INTERNAL_SOURCE + where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text + and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text) + and {{ target }}.dbt_valid_to is null; + insert into {{ target }} ({{ insert_cols_csv }}) select {% for column in insert_cols -%} DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %} {%- endfor %} from {{ source }} as DBT_INTERNAL_SOURCE - where DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'; - {% endset %} + where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text; - {% do adapter.add_query(insert_sql, auto_begin=False) %} - - update {{ target }} - set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to - from {{ source }} as DBT_INTERNAL_SOURCE - where DBT_INTERNAL_SOURCE.dbt_scd_id = {{ target.identifier }}.dbt_scd_id - and DBT_INTERNAL_SOURCE.dbt_change_type = 'update' - and {{ target.identifier }}.dbt_valid_to is null; {% endmacro %} diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py index 47a18f57..b75793bd 100644 --- a/tests/functional/adapter/test_caching.py +++ b/tests/functional/adapter/test_caching.py @@ -1,6 +1,5 @@ from dbt.tests.adapter.caching.test_caching import ( BaseCachingLowercaseModel, - BaseCachingUppercaseModel, BaseCachingSelectedSchemaOnly, ) @@ -10,4 +9,4 @@ class TestCachingLowerCaseModelDuckDB(BaseCachingLowercaseModel): class TestCachingSelectedSchemaOnlyDuckDB(BaseCachingSelectedSchemaOnly): - pass \ No newline at end of file + pass diff --git a/tests/functional/adapter/test_simple_snapshot.py b/tests/functional/adapter/test_simple_snapshot.py index 74da97d3..ad6ae8fd 100644 --- a/tests/functional/adapter/test_simple_snapshot.py +++ b/tests/functional/adapter/test_simple_snapshot.py @@ -1,10 +1,13 @@ import pytest -from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSnapshotCheck, BaseSimpleSnapshot +from dbt.tests.adapter.simple_snapshot.test_snapshot import ( + BaseSnapshotCheck, + BaseSimpleSnapshot, +) -@pytest.mark.skip class TestSimpleSnapshotDuckDB(BaseSimpleSnapshot): pass + class TestSnapshotCheckDuckDB(BaseSnapshotCheck): - pass \ No newline at end of file + pass From b2c74d1a87b8ca18c380309f95c8f3254216c421 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Apr 2023 11:00:28 +0200 Subject: [PATCH 05/18] Add support for model contracts in v1.5 --- dbt/adapters/duckdb/impl.py | 22 ++++++ dbt/include/duckdb/macros/adapters.sql | 29 +++++++- tests/functional/adapter/test_constraints.py | 71 ++++++++++++++++++++ 3 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 tests/functional/adapter/test_constraints.py diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index b00e9749..92d2482b 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -8,12 +8,14 @@ from dbt.adapters.base import BaseRelation from dbt.adapters.base.column import Column +from dbt.adapters.base.impl import ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.duckdb.connections import DuckDBConnectionManager from dbt.adapters.duckdb.glue import create_or_update_table from dbt.adapters.duckdb.relation import DuckDBRelation from dbt.adapters.sql import SQLAdapter from dbt.contracts.connection import AdapterResponse +from dbt.contracts.graph.nodes import ConstraintType from dbt.exceptions import DbtInternalError from dbt.exceptions import DbtRuntimeError @@ -22,6 +24,14 @@ class DuckDBAdapter(SQLAdapter): ConnectionManager = DuckDBConnectionManager Relation = DuckDBRelation + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.ENFORCED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.ENFORCED, + ConstraintType.primary_key: ConstraintSupport.ENFORCED, + ConstraintType.foreign_key: ConstraintSupport.ENFORCED, + } + @classmethod def date_function(cls) -> str: return "now()" @@ -176,6 +186,18 @@ def get_rows_different_sql( ) return sql + @available.parse(lambda *a, **k: []) + def get_column_schema_from_query(self, sql: str) -> List[Column]: + """Get a list of the Columns with names and data types from the given sql.""" + _, cursor = self.connections.add_select_query(sql) + columns = [ + # duckdb returns column_type as a string, rather than int (code) + self.Column.create(column_name, column_type_name) + # https://peps.python.org/pep-0249/#description + for column_name, column_type_name, *_ in cursor.description + ] + return columns + # Change `table_a/b` to `table_aaaaa/bbbbb` to avoid duckdb binding issues when relation_a/b # is called "table_a" or "table_b" in some of the dbt tests diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index 4f540ac0..2d90f5c4 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -28,6 +28,18 @@ {{ return(run_query(sql)) }} {% endmacro %} +{% macro get_column_names() %} + {# loop through user_provided_columns to get column names #} + {%- set user_provided_columns = model['columns'] -%} + ( + {% for i in user_provided_columns %} + {% set col = user_provided_columns[i] %} + {{ col['name'] }} {{ "," if not loop.last }} + {% endfor %} + ) +{% endmacro %} + + {% macro duckdb__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if language == 'sql' -%} {%- set sql_header = config.get('sql_header', none) -%} @@ -36,7 +48,18 @@ create {% if temporary: -%}temporary{%- endif %} table {{ relation.include(database=(not temporary and adapter.use_database()), schema=(not temporary)) }} - as ( + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced and not temporary %} + {{ get_assert_columns_equivalent(compiled_code) }} + {% if not temporary %} {#-- DuckDB doesnt support constraints on temp tables --#} + {{ get_table_columns_and_constraints() }} ; + insert into {{ relation }} {{ get_column_names() }} + {%- set compiled_code = get_select_subquery(compiled_code) %} + {% else %} + as + {% endif %} + {% endif %} + ( {{ compiled_code }} ); {%- elif language == 'python' -%} @@ -62,6 +85,10 @@ def materialize(df, con): {% endmacro %} {% macro duckdb__create_view_as(relation, sql) -%} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py new file mode 100644 index 00000000..c6ac1abc --- /dev/null +++ b/tests/functional/adapter/test_constraints.py @@ -0,0 +1,71 @@ +import pytest + +class DuckDBColumnEqualSetup: + @pytest.fixture + def int_type(self): + return "NUMBER" + + @pytest.fixture + def schema_int_type(self): + return "INT" + + @pytest.fixture + def data_types(self, schema_int_type, int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + # DuckDB's cursor doesn't seem to distinguish between: + # INT and NUMERIC/DECIMAL -- both just return as 'NUMBER' + # TIMESTAMP and TIMESTAMPTZ -- both just return as 'DATETIME' + # [1,2,3] and ['a','b','c'] -- both just return as 'list' + + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["true", "bool", "bool"], + ["'2013-11-03 00:00:00-07'::timestamptz", "timestamptz", "DATETIME"], + ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "DATETIME"], + ["ARRAY['a','b','c']", "text[]", "list"], + ["ARRAY[1,2,3]", "int[]", "list"], + #["'1'::numeric", "numeric", "DECIMAL"], -- no distinction, noted above + ["""{'bar': 'baz', 'balance': 7.77, 'active': false}""", "struct(bar text, balance decimal, active boolean)", "dict"], + ] + + +class TestTableConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseTableConstraintsColumnsEqual): + pass + + +class TestViewConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseViewConstraintsColumnsEqual): + pass + + +class TestIncrementalConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual): + pass + + +class TestTableConstraintsRuntimeDdlEnforcement(DuckDBColumnEqualSetup, BaseConstraintsRuntimeDdlEnforcement): + pass + + +class TestTableConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NOT NULL constraint failed"] + + +class TestIncrementalConstraintsRuntimeDdlEnforcement( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NOT NULL constraint failed"] + + +class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NOT NULL constraint failed"] + +class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NOT NULL constraint failed"] From 6626430b6b076d64a33180059b2ce0c880930179 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Apr 2023 11:39:36 +0200 Subject: [PATCH 06/18] Add references alias for foreign_key --- dbt/adapters/duckdb/impl.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 92d2482b..de1e0817 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -15,6 +15,7 @@ from dbt.adapters.duckdb.relation import DuckDBRelation from dbt.adapters.sql import SQLAdapter from dbt.contracts.connection import AdapterResponse +from dbt.contracts.graph.nodes import ColumnLevelConstraint from dbt.contracts.graph.nodes import ConstraintType from dbt.exceptions import DbtInternalError from dbt.exceptions import DbtRuntimeError @@ -198,6 +199,16 @@ def get_column_schema_from_query(self, sql: str) -> List[Column]: ] return columns + @classmethod + def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]: + """Render the given constraint as DDL text. Should be overriden by adapters which need custom constraint + rendering.""" + if constraint.type == ConstraintType.foreign_key: + # DuckDB doesn't support 'foreign key' as an alias + return f"references {constraint.expression}" + else: + return super().render_column_constraint(constraint) + # Change `table_a/b` to `table_aaaaa/bbbbb` to avoid duckdb binding issues when relation_a/b # is called "table_a" or "table_b" in some of the dbt tests From 64f351dc2e98348e1576600978e82f68c6d95932 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Apr 2023 11:48:51 +0200 Subject: [PATCH 07/18] Fixup create_table_as logic --- dbt/include/duckdb/macros/adapters.sql | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index 2d90f5c4..ab19b6b3 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -42,26 +42,28 @@ {% macro duckdb__create_table_as(temporary, relation, compiled_code, language='sql') -%} {%- if language == 'sql' -%} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(compiled_code) }} + {% endif %} {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} create {% if temporary: -%}temporary{%- endif %} table {{ relation.include(database=(not temporary and adapter.use_database()), schema=(not temporary)) }} - {% set contract_config = config.get('contract') %} {% if contract_config.enforced and not temporary %} - {{ get_assert_columns_equivalent(compiled_code) }} - {% if not temporary %} {#-- DuckDB doesnt support constraints on temp tables --#} - {{ get_table_columns_and_constraints() }} ; - insert into {{ relation }} {{ get_column_names() }} - {%- set compiled_code = get_select_subquery(compiled_code) %} - {% else %} - as - {% endif %} - {% endif %} - ( + {#-- DuckDB doesnt support constraints on temp tables --#} + {{ get_table_columns_and_constraints() }} ; + insert into {{ relation }} {{ get_column_names() }} ( + {{ compiled_code }} + ); + {%- set compiled_code = get_select_subquery(compiled_code) %} + {% else %} + as ( {{ compiled_code }} ); + {% endif %} {%- elif language == 'python' -%} {{ py_write_table(temporary=temporary, relation=relation, compiled_code=compiled_code) }} {%- else -%} From b5b488e6c724c8216f62a2e21c4d94324089c6ee Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Apr 2023 19:37:18 +0200 Subject: [PATCH 08/18] Add back the test imports I foolishly removed --- tests/functional/adapter/test_constraints.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index c6ac1abc..6b33960f 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -1,5 +1,16 @@ import pytest +from dbt.tests.adapter.constraints.test_constraints import ( + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, + BaseIncrementalConstraintsColumnsEqual, + BaseConstraintsRuntimeDdlEnforcement, + BaseConstraintsRollback, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsRollback, + BaseModelConstraintsRuntimeEnforcement, +) + class DuckDBColumnEqualSetup: @pytest.fixture def int_type(self): From 32a9dfa921090edb3250b32b5aefaf5af28dc2f7 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Sat, 22 Apr 2023 12:33:59 +0200 Subject: [PATCH 09/18] Reorder logic, fix tests --- dbt/include/duckdb/macros/adapters.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt/include/duckdb/macros/adapters.sql b/dbt/include/duckdb/macros/adapters.sql index ab19b6b3..b7eec0ae 100644 --- a/dbt/include/duckdb/macros/adapters.sql +++ b/dbt/include/duckdb/macros/adapters.sql @@ -56,9 +56,8 @@ {#-- DuckDB doesnt support constraints on temp tables --#} {{ get_table_columns_and_constraints() }} ; insert into {{ relation }} {{ get_column_names() }} ( - {{ compiled_code }} + {{ get_select_subquery(compiled_code) }} ); - {%- set compiled_code = get_select_subquery(compiled_code) %} {% else %} as ( {{ compiled_code }} From 40e56f1f11edcf16285ea037c748f9727e672eb7 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Sun, 23 Apr 2023 21:20:29 -0700 Subject: [PATCH 10/18] First up: this is a bit nicer --- dbt/adapters/duckdb/impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index de1e0817..b9348416 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -96,7 +96,7 @@ def use_database(self) -> bool: @available def get_binding_char(self): - return DuckDBConnectionManager.env().get_binding_char() + return self.connections.env().get_binding_char() @available def external_write_options(self, write_location: str, rendered_options: dict) -> str: @@ -155,7 +155,7 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str) -> AdapterRe connection = self.connections.get_if_exists() if not connection: connection = self.connections.get_thread_connection() - env = DuckDBConnectionManager.env() + env = self.connections.env() return env.submit_python_job(connection.handle, parsed_model, compiled_code) def get_rows_different_sql( From 6f7010444bab018ee08d5e60d1c05b7555bdfe97 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Sun, 23 Apr 2023 21:35:24 -0700 Subject: [PATCH 11/18] Just trying to see where this leads me --- dbt/adapters/duckdb/buenavista.py | 13 +++++++ dbt/adapters/duckdb/column.py | 12 +++++++ dbt/adapters/duckdb/environments.py | 14 ++++++-- dbt/adapters/duckdb/impl.py | 13 +++---- dbt/adapters/duckdb/utils.py | 2 +- tests/functional/adapter/test_constraints.py | 38 +++----------------- 6 files changed, 45 insertions(+), 47 deletions(-) create mode 100644 dbt/adapters/duckdb/column.py diff --git a/dbt/adapters/duckdb/buenavista.py b/dbt/adapters/duckdb/buenavista.py index e6ed4af6..90261f19 100644 --- a/dbt/adapters/duckdb/buenavista.py +++ b/dbt/adapters/duckdb/buenavista.py @@ -1,7 +1,10 @@ import json +from typing import List import psycopg2 +from psycopg2.extensions import string_types +from . import column from . import credentials from . import utils from .environments import Environment @@ -59,3 +62,13 @@ def load_source(self, plugin_name: str, source_config: utils.SourceConfig): cursor.execute(json.dumps(payload)) cursor.close() handle.close() + + def create_columns(self, cursor) -> List[column.DuckDBColumn]: + columns = [ + column.DuckDBColumn.create( + column_name, string_types[column_type_code] + ) + # https://peps.python.org/pep-0249/#description + for column_name, column_type_code, *_ in cursor.description + ] + return columns \ No newline at end of file diff --git a/dbt/adapters/duckdb/column.py b/dbt/adapters/duckdb/column.py new file mode 100644 index 00000000..7847aa33 --- /dev/null +++ b/dbt/adapters/duckdb/column.py @@ -0,0 +1,12 @@ +from dbt.adapters.base import Column + + +class DuckDBColumn(Column): + @property + def data_type(self): + # on duckdb, do not convert 'text' or 'varchar' to 'varchar()' + if self.dtype.lower() == "text" or ( + self.dtype.lower() == "character varying" and self.char_size is None + ): + return self.dtype + return super().data_type \ No newline at end of file diff --git a/dbt/adapters/duckdb/environments.py b/dbt/adapters/duckdb/environments.py index 7d49f2a9..4f9f45e9 100644 --- a/dbt/adapters/duckdb/environments.py +++ b/dbt/adapters/duckdb/environments.py @@ -3,12 +3,16 @@ import os import tempfile from typing import Dict +from typing import List import duckdb +from .column import DuckDBColumn from .credentials import DuckDBCredentials from .plugins import Plugin +from .utils import PG_TYPE_CODE_TO_NAME from .utils import SourceConfig + from dbt.contracts.connection import AdapterResponse from dbt.exceptions import DbtRuntimeError @@ -66,13 +70,17 @@ def handle(self): def submit_python_job(self, handle, parsed_model: dict, compiled_code: str) -> AdapterResponse: pass - def get_binding_char(self) -> str: - return "?" - @abc.abstractmethod def load_source(self, plugin_name: str, source_config: SourceConfig) -> str: pass + @abc.abstractmethod + def create_columns(self, cursor) -> List[DuckDBColumn]: + pass + + def get_binding_char(self) -> str: + return "?" + @classmethod def initialize_db(cls, creds: DuckDBCredentials): config = creds.config_options or {} diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index b9348416..a347641f 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -7,9 +7,9 @@ import duckdb from dbt.adapters.base import BaseRelation -from dbt.adapters.base.column import Column from dbt.adapters.base.impl import ConstraintSupport from dbt.adapters.base.meta import available +from dbt.adapters.duckdb.column import DuckDBColumn from dbt.adapters.duckdb.connections import DuckDBConnectionManager from dbt.adapters.duckdb.glue import create_or_update_table from dbt.adapters.duckdb.relation import DuckDBRelation @@ -24,6 +24,7 @@ class DuckDBAdapter(SQLAdapter): ConnectionManager = DuckDBConnectionManager Relation = DuckDBRelation + Column = DuckDBColumn CONSTRAINT_SUPPORT = { ConstraintType.check: ConstraintSupport.ENFORCED, @@ -191,14 +192,8 @@ def get_rows_different_sql( def get_column_schema_from_query(self, sql: str) -> List[Column]: """Get a list of the Columns with names and data types from the given sql.""" _, cursor = self.connections.add_select_query(sql) - columns = [ - # duckdb returns column_type as a string, rather than int (code) - self.Column.create(column_name, column_type_name) - # https://peps.python.org/pep-0249/#description - for column_name, column_type_name, *_ in cursor.description - ] - return columns - + return self.connections.env().create_columns(cursor) + @classmethod def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]: """Render the given constraint as DDL text. Should be overriden by adapters which need custom constraint diff --git a/dbt/adapters/duckdb/utils.py b/dbt/adapters/duckdb/utils.py index c8e6d14d..af09ce8c 100644 --- a/dbt/adapters/duckdb/utils.py +++ b/dbt/adapters/duckdb/utils.py @@ -40,4 +40,4 @@ def create(cls, source: SourceDefinition) -> "SourceConfig": schema=source.schema, database=source.database, meta=meta, - ) + ) \ No newline at end of file diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 6b33960f..a306570d 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -11,49 +11,19 @@ BaseModelConstraintsRuntimeEnforcement, ) -class DuckDBColumnEqualSetup: - @pytest.fixture - def int_type(self): - return "NUMBER" - - @pytest.fixture - def schema_int_type(self): - return "INT" - - @pytest.fixture - def data_types(self, schema_int_type, int_type, string_type): - # sql_column_value, schema_data_type, error_data_type - return [ - # DuckDB's cursor doesn't seem to distinguish between: - # INT and NUMERIC/DECIMAL -- both just return as 'NUMBER' - # TIMESTAMP and TIMESTAMPTZ -- both just return as 'DATETIME' - # [1,2,3] and ['a','b','c'] -- both just return as 'list' - - ["1", schema_int_type, int_type], - ["'1'", string_type, string_type], - ["true", "bool", "bool"], - ["'2013-11-03 00:00:00-07'::timestamptz", "timestamptz", "DATETIME"], - ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "DATETIME"], - ["ARRAY['a','b','c']", "text[]", "list"], - ["ARRAY[1,2,3]", "int[]", "list"], - #["'1'::numeric", "numeric", "DECIMAL"], -- no distinction, noted above - ["""{'bar': 'baz', 'balance': 7.77, 'active': false}""", "struct(bar text, balance decimal, active boolean)", "dict"], - ] - - -class TestTableConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseTableConstraintsColumnsEqual): +class TestTableConstraintsColumnsEqual(BaseTableConstraintsColumnsEqual): pass -class TestViewConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseViewConstraintsColumnsEqual): +class TestViewConstraintsColumnsEqual(BaseViewConstraintsColumnsEqual): pass -class TestIncrementalConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual): +class TestIncrementalConstraintsColumnsEqual(BaseIncrementalConstraintsColumnsEqual): pass -class TestTableConstraintsRuntimeDdlEnforcement(DuckDBColumnEqualSetup, BaseConstraintsRuntimeDdlEnforcement): +class TestTableConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): pass From 2aeb1fffcbecad333e46138af3b6930b689c6b00 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Sun, 23 Apr 2023 22:02:00 -0700 Subject: [PATCH 12/18] fix that --- dbt/adapters/duckdb/environments.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/duckdb/environments.py b/dbt/adapters/duckdb/environments.py index 4f9f45e9..415148b0 100644 --- a/dbt/adapters/duckdb/environments.py +++ b/dbt/adapters/duckdb/environments.py @@ -10,7 +10,6 @@ from .column import DuckDBColumn from .credentials import DuckDBCredentials from .plugins import Plugin -from .utils import PG_TYPE_CODE_TO_NAME from .utils import SourceConfig from dbt.contracts.connection import AdapterResponse From 6fd767c33481b848cd4ffef76f70cce51f6e3990 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Sun, 23 Apr 2023 22:03:38 -0700 Subject: [PATCH 13/18] stopping here for the night; need to do most of the localenv stuff tmrw --- dbt/adapters/duckdb/buenavista.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/duckdb/buenavista.py b/dbt/adapters/duckdb/buenavista.py index 90261f19..37f013c2 100644 --- a/dbt/adapters/duckdb/buenavista.py +++ b/dbt/adapters/duckdb/buenavista.py @@ -66,7 +66,7 @@ def load_source(self, plugin_name: str, source_config: utils.SourceConfig): def create_columns(self, cursor) -> List[column.DuckDBColumn]: columns = [ column.DuckDBColumn.create( - column_name, string_types[column_type_code] + column_name, string_types[column_type_code].name ) # https://peps.python.org/pep-0249/#description for column_name, column_type_code, *_ in cursor.description From 2465dba202e5a88c48711ef575b776f7bcca908e Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 24 Apr 2023 15:13:44 -0700 Subject: [PATCH 14/18] Renaming/refactoring some things --- .../__init__.py} | 100 +----------------- .../duckdb/{ => environments}/buenavista.py | 8 +- dbt/adapters/duckdb/environments/local.py | 98 +++++++++++++++++ 3 files changed, 107 insertions(+), 99 deletions(-) rename dbt/adapters/duckdb/{environments.py => environments/__init__.py} (59%) rename dbt/adapters/duckdb/{ => environments}/buenavista.py (95%) create mode 100644 dbt/adapters/duckdb/environments/local.py diff --git a/dbt/adapters/duckdb/environments.py b/dbt/adapters/duckdb/environments/__init__.py similarity index 59% rename from dbt/adapters/duckdb/environments.py rename to dbt/adapters/duckdb/environments/__init__.py index 415148b0..31f0dca9 100644 --- a/dbt/adapters/duckdb/environments.py +++ b/dbt/adapters/duckdb/environments/__init__.py @@ -7,10 +7,10 @@ import duckdb -from .column import DuckDBColumn -from .credentials import DuckDBCredentials -from .plugins import Plugin -from .utils import SourceConfig +from ..column import DuckDBColumn +from ..credentials import DuckDBCredentials +from ..plugins import Plugin +from ..utils import SourceConfig from dbt.contracts.connection import AdapterResponse from dbt.exceptions import DbtRuntimeError @@ -31,35 +31,6 @@ def _ensure_event_loop(): asyncio.set_event_loop(loop) -class DuckDBCursorWrapper: - def __init__(self, cursor): - self._cursor = cursor - - # forward along all non-execute() methods/attribute look-ups - def __getattr__(self, name): - return getattr(self._cursor, name) - - def execute(self, sql, bindings=None): - try: - if bindings is None: - return self._cursor.execute(sql) - else: - return self._cursor.execute(sql, bindings) - except RuntimeError as e: - raise DbtRuntimeError(str(e)) - - -class DuckDBConnectionWrapper: - def __init__(self, cursor): - self._cursor = DuckDBCursorWrapper(cursor) - - def close(self): - self._cursor.close() - - def cursor(self): - return self._cursor - - class Environment(abc.ABC): @abc.abstractmethod def handle(self): @@ -166,71 +137,10 @@ def run_python_job(cls, con, load_df_function, identifier: str, compiled_code: s os.unlink(mod_file.name) -class LocalEnvironment(Environment): - def __init__(self, credentials: DuckDBCredentials): - self.conn = self.initialize_db(credentials) - self._plugins = self.initialize_plugins(credentials) - self.creds = credentials - - def handle(self): - # Extensions/settings need to be configured per cursor - cursor = self.initialize_cursor(self.creds, self.conn.cursor()) - return DuckDBConnectionWrapper(cursor) - - def submit_python_job(self, handle, parsed_model: dict, compiled_code: str) -> AdapterResponse: - con = handle.cursor() - - def ldf(table_name): - return con.query(f"select * from {table_name}") - - self.run_python_job(con, ldf, parsed_model["alias"], compiled_code) - return AdapterResponse(_message="OK") - - def load_source(self, plugin_name: str, source_config: SourceConfig): - if plugin_name not in self._plugins: - raise Exception( - f"Plugin {plugin_name} not found; known plugins are: " - + ",".join(self._plugins.keys()) - ) - plugin = self._plugins[plugin_name] - handle = self.handle() - cursor = handle.cursor() - save_mode = source_config.meta.get("save_mode", "overwrite") - if save_mode in ("ignore", "error_if_exists"): - schema, identifier = source_config.schema, source_config.identifier - q = f"""SELECT COUNT(1) - FROM information_schema.tables - WHERE table_schema = '{schema}' - AND table_name = '{identifier}' - """ - if cursor.execute(q).fetchone()[0]: - if save_mode == "error_if_exists": - raise Exception(f"Source {source_config.table_name()} already exists!") - else: - # Nothing to do (we ignore the existing table) - return - df = plugin.load(source_config) - assert df is not None - materialization = source_config.meta.get("materialization", "table") - cursor.execute( - f"CREATE OR REPLACE {materialization} {source_config.table_name()} AS SELECT * FROM df" - ) - cursor.close() - handle.close() - - def close(self): - if self.conn: - self.conn.close() - self.conn = None - - def __del__(self): - self.close() - - def create(creds: DuckDBCredentials) -> Environment: if creds.remote: from .buenavista import BVEnvironment - return BVEnvironment(creds) else: + from .local import LocalEnvironment return LocalEnvironment(creds) diff --git a/dbt/adapters/duckdb/buenavista.py b/dbt/adapters/duckdb/environments/buenavista.py similarity index 95% rename from dbt/adapters/duckdb/buenavista.py rename to dbt/adapters/duckdb/environments/buenavista.py index 37f013c2..0cd7638e 100644 --- a/dbt/adapters/duckdb/buenavista.py +++ b/dbt/adapters/duckdb/environments/buenavista.py @@ -4,10 +4,10 @@ import psycopg2 from psycopg2.extensions import string_types -from . import column -from . import credentials -from . import utils -from .environments import Environment +from .. import column +from .. import credentials +from .. import utils +from . import Environment from dbt.contracts.connection import AdapterResponse diff --git a/dbt/adapters/duckdb/environments/local.py b/dbt/adapters/duckdb/environments/local.py new file mode 100644 index 00000000..eda421b0 --- /dev/null +++ b/dbt/adapters/duckdb/environments/local.py @@ -0,0 +1,98 @@ + +from .. import column +from .. import credentials +from .. import utils +from ..environments import Environment + +from dbt.contracts.connection import AdapterResponse +from dbt.exceptions import DbtRuntimeError + + +class DuckDBCursorWrapper: + def __init__(self, cursor): + self._cursor = cursor + + # forward along all non-execute() methods/attribute look-ups + def __getattr__(self, name): + return getattr(self._cursor, name) + + def execute(self, sql, bindings=None): + try: + if bindings is None: + return self._cursor.execute(sql) + else: + return self._cursor.execute(sql, bindings) + except RuntimeError as e: + raise DbtRuntimeError(str(e)) + + +class DuckDBConnectionWrapper: + def __init__(self, cursor): + self._cursor = DuckDBCursorWrapper(cursor) + + def close(self): + self._cursor.close() + + def cursor(self): + return self._cursor + + +class LocalEnvironment(Environment): + def __init__(self, credentials: credentials.DuckDBCredentials): + self.conn = self.initialize_db(credentials) + self._plugins = self.initialize_plugins(credentials) + self.creds = credentials + + def handle(self): + # Extensions/settings need to be configured per cursor + cursor = self.initialize_cursor(self.creds, self.conn.cursor()) + return DuckDBConnectionWrapper(cursor) + + def submit_python_job(self, handle, parsed_model: dict, compiled_code: str) -> AdapterResponse: + con = handle.cursor() + + def ldf(table_name): + return con.query(f"select * from {table_name}") + + self.run_python_job(con, ldf, parsed_model["alias"], compiled_code) + return AdapterResponse(_message="OK") + + def load_source(self, plugin_name: str, source_config: utils.SourceConfig): + if plugin_name not in self._plugins: + raise Exception( + f"Plugin {plugin_name} not found; known plugins are: " + + ",".join(self._plugins.keys()) + ) + plugin = self._plugins[plugin_name] + handle = self.handle() + cursor = handle.cursor() + save_mode = source_config.meta.get("save_mode", "overwrite") + if save_mode in ("ignore", "error_if_exists"): + schema, identifier = source_config.schema, source_config.identifier + q = f"""SELECT COUNT(1) + FROM information_schema.tables + WHERE table_schema = '{schema}' + AND table_name = '{identifier}' + """ + if cursor.execute(q).fetchone()[0]: + if save_mode == "error_if_exists": + raise Exception(f"Source {source_config.table_name()} already exists!") + else: + # Nothing to do (we ignore the existing table) + return + df = plugin.load(source_config) + assert df is not None + materialization = source_config.meta.get("materialization", "table") + cursor.execute( + f"CREATE OR REPLACE {materialization} {source_config.table_name()} AS SELECT * FROM df" + ) + cursor.close() + handle.close() + + def close(self): + if self.conn: + self.conn.close() + self.conn = None + + def __del__(self): + self.close() \ No newline at end of file From 7d81bd9eecf9389436cdc08025e628c2e2680553 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 24 Apr 2023 16:38:17 -0700 Subject: [PATCH 15/18] now with format fixes --- dbt/adapters/duckdb/column.py | 2 +- dbt/adapters/duckdb/environments/__init__.py | 7 +-- .../duckdb/environments/buenavista.py | 11 ++-- dbt/adapters/duckdb/environments/local.py | 53 +++++++++++++++++-- dbt/adapters/duckdb/impl.py | 13 ++--- dbt/adapters/duckdb/utils.py | 2 +- tests/functional/adapter/test_constraints.py | 34 +++++++++--- 7 files changed, 94 insertions(+), 28 deletions(-) diff --git a/dbt/adapters/duckdb/column.py b/dbt/adapters/duckdb/column.py index 7847aa33..40a22104 100644 --- a/dbt/adapters/duckdb/column.py +++ b/dbt/adapters/duckdb/column.py @@ -9,4 +9,4 @@ def data_type(self): self.dtype.lower() == "character varying" and self.char_size is None ): return self.dtype - return super().data_type \ No newline at end of file + return super().data_type diff --git a/dbt/adapters/duckdb/environments/__init__.py b/dbt/adapters/duckdb/environments/__init__.py index 31f0dca9..70fe9427 100644 --- a/dbt/adapters/duckdb/environments/__init__.py +++ b/dbt/adapters/duckdb/environments/__init__.py @@ -7,11 +7,10 @@ import duckdb -from ..column import DuckDBColumn from ..credentials import DuckDBCredentials from ..plugins import Plugin from ..utils import SourceConfig - +from dbt.adapters.base.column import Column from dbt.contracts.connection import AdapterResponse from dbt.exceptions import DbtRuntimeError @@ -45,7 +44,7 @@ def load_source(self, plugin_name: str, source_config: SourceConfig) -> str: pass @abc.abstractmethod - def create_columns(self, cursor) -> List[DuckDBColumn]: + def create_columns(self, cursor) -> List[Column]: pass def get_binding_char(self) -> str: @@ -140,7 +139,9 @@ def run_python_job(cls, con, load_df_function, identifier: str, compiled_code: s def create(creds: DuckDBCredentials) -> Environment: if creds.remote: from .buenavista import BVEnvironment + return BVEnvironment(creds) else: from .local import LocalEnvironment + return LocalEnvironment(creds) diff --git a/dbt/adapters/duckdb/environments/buenavista.py b/dbt/adapters/duckdb/environments/buenavista.py index 0cd7638e..8dab9b39 100644 --- a/dbt/adapters/duckdb/environments/buenavista.py +++ b/dbt/adapters/duckdb/environments/buenavista.py @@ -4,10 +4,11 @@ import psycopg2 from psycopg2.extensions import string_types +from . import Environment from .. import column from .. import credentials from .. import utils -from . import Environment +from dbt.adapters.base.column import Column from dbt.contracts.connection import AdapterResponse @@ -63,12 +64,10 @@ def load_source(self, plugin_name: str, source_config: utils.SourceConfig): cursor.close() handle.close() - def create_columns(self, cursor) -> List[column.DuckDBColumn]: + def create_columns(self, cursor) -> List[Column]: columns = [ - column.DuckDBColumn.create( - column_name, string_types[column_type_code].name - ) + column.DuckDBColumn.create(column_name, string_types[column_type_code].name) # https://peps.python.org/pep-0249/#description for column_name, column_type_code, *_ in cursor.description ] - return columns \ No newline at end of file + return columns diff --git a/dbt/adapters/duckdb/environments/local.py b/dbt/adapters/duckdb/environments/local.py index eda421b0..d5a12765 100644 --- a/dbt/adapters/duckdb/environments/local.py +++ b/dbt/adapters/duckdb/environments/local.py @@ -1,9 +1,12 @@ +from typing import List +import pyarrow as pa + +from . import Environment from .. import column from .. import credentials from .. import utils -from ..environments import Environment - +from dbt.adapters.base.column import Column from dbt.contracts.connection import AdapterResponse from dbt.exceptions import DbtRuntimeError @@ -37,6 +40,44 @@ def cursor(self): return self._cursor +def convert_type(t: pa.DataType) -> str: + if pa.types.is_int64(t): + return "BIGINT" + elif pa.types.is_integer(t): + return "INTEGER" + elif pa.types.is_string(t): + return "TEXT" + elif pa.types.is_date(t): + return "DATE" + elif pa.types.is_time(t): + return "TIME" + elif pa.types.is_timestamp(t): + return "DATETIME" + elif pa.types.is_floating(t): + return "FLOAT" + elif pa.types.is_decimal(t): + return "DECIMAL" + elif pa.types.is_boolean(t): + return "BOOL" + elif pa.types.is_binary(t): + return "BINARY" + elif pa.types.is_interval(t): + return "INTERVAL" + elif pa.types.is_list(t): + field_type = t.field(0).type + if pa.types.is_integer(field_type): + return "INTEGERARRAY" + elif pa.types.is_string(field_type): + return "STRINGARRAY" + else: + return "ARRAY" + elif pa.types.is_struct(t) or pa.types.is_map(t): + # TODO: support detailed nested types + return "JSON" + else: + return "UNKNOWN" + + class LocalEnvironment(Environment): def __init__(self, credentials: credentials.DuckDBCredentials): self.conn = self.initialize_db(credentials) @@ -89,10 +130,16 @@ def load_source(self, plugin_name: str, source_config: utils.SourceConfig): cursor.close() handle.close() + def create_columns(self, cursor) -> List[Column]: + columns = [] + for field in cursor.fetch_record_batch().schema: + columns.append(column.DuckDBColumn.create(field.name, convert_type(field.type))) + return columns + def close(self): if self.conn: self.conn.close() self.conn = None def __del__(self): - self.close() \ No newline at end of file + self.close() diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index a347641f..43a87cd7 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -7,6 +7,7 @@ import duckdb from dbt.adapters.base import BaseRelation +from dbt.adapters.base.column import Column as DBTColumn from dbt.adapters.base.impl import ConstraintSupport from dbt.adapters.base.meta import available from dbt.adapters.duckdb.column import DuckDBColumn @@ -74,7 +75,7 @@ def register_glue_table( self, glue_database: str, table: str, - column_list: Sequence[Column], + column_list: Sequence[DBTColumn], location: str, file_format: str, ) -> None: @@ -97,7 +98,7 @@ def use_database(self) -> bool: @available def get_binding_char(self): - return self.connections.env().get_binding_char() + return DuckDBConnectionManager.env().get_binding_char() @available def external_write_options(self, write_location: str, rendered_options: dict) -> str: @@ -156,7 +157,7 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str) -> AdapterRe connection = self.connections.get_if_exists() if not connection: connection = self.connections.get_thread_connection() - env = self.connections.env() + env = DuckDBConnectionManager.env() return env.submit_python_job(connection.handle, parsed_model, compiled_code) def get_rows_different_sql( @@ -189,11 +190,11 @@ def get_rows_different_sql( return sql @available.parse(lambda *a, **k: []) - def get_column_schema_from_query(self, sql: str) -> List[Column]: + def get_column_schema_from_query(self, sql: str) -> List[DBTColumn]: """Get a list of the Columns with names and data types from the given sql.""" _, cursor = self.connections.add_select_query(sql) - return self.connections.env().create_columns(cursor) - + return DuckDBConnectionManager.env().create_columns(cursor) + @classmethod def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]: """Render the given constraint as DDL text. Should be overriden by adapters which need custom constraint diff --git a/dbt/adapters/duckdb/utils.py b/dbt/adapters/duckdb/utils.py index af09ce8c..c8e6d14d 100644 --- a/dbt/adapters/duckdb/utils.py +++ b/dbt/adapters/duckdb/utils.py @@ -40,4 +40,4 @@ def create(cls, source: SourceDefinition) -> "SourceConfig": schema=source.schema, database=source.database, meta=meta, - ) \ No newline at end of file + ) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index a306570d..3f0d83fa 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -11,42 +11,60 @@ BaseModelConstraintsRuntimeEnforcement, ) -class TestTableConstraintsColumnsEqual(BaseTableConstraintsColumnsEqual): + +class DuckDBColumnEqualSetup: + @pytest.fixture + def data_types(self, schema_int_type, int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["true", "bool", "bool"], + ["'2013-11-03 00:00:00-07'::timestamptz", "timestamptz", "DATETIME"], + ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "DATETIME"], + ["ARRAY['a','b','c']", "text[]", "STRINGARRAY"], + ["ARRAY[1,2,3]", "int[]", "INTEGERARRAY"], + ["'1'::numeric", "numeric", "DECIMAL"], + ["""{'bar': 'baz', 'balance': 7.77, 'active': false}""", "struct(bar text, balance decimal, active boolean)", "json"], + ] + + +class TestTableConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseTableConstraintsColumnsEqual): pass -class TestViewConstraintsColumnsEqual(BaseViewConstraintsColumnsEqual): +class TestViewConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseViewConstraintsColumnsEqual): pass -class TestIncrementalConstraintsColumnsEqual(BaseIncrementalConstraintsColumnsEqual): +class TestIncrementalConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual): pass -class TestTableConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): +class TestTableConstraintsRuntimeDdlEnforcement(DuckDBColumnEqualSetup, BaseConstraintsRuntimeDdlEnforcement): pass -class TestTableConstraintsRollback(BaseConstraintsRollback): +class TestTableConstraintsRollback(DuckDBColumnEqualSetup, BaseConstraintsRollback): @pytest.fixture(scope="class") def expected_error_messages(self): return ["NOT NULL constraint failed"] class TestIncrementalConstraintsRuntimeDdlEnforcement( - BaseIncrementalConstraintsRuntimeDdlEnforcement + DuckDBColumnEqualSetup, BaseIncrementalConstraintsRuntimeDdlEnforcement ): @pytest.fixture(scope="class") def expected_error_messages(self): return ["NOT NULL constraint failed"] -class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): +class TestIncrementalConstraintsRollback(DuckDBColumnEqualSetup, BaseIncrementalConstraintsRollback): @pytest.fixture(scope="class") def expected_error_messages(self): return ["NOT NULL constraint failed"] -class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): +class TestModelConstraintsRuntimeEnforcement(DuckDBColumnEqualSetup, BaseModelConstraintsRuntimeEnforcement): @pytest.fixture(scope="class") def expected_error_messages(self): return ["NOT NULL constraint failed"] From 8d5b1338b1e6a5839ee30233aa04ad9427599060 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 25 Apr 2023 14:53:31 -0700 Subject: [PATCH 16/18] let's try whistling this --- dbt/adapters/duckdb/environments/__init__.py | 13 ++--- .../duckdb/environments/buenavista.py | 12 ----- dbt/adapters/duckdb/environments/local.py | 50 ------------------- dbt/adapters/duckdb/impl.py | 12 ++++- tests/functional/adapter/test_constraints.py | 50 ++++++++++++++----- tox.ini | 9 ++++ 6 files changed, 63 insertions(+), 83 deletions(-) diff --git a/dbt/adapters/duckdb/environments/__init__.py b/dbt/adapters/duckdb/environments/__init__.py index 70fe9427..bebf9112 100644 --- a/dbt/adapters/duckdb/environments/__init__.py +++ b/dbt/adapters/duckdb/environments/__init__.py @@ -3,14 +3,12 @@ import os import tempfile from typing import Dict -from typing import List import duckdb from ..credentials import DuckDBCredentials from ..plugins import Plugin from ..utils import SourceConfig -from dbt.adapters.base.column import Column from dbt.contracts.connection import AdapterResponse from dbt.exceptions import DbtRuntimeError @@ -31,6 +29,11 @@ def _ensure_event_loop(): class Environment(abc.ABC): + """An Environment is an abstraction to describe *where* the code you execute in your dbt-duckdb project + actually runs. This could be the local Python process that runs dbt (which is the default), + a remote server (like a Buena Vista instance), or even a Jupyter notebook kernel. + """ + @abc.abstractmethod def handle(self): pass @@ -43,10 +46,6 @@ def submit_python_job(self, handle, parsed_model: dict, compiled_code: str) -> A def load_source(self, plugin_name: str, source_config: SourceConfig) -> str: pass - @abc.abstractmethod - def create_columns(self, cursor) -> List[Column]: - pass - def get_binding_char(self) -> str: return "?" @@ -137,6 +136,8 @@ def run_python_job(cls, con, load_df_function, identifier: str, compiled_code: s def create(creds: DuckDBCredentials) -> Environment: + """Create an Environment based on the credentials passed in.""" + if creds.remote: from .buenavista import BVEnvironment diff --git a/dbt/adapters/duckdb/environments/buenavista.py b/dbt/adapters/duckdb/environments/buenavista.py index 8dab9b39..bbde2068 100644 --- a/dbt/adapters/duckdb/environments/buenavista.py +++ b/dbt/adapters/duckdb/environments/buenavista.py @@ -1,14 +1,10 @@ import json -from typing import List import psycopg2 -from psycopg2.extensions import string_types from . import Environment -from .. import column from .. import credentials from .. import utils -from dbt.adapters.base.column import Column from dbt.contracts.connection import AdapterResponse @@ -63,11 +59,3 @@ def load_source(self, plugin_name: str, source_config: utils.SourceConfig): cursor.execute(json.dumps(payload)) cursor.close() handle.close() - - def create_columns(self, cursor) -> List[Column]: - columns = [ - column.DuckDBColumn.create(column_name, string_types[column_type_code].name) - # https://peps.python.org/pep-0249/#description - for column_name, column_type_code, *_ in cursor.description - ] - return columns diff --git a/dbt/adapters/duckdb/environments/local.py b/dbt/adapters/duckdb/environments/local.py index d5a12765..2e557e80 100644 --- a/dbt/adapters/duckdb/environments/local.py +++ b/dbt/adapters/duckdb/environments/local.py @@ -1,12 +1,6 @@ -from typing import List - -import pyarrow as pa - from . import Environment -from .. import column from .. import credentials from .. import utils -from dbt.adapters.base.column import Column from dbt.contracts.connection import AdapterResponse from dbt.exceptions import DbtRuntimeError @@ -40,44 +34,6 @@ def cursor(self): return self._cursor -def convert_type(t: pa.DataType) -> str: - if pa.types.is_int64(t): - return "BIGINT" - elif pa.types.is_integer(t): - return "INTEGER" - elif pa.types.is_string(t): - return "TEXT" - elif pa.types.is_date(t): - return "DATE" - elif pa.types.is_time(t): - return "TIME" - elif pa.types.is_timestamp(t): - return "DATETIME" - elif pa.types.is_floating(t): - return "FLOAT" - elif pa.types.is_decimal(t): - return "DECIMAL" - elif pa.types.is_boolean(t): - return "BOOL" - elif pa.types.is_binary(t): - return "BINARY" - elif pa.types.is_interval(t): - return "INTERVAL" - elif pa.types.is_list(t): - field_type = t.field(0).type - if pa.types.is_integer(field_type): - return "INTEGERARRAY" - elif pa.types.is_string(field_type): - return "STRINGARRAY" - else: - return "ARRAY" - elif pa.types.is_struct(t) or pa.types.is_map(t): - # TODO: support detailed nested types - return "JSON" - else: - return "UNKNOWN" - - class LocalEnvironment(Environment): def __init__(self, credentials: credentials.DuckDBCredentials): self.conn = self.initialize_db(credentials) @@ -130,12 +86,6 @@ def load_source(self, plugin_name: str, source_config: utils.SourceConfig): cursor.close() handle.close() - def create_columns(self, cursor) -> List[Column]: - columns = [] - for field in cursor.fetch_record_batch().schema: - columns.append(column.DuckDBColumn.create(field.name, convert_type(field.type))) - return columns - def close(self): if self.conn: self.conn.close() diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 43a87cd7..cfb8f814 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -192,8 +192,16 @@ def get_rows_different_sql( @available.parse(lambda *a, **k: []) def get_column_schema_from_query(self, sql: str) -> List[DBTColumn]: """Get a list of the Columns with names and data types from the given sql.""" - _, cursor = self.connections.add_select_query(sql) - return DuckDBConnectionManager.env().create_columns(cursor) + + # Taking advantage of yet another amazing DuckDB SQL feature right here: the + # ability to DESCRIBE a query instead of a relation + describe_sql = f"DESCRIBE ({sql})" + _, cursor = self.connections.add_select_query(describe_sql) + ret = [] + for row in cursor.fetchall(): + name, dtype = row[0], row[1] + ret.append(DuckDBColumn.create(name, dtype)) + return ret @classmethod def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]: diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 3f0d83fa..58b48c9f 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -13,35 +13,54 @@ class DuckDBColumnEqualSetup: + @pytest.fixture + def int_type(self): + return "INT" + + @pytest.fixture + def string_type(self): + return "VARCHAR" + @pytest.fixture def data_types(self, schema_int_type, int_type, string_type): # sql_column_value, schema_data_type, error_data_type return [ ["1", schema_int_type, int_type], ["'1'", string_type, string_type], - ["true", "bool", "bool"], - ["'2013-11-03 00:00:00-07'::timestamptz", "timestamptz", "DATETIME"], - ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "DATETIME"], - ["ARRAY['a','b','c']", "text[]", "STRINGARRAY"], - ["ARRAY[1,2,3]", "int[]", "INTEGERARRAY"], + ["true", "bool", "BOOL"], + ["'2013-11-03 00:00:00-07'::timestamp", "TIMESTAMP", "TIMESTAMP"], + ["ARRAY['a','b','c']", "VARCHAR[]", "VARCHAR[]"], + ["ARRAY[1,2,3]", "INTEGER[]", "INTEGER[]"], ["'1'::numeric", "numeric", "DECIMAL"], - ["""{'bar': 'baz', 'balance': 7.77, 'active': false}""", "struct(bar text, balance decimal, active boolean)", "json"], + [ + """'{"bar": "baz", "balance": 7.77, "active": false}'::json""", + "json", + "JSON", + ], ] -class TestTableConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseTableConstraintsColumnsEqual): +class TestTableConstraintsColumnsEqual( + DuckDBColumnEqualSetup, BaseTableConstraintsColumnsEqual +): pass -class TestViewConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseViewConstraintsColumnsEqual): +class TestViewConstraintsColumnsEqual( + DuckDBColumnEqualSetup, BaseViewConstraintsColumnsEqual +): pass -class TestIncrementalConstraintsColumnsEqual(DuckDBColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual): +class TestIncrementalConstraintsColumnsEqual( + DuckDBColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual +): pass -class TestTableConstraintsRuntimeDdlEnforcement(DuckDBColumnEqualSetup, BaseConstraintsRuntimeDdlEnforcement): +class TestTableConstraintsRuntimeDdlEnforcement( + DuckDBColumnEqualSetup, BaseConstraintsRuntimeDdlEnforcement +): pass @@ -59,12 +78,17 @@ def expected_error_messages(self): return ["NOT NULL constraint failed"] -class TestIncrementalConstraintsRollback(DuckDBColumnEqualSetup, BaseIncrementalConstraintsRollback): +class TestIncrementalConstraintsRollback( + DuckDBColumnEqualSetup, BaseIncrementalConstraintsRollback +): @pytest.fixture(scope="class") def expected_error_messages(self): return ["NOT NULL constraint failed"] - -class TestModelConstraintsRuntimeEnforcement(DuckDBColumnEqualSetup, BaseModelConstraintsRuntimeEnforcement): + + +class TestModelConstraintsRuntimeEnforcement( + DuckDBColumnEqualSetup, BaseModelConstraintsRuntimeEnforcement +): @pytest.fixture(scope="class") def expected_error_messages(self): return ["NOT NULL constraint failed"] diff --git a/tox.ini b/tox.ini index bcd4314a..5f8b66ad 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,15 @@ deps = -rdev-requirements.txt -e. +[testenv:{buenavista,py39,py310,py311,py}] +description = adapter functional testing using a Buena Vista server +skip_install = True +passenv = * +commands = {envpython} -m pytest --profile=buenavista {posargs} tests/functional/adapter +deps = + -rdev-requirements.txt + -e. + [testenv:{fsspec,py37,py38,py39,py310,py311,py}] description = adapter fsspec testing skip_install = True From 259fd3b31097df5ae8621bfd75a6b90c9abc8628 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 25 Apr 2023 21:16:14 -0700 Subject: [PATCH 17/18] simplify this back down --- dbt/adapters/duckdb/column.py | 12 ------------ dbt/adapters/duckdb/impl.py | 10 ++++------ 2 files changed, 4 insertions(+), 18 deletions(-) delete mode 100644 dbt/adapters/duckdb/column.py diff --git a/dbt/adapters/duckdb/column.py b/dbt/adapters/duckdb/column.py deleted file mode 100644 index 40a22104..00000000 --- a/dbt/adapters/duckdb/column.py +++ /dev/null @@ -1,12 +0,0 @@ -from dbt.adapters.base import Column - - -class DuckDBColumn(Column): - @property - def data_type(self): - # on duckdb, do not convert 'text' or 'varchar' to 'varchar()' - if self.dtype.lower() == "text" or ( - self.dtype.lower() == "character varying" and self.char_size is None - ): - return self.dtype - return super().data_type diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index cfb8f814..ad27531d 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -7,10 +7,9 @@ import duckdb from dbt.adapters.base import BaseRelation -from dbt.adapters.base.column import Column as DBTColumn +from dbt.adapters.base.column import Column from dbt.adapters.base.impl import ConstraintSupport from dbt.adapters.base.meta import available -from dbt.adapters.duckdb.column import DuckDBColumn from dbt.adapters.duckdb.connections import DuckDBConnectionManager from dbt.adapters.duckdb.glue import create_or_update_table from dbt.adapters.duckdb.relation import DuckDBRelation @@ -25,7 +24,6 @@ class DuckDBAdapter(SQLAdapter): ConnectionManager = DuckDBConnectionManager Relation = DuckDBRelation - Column = DuckDBColumn CONSTRAINT_SUPPORT = { ConstraintType.check: ConstraintSupport.ENFORCED, @@ -75,7 +73,7 @@ def register_glue_table( self, glue_database: str, table: str, - column_list: Sequence[DBTColumn], + column_list: Sequence[Column], location: str, file_format: str, ) -> None: @@ -190,7 +188,7 @@ def get_rows_different_sql( return sql @available.parse(lambda *a, **k: []) - def get_column_schema_from_query(self, sql: str) -> List[DBTColumn]: + def get_column_schema_from_query(self, sql: str) -> List[Column]: """Get a list of the Columns with names and data types from the given sql.""" # Taking advantage of yet another amazing DuckDB SQL feature right here: the @@ -200,7 +198,7 @@ def get_column_schema_from_query(self, sql: str) -> List[DBTColumn]: ret = [] for row in cursor.fetchall(): name, dtype = row[0], row[1] - ret.append(DuckDBColumn.create(name, dtype)) + ret.append(Column.create(name, dtype)) return ret @classmethod From 5a76e457c8004513920576eea082d58f4931e89c Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Wed, 26 Apr 2023 10:00:23 -0700 Subject: [PATCH 18/18] add in tz-aware timestamp test --- tests/functional/adapter/test_constraints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 58b48c9f..5862a638 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -29,6 +29,7 @@ def data_types(self, schema_int_type, int_type, string_type): ["'1'", string_type, string_type], ["true", "bool", "BOOL"], ["'2013-11-03 00:00:00-07'::timestamp", "TIMESTAMP", "TIMESTAMP"], + ["'2013-11-03 00:00:00-07'::timestamptz", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"], ["ARRAY['a','b','c']", "VARCHAR[]", "VARCHAR[]"], ["ARRAY[1,2,3]", "INTEGER[]", "INTEGER[]"], ["'1'::numeric", "numeric", "DECIMAL"],