Merge branch 'main' into support-python-submissions

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,72 @@
+name: 🐞 Bug
+description: Report a bug or an issue you've found in dbt-athena
+title: "[Bug] <title>"
+labels: ["bug"]
+body:
+- type: markdown
+  attributes:
+    value: |
+      Thanks for taking the time to fill out this bug report!
+- type: checkboxes
+  attributes:
+    label: Is this a new bug in dbt-athena?
+    description: >
+      If this is a request for help or troubleshooting code in your own dbt project,
+      please join [dbt Community Slack](https://www.getdbt.com/community/join-the-community/)
+      #db-athena or open a [Discussion question](https://github.com/dbt-athena/dbt-athena/discussions).
+
+      Please search to see if an issue already exists for the bug you encountered.
+    options:
+      - label: I believe this is a new bug in dbt-athena
+        required: true
+      - label: I have searched the existing issues, and I could not find an existing issue for this bug
+        required: true
+- type: textarea
+  attributes:
+    label: Current Behavior
+    description: A concise description of what you're experiencing.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Expected Behavior
+    description: A concise description of what you expected to happen.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Steps To Reproduce
+    description: Steps to reproduce the behavior.
+    placeholder: |
+      1. In this environment...
+      2. With this config...
+      3. Run '...'
+      4. See error...
+  validations:
+    required: false
+- type: textarea
+  attributes:
+    label: Environment
+    description: |
+      examples:
+        - **OS**: Ubuntu 20.04
+        - **Python**: 3.9.12 (`python3 --version`)
+        - **dbt-core**: 1.1.1 (`dbt --version`)
+        - **dbt-athena-community**: 1.1.1
+    value: |
+      - OS:
+      - Python:
+      - dbt:
+      - dbt-athena-community:
+    render: markdown
+  validations:
+    required: false
+- type: textarea
+  attributes:
+    label: Additional Context
+    description: |
+      Links? References? Anything that will give us more context about the issue you are encountering!
+
+      **Tip**: You can attach images or log files by clicking this area to highlight it and then dragging files in.
+  validations:
+    required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,55 @@
+name: ✨ Feature
+description: Propose a straightforward extension of dbt functionality
+title: "[Feature] <title>"
+labels: ["feature"]
+body:
+- type: markdown
+  attributes:
+    value: |
+      Thanks for taking the time to fill out this feature request!
+- type: checkboxes
+  attributes:
+    label: Is this your first time submitting a feature request?
+    description: >
+      We want to make sure that features are distinct and discoverable,
+      so that other members of the community can find them and offer their thoughts.
+
+      Issues are the right place to request straightforward extensions of existing dbt functionality.
+      For "big ideas" about future capabilities of dbt, we ask that you open a
+    options:
+      - label: I have searched the existing issues, and I could not find an existing issue for this feature
+        required: true
+- type: textarea
+  attributes:
+    label: Describe the feature
+    description: A clear and concise description of what you want to happen.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Describe alternatives you've considered
+    description: |
+      A clear and concise description of any alternative solutions or features you've considered.
+  validations:
+    required: false
+- type: textarea
+  attributes:
+    label: Who will this benefit?
+    description: |
+      What kind of use case will this feature be useful for?
+      Please be specific and provide examples, this will help us prioritize properly.
+  validations:
+    required: false
+- type: input
+  attributes:
+    label: Are you interested in contributing this feature?
+    description: Let us know if you want to write some code, and how we can help.
+  validations:
+    required: false
+- type: textarea
+  attributes:
+    label: Anything else?
+    description: |
+      Links? References? Anything that will give us more context about the feature you are suggesting!
+  validations:
+    required: false

.github/ISSUE_TEMPLATE/other.yml

@@ -0,0 +1,30 @@
+name: Other
+description: Anything that is not a bug or a feature
+title: "<title>"
+labels: []
+body:
+- type: checkboxes
+  attributes:
+    label: Is this your first time submitting an issue in dbt-athena?
+    description: >
+      We want to make sure that features are distinct and discoverable,
+      so that other members of the community can find them and offer their thoughts.
+
+      Issues are the right place to request straightforward extensions of existing dbt functionality.
+      For "big ideas" about future capabilities of dbt, we ask that you open a
+    options:
+      - label: I have searched the existing issues, and I could not find an existing issue for this feature
+        required: true
+- type: textarea
+  attributes:
+    label: Describe your request/question
+    description: A clear and concise description of what you want to happen.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Anything else?
+    description: |
+      Links? References? Anything that will give us more context about the requests or question
+  validations:
+    required: false

dbt/adapters/athena/connections.py

@@ -165,6 +165,9 @@ def inner() -> AthenaCursor:
                 cache_size=cache_size,
                 cache_expiration_time=cache_expiration_time,
             )
+
+            LOGGER.debug(f"Athena query ID {query_id}")
+
             query_execution = self._executor.submit(self._collect_result_set, query_id).result()
             if query_execution.state == AthenaQueryExecution.STATE_SUCCEEDED:
                 self.result_set = self._result_set_class(

dbt/adapters/athena/impl.py

@@ -126,6 +126,8 @@ class AthenaAdapter(SQLAdapter):
     AdapterSpecificConfigs = AthenaConfig
     Column = AthenaColumn
 
+    quote_character: str = '"'  # Presto quote character
+
     # There is no such concept as constraints in Athena
     CONSTRAINT_SUPPORT = {
         ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
@@ -399,9 +401,22 @@ def clean_up_table(self, relation: AthenaRelation) -> None:
         if table_location := self.get_glue_table_location(relation):
             self.delete_from_s3(table_location)
 
+    def quote(self, identifier: str) -> str:
+        return f"{self.quote_character}{identifier}{self.quote_character}"
+
     @available
-    def quote_seed_column(self, column: str, quote_config: Optional[bool]) -> str:
-        return str(super().quote_seed_column(column, False))
+    def quote_seed_column(
+        self, column: str, quote_config: Optional[bool], quote_character: Optional[str] = None
+    ) -> str:
+        if quote_character:
+            old_value = self.quote_character
+            object.__setattr__(self, "quote_character", quote_character)
+            quoted_column = str(super().quote_seed_column(column, quote_config))
+            object.__setattr__(self, "quote_character", old_value)
+        else:
+            quoted_column = str(super().quote_seed_column(column, quote_config))
+
+        return quoted_column
 
     @available
     def upload_seed_to_s3(
@@ -876,9 +891,7 @@ def _get_glue_table_versions_to_expire(self, relation: AthenaRelation, to_keep:
         return table_versions_ordered[int(to_keep) :]
 
     @available
-    def expire_glue_table_versions(
-        self, relation: AthenaRelation, to_keep: int, delete_s3: bool
-    ) -> List[TableVersionTypeDef]:
+    def expire_glue_table_versions(self, relation: AthenaRelation, to_keep: int, delete_s3: bool) -> List[str]:
         conn = self.connections.get_thread_connection()
         creds = conn.credentials
         client = conn.handle
@@ -911,11 +924,9 @@ def expire_glue_table_versions(
                 LOGGER.debug(f"Deleted version {version} of table {relation.render()} ")
                 if delete_s3:
                     self.delete_from_s3(location)
+                    LOGGER.debug(f"{location} was deleted")
             except Exception as err:
                 LOGGER.debug(f"There was an error when expiring table version {version} with error: {err}")
-
-            LOGGER.debug(f"{location} was deleted")
-
         return deleted_versions
 
     @available
@@ -969,11 +980,12 @@ def persist_docs_to_glue(
             glue_table_description = table.get("Description", "")
             # Get current description parameter from Glue
             glue_table_comment = table["Parameters"].get("comment", "")
-            # Update description if it's different
+            # Check that description is already attached to Glue table
             if clean_table_description != glue_table_description or clean_table_description != glue_table_comment:
-                table_input["Description"] = clean_table_description
-                table_parameters["comment"] = clean_table_description
                 need_to_update_table = True
+            # Save dbt description
+            table_input["Description"] = clean_table_description
+            table_parameters["comment"] = clean_table_description
 
             # Get dbt model meta if available
             meta: Dict[str, Any] = model.get("config", {}).get("meta", {})
@@ -993,9 +1005,9 @@ def persist_docs_to_glue(
                         # Check that meta value is already attached to Glue table
                         current_meta_value: Optional[str] = table_parameters.get(meta_key)
                         if current_meta_value is None or current_meta_value != meta_value:
-                            # Update Glue table parameter only if needed
-                            table_parameters[meta_key] = meta_value
                             need_to_update_table = True
+                        # Save Glue table parameter
+                        table_parameters[meta_key] = meta_value
                     else:
                         LOGGER.warning(f"Meta value for key '{meta_key}' is not supported and will be ignored")
                 else:
@@ -1013,10 +1025,11 @@ def persist_docs_to_glue(
                     clean_col_comment = ellipsis_comment(clean_sql_comment(col_comment))
                     # Get current column comment from Glue
                     glue_col_comment = col_obj.get("Comment", "")
-                    # Update column description if it's different
+                    # Check that column description is already attached to Glue table
                     if glue_col_comment != clean_col_comment:
-                        col_obj["Comment"] = clean_col_comment
                         need_to_update_table = True
+                    # Save column description from dbt
+                    col_obj["Comment"] = clean_col_comment
 
         # Update Glue Table only if table/column description is modified.
         # It prevents redundant schema version creating after incremental runs.

dbt/include/athena/macros/adapters/persist_docs.sql

@@ -1,12 +1,14 @@
-{% macro athena__persist_docs(relation, model, for_relation, for_columns) -%}
+{% macro athena__persist_docs(relation, model, for_relation=true, for_columns=true) -%}
   {% set persist_relation_docs = for_relation and config.persist_relation_docs() and model.description %}
   {% set persist_column_docs = for_columns and config.persist_column_docs() and model.columns %}
-  {% set skip_archive_table_version = not is_incremental() %}
   {% if persist_relation_docs or persist_column_docs %}
-    {% do adapter.persist_docs_to_glue(relation,
-                                       model,
-                                       persist_relation_docs,
-                                       persist_column_docs,
-                                       skip_archive_table_version=skip_archive_table_version) %}}
+    {% do adapter.persist_docs_to_glue(
+            relation=relation,
+            model=model,
+            persist_relation_docs=persist_relation_docs,
+            persist_column_docs=persist_column_docs,
+            skip_archive_table_version=true
+         )
+    %}}
   {% endif %}
 {% endmacro %}

dbt/include/athena/macros/materializations/models/incremental/incremental.sql

@@ -5,7 +5,7 @@
   {% set model_language = model['language'] %}
   {% set strategy = validate_get_incremental_strategy(raw_strategy, table_type) %}
   {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}
-
+  {% set versions_to_keep = config.get('versions_to_keep', 1) | as_number %}
   {% set lf_tags_config = config.get('lf_tags_config') %}
   {% set lf_grants = config.get('lf_grants') %}
   {% set partitioned_by = config.get('partitioned_by') %}
@@ -156,6 +156,8 @@
 
   {% do persist_docs(target_relation, model) %}
 
+  {% do adapter.expire_glue_table_versions(target_relation, versions_to_keep, False) %}
+
   {{ return({'relations': [target_relation]}) }}
 
 {%- endmaterialization %}

dbt/include/athena/macros/materializations/models/view/view.sql

@@ -8,7 +8,7 @@
 
     {% set to_return = create_or_replace_view(run_outside_transaction_hooks=False) %}
 
-    {% do adapter.expire_glue_table_versions(target_relation, versions_to_keep, True) %}
+    {% do adapter.expire_glue_table_versions(target_relation, versions_to_keep, False) %}
 
     {% set target_relation = this.incorporate(type='view') %}
     {% do persist_docs(target_relation, model) %}

dbt/include/athena/macros/materializations/seeds/helpers.sql

@@ -64,7 +64,7 @@
           {%- set type = column_override.get(col_name, inferred_type) -%}
           {%- set type = type if type != "string" else "varchar" -%}
           {%- set column_name = (col_name | string) -%}
-          {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ ddl_data_type(type) }} {%- if not loop.last -%}, {% endif -%}
+          {{ adapter.quote_seed_column(column_name, quote_seed_column, "`") }} {{ ddl_data_type(type) }} {%- if not loop.last -%}, {% endif -%}
         {%- endfor -%}
     )
     location '{{ location }}'
@@ -131,7 +131,7 @@
     create external table {{ tmp_relation.render_hive() }} (
         {%- for col_name in agate_table.column_names -%}
             {%- set column_name = (col_name | string) -%}
-            {{ adapter.quote_seed_column(column_name, quote_seed_column) }} string {%- if not loop.last -%}, {% endif -%}
+            {{ adapter.quote_seed_column(column_name, quote_seed_column, "`") }} string {%- if not loop.last -%}, {% endif -%}
         {%- endfor -%}
     )
     row format serde 'org.apache.hadoop.hive.serde2.OpenCSVSerde'

dev-requirements.txt

@@ -1,11 +1,11 @@
 autoflake~=1.7
-black~=23.11
-boto3-stubs[s3]~=1.29
-dbt-tests-adapter~=1.7.2
+black~=23.12
+boto3-stubs[s3]~=1.34
+dbt-tests-adapter~=1.7.4
 flake8~=6.1
 Flake8-pyproject~=1.2
-isort~=5.11
-moto~=4.2.10
+isort~=5.13
+moto~=4.2.12
 pre-commit~=3.5
 pyparsing~=3.1.1
 pytest~=7.4

tests/functional/adapter/test_quote_seed_column.py

@@ -0,0 +1,39 @@
+import pytest
+
+from dbt.tests.adapter.basic.files import (
+    base_materialized_var_sql,
+    base_table_sql,
+    base_view_sql,
+    schema_base_yml,
+    seeds_base_csv,
+)
+from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
+
+seed_base_csv_underscore_column = seeds_base_csv.replace("name", "_name")
+
+quote_columns_seed_schema = """
+
+seeds:
+- name: base
+  config:
+    quote_columns: true
+
+"""
+
+
+class TestSimpleMaterializationsHive(BaseSimpleMaterializations):
+    @pytest.fixture(scope="class")
+    def models(self):
+        schema = schema_base_yml + quote_columns_seed_schema
+        return {
+            "view_model.sql": base_view_sql,
+            "table_model.sql": base_table_sql,
+            "swappable.sql": base_materialized_var_sql,
+            "schema.yml": schema,
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "base.csv": seed_base_csv_underscore_column,
+        }