Merge branch 'master' into kafka-sample-values

datahub-project · Jan 14, 2025 · 6da8f58 · 6da8f58
2 parents ce242bd + e1d57e3
commit 6da8f58
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 3 deletions.
diff --git a/build.gradle b/build.gradle
@@ -195,7 +195,7 @@ project.ext.externalDependency = [
     'kafkaAvroSerde': "io.confluent:kafka-streams-avro-serde:$kafkaVersion",
     'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
     'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion-ccs",
-    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.4',
+    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.5',
     'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
     'logbackClassicJava8' : "ch.qos.logback:logback-classic:$logbackClassicJava8",
     'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
@@ -276,6 +276,7 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
         "HLLSKETCH": NullType,
         "TIMETZ": TimeType,
         "VARBYTE": StringType,
+        "SUPER": NullType,
     }
 
     def get_platform_instance_id(self) -> str:

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
@@ -93,7 +93,7 @@
     "regtype": None,
     "regrole": None,
     "regnamespace": None,
-    "super": None,
+    "super": NullType,
     "uuid": StringType,
     "pg_lsn": None,
     "tsvector": None,  # text search vector

diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py
@@ -9,7 +9,12 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.source.dbt import dbt_cloud
 from datahub.ingestion.source.dbt.dbt_cloud import DBTCloudConfig
-from datahub.ingestion.source.dbt.dbt_common import DBTNode
+from datahub.ingestion.source.dbt.dbt_common import (
+    DBTNode,
+    DBTSourceReport,
+    NullTypeClass,
+    get_column_type,
+)
 from datahub.ingestion.source.dbt.dbt_core import (
     DBTCoreConfig,
     DBTCoreSource,
@@ -461,3 +466,30 @@ def test_dbt_time_parsing() -> None:
         assert timestamp.tzinfo is not None and timestamp.tzinfo.utcoffset(
             timestamp
         ) == timedelta(0)
+
+
+def test_get_column_type_redshift():
+    report = DBTSourceReport()
+    dataset_name = "test_dataset"
+
+    # Test 'super' type which should not show any warnings/errors
+    result_super = get_column_type(report, dataset_name, "super", "redshift")
+    assert isinstance(result_super.type, NullTypeClass)
+    assert (
+        len(report.infos) == 0
+    ), "No warnings should be generated for known SUPER type"
+
+    # Test unknown type, which generates a warning but resolves to NullTypeClass
+    unknown_type = "unknown_type"
+    result_unknown = get_column_type(report, dataset_name, unknown_type, "redshift")
+    assert isinstance(result_unknown.type, NullTypeClass)
+
+    # exact warning message for an unknown type
+    expected_context = f"{dataset_name} - {unknown_type}"
+    messages = [info for info in report.infos if expected_context in str(info.context)]
+    assert len(messages) == 1
+    assert messages[0].title == "Unable to map column types to DataHub types"
+    assert (
+        messages[0].message
+        == "Got an unexpected column type. The column's parsed field type will not be populated."
+    )