From b56c356acfec06d0d26599b124fc99c73ef27f61 Mon Sep 17 00:00:00 2001 From: Chandra Sanapala Date: Wed, 24 Jul 2024 11:39:22 +0530 Subject: [PATCH 1/5] Make supported types in dialect yaml files as a map --- bft/dialects/types.py | 2 +- dialects/cudf.yaml | 20 ++++++++++---------- dialects/datafusion.yaml | 25 ++++++++++++------------- dialects/duckdb.yaml | 28 ++++++++++++++-------------- dialects/postgres.yaml | 24 ++++++++++++------------ dialects/snowflake.yaml | 18 +++++++++--------- dialects/sqlite.yaml | 16 ++++++++-------- dialects/velox_presto.yaml | 8 ++++---- 8 files changed, 70 insertions(+), 71 deletions(-) diff --git a/bft/dialects/types.py b/bft/dialects/types.py index a7f050f..879c160 100644 --- a/bft/dialects/types.py +++ b/bft/dialects/types.py @@ -83,7 +83,7 @@ class DialectFile(NamedTuple): scalar_functions: List[DialectFunction] aggregate_functions: List[DialectFunction] uri_to_func_prefix: Dict[str, str] - supported_types: List[str] + supported_types: Dict[str, str] class SqlMapping(NamedTuple): diff --git a/dialects/cudf.yaml b/dialects/cudf.yaml index bd2a760..7c0f037 100644 --- a/dialects/cudf.yaml +++ b/dialects/cudf.yaml @@ -26,16 +26,16 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - - i8 - - i16 - - i32 - - i64 - - fp32 - - fp64 - - bool - - str - - ts - - date + i8: int8 + i16: int16 + i32: int32 + i64: int64 + fp32: float32 + fp64: float64 + bool: bool + str: string + ts: datetime64[s] + date: datetime64[s] scalar_functions: - name: arithmetic.add local_name: add diff --git a/dialects/datafusion.yaml b/dialects/datafusion.yaml index 1f7b5b4..b717ef8 100644 --- a/dialects/datafusion.yaml +++ b/dialects/datafusion.yaml @@ -26,19 +26,18 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - - i8 - - i16 - - i32 - - i64 - - fp32 - - fp64 - - bool - - str - - date - - time - - ts - - tstz - - interval + i8: int8 + i16: int16 + i32: int32 + i64: int64 + fp32: float32 + fp64: float64 + bool: bool_ + str: string + date: timestamp + time: timestamp + ts: timestamp + tstz: timestamp scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/duckdb.yaml b/dialects/duckdb.yaml index 9a971f7..6e96f88 100644 --- a/dialects/duckdb.yaml +++ b/dialects/duckdb.yaml @@ -26,20 +26,20 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - - i8 - - i16 - - i32 - - i64 - - fp32 - - fp64 - - bool - - str - - date - - time - - ts - - tstz - - interval - - dec + i8: TINYINT, + i16: SMALLINT + i32: INTEGER + i64: BIGINT + fp32: REAL + fp64: DOUBLE + bool: BOOLEAN + str: VARCHAR + date: DATE + time: TIME + ts: TIMESTAMP + tstz: TIMESTAMPTZ + interval: INTERVAL + decimal: DECIMAL scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/postgres.yaml b/dialects/postgres.yaml index 2807fe1..6b155b4 100644 --- a/dialects/postgres.yaml +++ b/dialects/postgres.yaml @@ -26,18 +26,18 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - - i16 - - i32 - - i64 - - fp32 - - fp64 - - bool - - str - - date - - time - - ts - - tstz - - interval + i16: smallint + i32: integer + i64: bigint + fp32: float4 + fp64: float8 + bool: boolean + str: text + date: date + time: time + ts: timestamp + tstz: timestamptz + interval: interval scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/snowflake.yaml b/dialects/snowflake.yaml index f00f463..435aba7 100644 --- a/dialects/snowflake.yaml +++ b/dialects/snowflake.yaml @@ -26,15 +26,15 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - - fp64 - - bool - - str - - date - - time - - ts - - tsz - - interval - - dec + fp64: FLOAT + bool: BOOLEAN + str: VARCHAR + date: DATE + time: TIME + ts: TIMESTAMP + tstz: TIMESTAMPTZ + interval: INTERVAL + dec: NUMERIC scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/sqlite.yaml b/dialects/sqlite.yaml index 9fdfbc0..c9c4b8a 100644 --- a/dialects/sqlite.yaml +++ b/dialects/sqlite.yaml @@ -26,14 +26,14 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - - i8 - - i16 - - i32 - - i64 - - fp32 - - fp64 - - bool - - str + i8: TINYINT + i16: SMALLINT + i32: INT + i64: HUGEINT + fp32: REAL + fp64: REAL + bool: BOOLEAN + str: TEXT scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/velox_presto.yaml b/dialects/velox_presto.yaml index c1d47f7..1fda49d 100644 --- a/dialects/velox_presto.yaml +++ b/dialects/velox_presto.yaml @@ -26,10 +26,10 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - - i64 - - fp64 - - bool - - str + i64: i64 + fp64: fp64 + bool: boolean + str: string scalar_functions: - name: arithmetic.add local_name: + From 51b5bb70116a12674269d80267bdbaa9caf71826 Mon Sep 17 00:00:00 2001 From: Chandra Sanapala Date: Thu, 25 Jul 2024 15:38:20 +0530 Subject: [PATCH 2/5] Make supported types as map[sting]struct --- dialects/cudf.yaml | 30 ++++++++++++++++++--------- dialects/datafusion.yaml | 36 +++++++++++++++++++++----------- dialects/duckdb.yaml | 42 +++++++++++++++++++++++++------------- dialects/postgres.yaml | 36 +++++++++++++++++++++----------- dialects/snowflake.yaml | 27 ++++++++++++++++-------- dialects/sqlite.yaml | 24 ++++++++++++++-------- dialects/velox_presto.yaml | 12 +++++++---- 7 files changed, 138 insertions(+), 69 deletions(-) diff --git a/dialects/cudf.yaml b/dialects/cudf.yaml index 7c0f037..2080cd1 100644 --- a/dialects/cudf.yaml +++ b/dialects/cudf.yaml @@ -26,16 +26,26 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - i8: int8 - i16: int16 - i32: int32 - i64: int64 - fp32: float32 - fp64: float64 - bool: bool - str: string - ts: datetime64[s] - date: datetime64[s] + i8: + df_type_name: int8 + i16: + df_type_name: int16 + i32: + df_type_name: int32 + i64: + df_type_name: int64 + fp32: + df_type_name: float32 + fp64: + df_type_name: float64 + bool: + df_type_name: bool + str: + df_type_name: string + ts: + df_type_name: datetime64[s] + date: + df_type_name: datetime64[s] scalar_functions: - name: arithmetic.add local_name: add diff --git a/dialects/datafusion.yaml b/dialects/datafusion.yaml index b717ef8..9ee461d 100644 --- a/dialects/datafusion.yaml +++ b/dialects/datafusion.yaml @@ -26,18 +26,30 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - i8: int8 - i16: int16 - i32: int32 - i64: int64 - fp32: float32 - fp64: float64 - bool: bool_ - str: string - date: timestamp - time: timestamp - ts: timestamp - tstz: timestamp + i8: + sql_type_name: int8 + i16: + sql_type_name: int16 + i32: + sql_type_name: int32 + i64: + sql_type_name: int64 + fp32: + sql_type_name: float32 + fp64: + sql_type_name: float64 + bool: + sql_type_name: bool_ + str: + sql_type_name: string + date: + sql_type_name: timestamp + time: + sql_type_name: timestamp + ts: + sql_type_name: timestamp + tstz: + sql_type_name: timestamp scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/duckdb.yaml b/dialects/duckdb.yaml index 6e96f88..272a6e4 100644 --- a/dialects/duckdb.yaml +++ b/dialects/duckdb.yaml @@ -26,20 +26,34 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - i8: TINYINT, - i16: SMALLINT - i32: INTEGER - i64: BIGINT - fp32: REAL - fp64: DOUBLE - bool: BOOLEAN - str: VARCHAR - date: DATE - time: TIME - ts: TIMESTAMP - tstz: TIMESTAMPTZ - interval: INTERVAL - decimal: DECIMAL + i8: + sql_type_name: TINYINT, + i16: + sql_type_name: SMALLINT + i32: + sql_type_name: INTEGER + i64: + sql_type_name: BIGINT + fp32: + sql_type_name: REAL + fp64: + sql_type_name: DOUBLE + bool: + sql_type_name: BOOLEAN + str: + sql_type_name: VARCHAR + date: + sql_type_name: DATE + time: + sql_type_name: TIME + ts: + sql_type_name: TIMESTAMP + tstz: + sql_type_name: TIMESTAMPTZ + interval: + sql_type_name: INTERVAL + decimal: + sql_type_name: DECIMAL scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/postgres.yaml b/dialects/postgres.yaml index 6b155b4..4fd80a3 100644 --- a/dialects/postgres.yaml +++ b/dialects/postgres.yaml @@ -26,18 +26,30 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - i16: smallint - i32: integer - i64: bigint - fp32: float4 - fp64: float8 - bool: boolean - str: text - date: date - time: time - ts: timestamp - tstz: timestamptz - interval: interval + i16: + sql_type_name: smallint + i32: + sql_type_name: integer + i64: + sql_type_name: bigint + fp32: + sql_type_name: float4 + fp64: + sql_type_name: float8 + bool: + sql_type_name: boolean + str: + sql_type_name: text + date: + sql_type_name: date + time: + sql_type_name: time + ts: + sql_type_name: timestamp + tstz: + sql_type_name: timestamptz + interval: + sql_type_name: interval scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/snowflake.yaml b/dialects/snowflake.yaml index 435aba7..47876ed 100644 --- a/dialects/snowflake.yaml +++ b/dialects/snowflake.yaml @@ -26,15 +26,24 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - fp64: FLOAT - bool: BOOLEAN - str: VARCHAR - date: DATE - time: TIME - ts: TIMESTAMP - tstz: TIMESTAMPTZ - interval: INTERVAL - dec: NUMERIC + fp64: + sql_type_name: FLOAT + bool: + sql_type_name: BOOLEAN + str: + sql_type_name: VARCHAR + date: + sql_type_name: DATE + time: + sql_type_name: TIME + ts: + sql_type_name: TIMESTAMP + tstz: + sql_type_name: TIMESTAMPTZ + interval: + sql_type_name: INTERVAL + dec: + sql_type_name: NUMERIC scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/sqlite.yaml b/dialects/sqlite.yaml index c9c4b8a..9a16fd3 100644 --- a/dialects/sqlite.yaml +++ b/dialects/sqlite.yaml @@ -26,14 +26,22 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - i8: TINYINT - i16: SMALLINT - i32: INT - i64: HUGEINT - fp32: REAL - fp64: REAL - bool: BOOLEAN - str: TEXT + i8: + sql_type_name: TINYINT + i16: + sql_type_name: SMALLINT + i32: + sql_type_name: INT + i64: + sql_type_name: HUGEINT + fp32: + sql_type_name: REAL + fp64: + sql_type_name: REAL + bool: + sql_type_name: BOOLEAN + str: + sql_type_name: TEXT scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/velox_presto.yaml b/dialects/velox_presto.yaml index 1fda49d..afddeb1 100644 --- a/dialects/velox_presto.yaml +++ b/dialects/velox_presto.yaml @@ -26,10 +26,14 @@ dependencies: string: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: - i64: i64 - fp64: fp64 - bool: boolean - str: string + i64: + sql_type_name: i64 + fp64: + sql_type_name: fp64 + bool: + sql_type_name: boolean + str: + sql_type_name: string scalar_functions: - name: arithmetic.add local_name: + From d3aeb37b71313cd610179442b8993a4cf2a70e8f Mon Sep 17 00:00:00 2001 From: Chandra Sanapala Date: Mon, 16 Sep 2024 13:02:38 +0530 Subject: [PATCH 3/5] Set supported_as_column to false for snowflake interval --- dialects/snowflake.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dialects/snowflake.yaml b/dialects/snowflake.yaml index 47876ed..bd60a62 100644 --- a/dialects/snowflake.yaml +++ b/dialects/snowflake.yaml @@ -42,6 +42,7 @@ supported_types: sql_type_name: TIMESTAMPTZ interval: sql_type_name: INTERVAL + supported_as_column: false dec: sql_type_name: NUMERIC scalar_functions: From 9d1df845f13dc08914f1e27c0aa9b96ea5423683 Mon Sep 17 00:00:00 2001 From: Chandra Sanapala Date: Mon, 16 Sep 2024 19:06:08 +0530 Subject: [PATCH 4/5] Address review comments --- bft/dialects/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bft/dialects/types.py b/bft/dialects/types.py index 879c160..f36ff58 100644 --- a/bft/dialects/types.py +++ b/bft/dialects/types.py @@ -83,7 +83,7 @@ class DialectFile(NamedTuple): scalar_functions: List[DialectFunction] aggregate_functions: List[DialectFunction] uri_to_func_prefix: Dict[str, str] - supported_types: Dict[str, str] + supported_types: Dict[str, Dict] class SqlMapping(NamedTuple): From c97604508e000b43d51ac4d82b927e8b33aa8936 Mon Sep 17 00:00:00 2001 From: Chandra Sanapala Date: Mon, 16 Sep 2024 19:32:44 +0530 Subject: [PATCH 5/5] use lower case for local types --- dialects/duckdb.yaml | 28 ++++++++++++++-------------- dialects/snowflake.yaml | 18 +++++++++--------- dialects/sqlite.yaml | 16 ++++++++-------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/dialects/duckdb.yaml b/dialects/duckdb.yaml index 272a6e4..a8e4f80 100644 --- a/dialects/duckdb.yaml +++ b/dialects/duckdb.yaml @@ -27,33 +27,33 @@ dependencies: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: i8: - sql_type_name: TINYINT, + sql_type_name: tinyint i16: - sql_type_name: SMALLINT + sql_type_name: smallint i32: - sql_type_name: INTEGER + sql_type_name: integer i64: - sql_type_name: BIGINT + sql_type_name: bigint fp32: - sql_type_name: REAL + sql_type_name: real fp64: - sql_type_name: DOUBLE + sql_type_name: double bool: - sql_type_name: BOOLEAN + sql_type_name: boolean str: - sql_type_name: VARCHAR + sql_type_name: varchar date: - sql_type_name: DATE + sql_type_name: date time: - sql_type_name: TIME + sql_type_name: time ts: - sql_type_name: TIMESTAMP + sql_type_name: timestamp tstz: - sql_type_name: TIMESTAMPTZ + sql_type_name: timestamptz interval: - sql_type_name: INTERVAL + sql_type_name: interval decimal: - sql_type_name: DECIMAL + sql_type_name: decimal scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/snowflake.yaml b/dialects/snowflake.yaml index bd60a62..9f37c83 100644 --- a/dialects/snowflake.yaml +++ b/dialects/snowflake.yaml @@ -27,24 +27,24 @@ dependencies: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: fp64: - sql_type_name: FLOAT + sql_type_name: float bool: - sql_type_name: BOOLEAN + sql_type_name: boolean str: - sql_type_name: VARCHAR + sql_type_name: varchar date: - sql_type_name: DATE + sql_type_name: date time: - sql_type_name: TIME + sql_type_name: time ts: - sql_type_name: TIMESTAMP + sql_type_name: timestamp tstz: - sql_type_name: TIMESTAMPTZ + sql_type_name: timestamptz interval: - sql_type_name: INTERVAL + sql_type_name: interval supported_as_column: false dec: - sql_type_name: NUMERIC + sql_type_name: numeric scalar_functions: - name: arithmetic.add local_name: + diff --git a/dialects/sqlite.yaml b/dialects/sqlite.yaml index 9a16fd3..930d051 100644 --- a/dialects/sqlite.yaml +++ b/dialects/sqlite.yaml @@ -27,21 +27,21 @@ dependencies: https://github.com/substrait-io/substrait/blob/main/extensions/substrait/extensions/functions_string.yaml supported_types: i8: - sql_type_name: TINYINT + sql_type_name: tinyint i16: - sql_type_name: SMALLINT + sql_type_name: smallint i32: - sql_type_name: INT + sql_type_name: int i64: - sql_type_name: HUGEINT + sql_type_name: hugeint fp32: - sql_type_name: REAL + sql_type_name: real fp64: - sql_type_name: REAL + sql_type_name: real bool: - sql_type_name: BOOLEAN + sql_type_name: boolean str: - sql_type_name: TEXT + sql_type_name: text scalar_functions: - name: arithmetic.add local_name: +