diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index be3334373594c..389bb638ed401 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -247,6 +247,12 @@ def _normalize_external_tables(self, external_tables=None) -> ExternalData | Non n += 1 if not (schema := obj.schema): raise TypeError(f"Schema is empty for external table {name}") + if null_fields := schema.null_fields: + raise com.IbisTypeError( + "ClickHouse doesn't support NULL-typed fields. " + "Consider assigning a type through casting or on construction. " + f"Got null typed fields: {null_fields}" + ) structure = [ f"{name} {type_mapper.to_string(typ.copy(nullable=not typ.is_nested()))}" diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index e681056461fe6..5dd8bea900561 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -174,11 +174,11 @@ def create_table( if schema is None: schema = table.schema() - if schema.nulls: - raise exc.UnsupportedBackendType( + if null_fields := schema.null_fields: + raise exc.IbisTypeError( "DuckDB does not support creating tables with NULL typed columns. " "Ensure that every column has non-NULL type. " - f"NULL columns: {schema.nulls}" + f"NULL columns: {null_fields}" ) if overwrite: @@ -1432,7 +1432,7 @@ def to_pyarrow( ) -> pa.Table: table = self._to_duckdb_relation(expr, params=params, limit=limit).arrow() schema = expr.as_table().schema() - if not schema.nulls: + if not schema.null_fields: return expr.__pyarrow_result__(table) arrays = [ diff --git a/ibis/backends/duckdb/tests/test_client.py b/ibis/backends/duckdb/tests/test_client.py index 8dc69b9daf119..5382a7f0c0f61 100644 --- a/ibis/backends/duckdb/tests/test_client.py +++ b/ibis/backends/duckdb/tests/test_client.py @@ -428,15 +428,17 @@ def test_create_table_with_nulls(con): schema = t.schema() assert schema == ibis.schema({"a": "null"}) - assert schema.nulls == ("a",) + assert schema.null_fields == ("a",) name = gen_name("duckdb_all_nulls") - with pytest.raises(com.UnsupportedBackendType, match="NULL typed columns"): + match = "NULL typed columns" + + with pytest.raises(com.IbisTypeError, match=match): con.create_table(name, obj=t) - with pytest.raises(com.UnsupportedBackendType, match="NULL typed columns"): + with pytest.raises(com.IbisTypeError, match=match): con.create_table(name, obj=t, schema=schema) - with pytest.raises(com.UnsupportedBackendType, match="NULL typed columns"): + with pytest.raises(com.IbisTypeError, match=match): con.create_table(name, schema=schema) diff --git a/ibis/backends/exasol/__init__.py b/ibis/backends/exasol/__init__.py index b1bdb134bd67a..40b3e6e359f68 100644 --- a/ibis/backends/exasol/__init__.py +++ b/ibis/backends/exasol/__init__.py @@ -281,7 +281,7 @@ def _in_memory_table_exists(self, name: str) -> bool: def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: schema = op.schema - if null_columns := [col for col, dtype in schema.items() if dtype.is_null()]: + if null_columns := schema.null_fields: raise com.IbisTypeError( "Exasol cannot yet reliably handle `null` typed columns; " f"got null typed columns: {null_columns}" diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index 97ea8a06a8f54..7519f4feb951e 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -374,6 +374,13 @@ def execute(self, expr: ir.Expr, **kwargs: Any) -> Any: self._register_udfs(expr) table_expr = expr.as_table() + + if null_columns := table_expr.schema().null_fields: + raise exc.IbisTypeError( + f"{self.name} cannot yet reliably handle `null` typed columns; " + f"got null typed columns: {null_columns}" + ) + sql = self.compile(table_expr, **kwargs) df = self._table_env.sql_query(sql).to_pandas() diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py index 522c0538bf172..cd384e9fcf6f8 100644 --- a/ibis/backends/impala/__init__.py +++ b/ibis/backends/impala/__init__.py @@ -1229,7 +1229,7 @@ def _in_memory_table_exists(self, name: str) -> bool: def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: schema = op.schema - if null_columns := [col for col, dtype in schema.items() if dtype.is_null()]: + if null_columns := schema.null_fields: raise com.IbisTypeError( "Impala cannot yet reliably handle `null` typed columns; " f"got null typed columns: {null_columns}" diff --git a/ibis/backends/mssql/__init__.py b/ibis/backends/mssql/__init__.py index d424d903179ea..392a20d2493d8 100644 --- a/ibis/backends/mssql/__init__.py +++ b/ibis/backends/mssql/__init__.py @@ -748,7 +748,7 @@ def _in_memory_table_exists(self, name: str) -> bool: def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: schema = op.schema - if null_columns := [col for col, dtype in schema.items() if dtype.is_null()]: + if null_columns := schema.null_fields: raise com.IbisTypeError( "MS SQL cannot yet reliably handle `null` typed columns; " f"got null typed columns: {null_columns}" diff --git a/ibis/backends/mysql/__init__.py b/ibis/backends/mysql/__init__.py index 2490bc275ef3f..8f0c87396265e 100644 --- a/ibis/backends/mysql/__init__.py +++ b/ibis/backends/mysql/__init__.py @@ -479,7 +479,7 @@ def _in_memory_table_exists(self, name: str) -> bool: def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: schema = op.schema - if null_columns := [col for col, dtype in schema.items() if dtype.is_null()]: + if null_columns := schema.null_fields: raise com.IbisTypeError( "MySQL cannot yet reliably handle `null` typed columns; " f"got null typed columns: {null_columns}" diff --git a/ibis/backends/oracle/__init__.py b/ibis/backends/oracle/__init__.py index b189ba3d37783..40f1039e009ef 100644 --- a/ibis/backends/oracle/__init__.py +++ b/ibis/backends/oracle/__init__.py @@ -524,6 +524,11 @@ def _in_memory_table_exists(self, name: str) -> bool: def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: schema = op.schema + if null_columns := schema.null_fields: + raise exc.IbisTypeError( + f"{self.name} cannot yet reliably handle `null` typed columns; " + f"got null typed columns: {null_columns}" + ) name = op.name quoted = self.compiler.quoted diff --git a/ibis/backends/postgres/__init__.py b/ibis/backends/postgres/__init__.py index ddc3d4e169c4f..9b33b2a65df5d 100644 --- a/ibis/backends/postgres/__init__.py +++ b/ibis/backends/postgres/__init__.py @@ -108,7 +108,7 @@ def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: from psycopg2.extras import execute_batch schema = op.schema - if null_columns := [col for col, dtype in schema.items() if dtype.is_null()]: + if null_columns := schema.null_fields: raise exc.IbisTypeError( f"{self.name} cannot yet reliably handle `null` typed columns; " f"got null typed columns: {null_columns}" diff --git a/ibis/backends/risingwave/__init__.py b/ibis/backends/risingwave/__init__.py index 05927651b4b4a..c0a31717a411a 100644 --- a/ibis/backends/risingwave/__init__.py +++ b/ibis/backends/risingwave/__init__.py @@ -279,7 +279,7 @@ def _in_memory_table_exists(self, name: str) -> bool: def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: schema = op.schema - if null_columns := [col for col, dtype in schema.items() if dtype.is_null()]: + if null_columns := schema.null_fields: raise com.IbisTypeError( f"{self.name} cannot yet reliably handle `null` typed columns; " f"got null typed columns: {null_columns}" diff --git a/ibis/backends/tests/test_export.py b/ibis/backends/tests/test_export.py index 57bca509b2cc1..a110d1ddd1246 100644 --- a/ibis/backends/tests/test_export.py +++ b/ibis/backends/tests/test_export.py @@ -7,6 +7,7 @@ from pytest import param import ibis +import ibis.common.exceptions as com import ibis.expr.datatypes as dt from ibis import util from ibis.backends.tests.errors import ( @@ -586,9 +587,27 @@ def test_scalar_to_memory(limit, awards_players, output_format, converter): expr = awards_players.filter(awards_players.awardID == "DEADBEEF").yearID.min() res = method(expr) + assert converter(res) is None +# flink +@pytest.mark.notyet( + [ + "clickhouse", + "exasol", + "flink", + "impala", + "mssql", + "mysql", + "oracle", + "postgres", + "risingwave", + "trino", + ], + raises=com.IbisTypeError, + reason="unable to handle null typed columns as input", +) def test_all_null_column(con): t = ibis.memtable({"a": [None]}) result = con.to_pyarrow(t) diff --git a/ibis/backends/trino/__init__.py b/ibis/backends/trino/__init__.py index 9ee1d628a0072..6becdc3062aea 100644 --- a/ibis/backends/trino/__init__.py +++ b/ibis/backends/trino/__init__.py @@ -565,7 +565,7 @@ def _in_memory_table_exists(self, name: str) -> bool: def _register_in_memory_table(self, op: ops.InMemoryTable) -> None: schema = op.schema - if null_columns := [col for col, dtype in schema.items() if dtype.is_null()]: + if null_columns := schema.null_fields: raise com.IbisTypeError( "Trino cannot yet reliably handle `null` typed columns; " f"got null typed columns: {null_columns}" diff --git a/ibis/expr/schema.py b/ibis/expr/schema.py index e40bdfd600ff0..51026744065b1 100644 --- a/ibis/expr/schema.py +++ b/ibis/expr/schema.py @@ -71,7 +71,7 @@ def geospatial(self) -> tuple[str, ...]: return tuple(name for name, typ in self.fields.items() if typ.is_geospatial()) @attribute - def nulls(self) -> tuple[str, ...]: + def null_fields(self) -> tuple[str, ...]: return tuple(name for name, typ in self.fields.items() if typ.is_null()) @attribute diff --git a/ibis/expr/tests/test_schema.py b/ibis/expr/tests/test_schema.py index 7afcbe55dbc72..a9711c07160f5 100644 --- a/ibis/expr/tests/test_schema.py +++ b/ibis/expr/tests/test_schema.py @@ -474,7 +474,7 @@ def test_schema_from_to_pandas_dtypes(): assert restored_dtypes == expected_dtypes -def test_nulls(): - assert sch.schema({"a": "int64", "b": "string"}).nulls == () - assert sch.schema({"a": "null", "b": "string"}).nulls == ("a",) - assert sch.schema({"a": "null", "b": "null"}).nulls == ("a", "b") +def test_null_fields(): + assert sch.schema({"a": "int64", "b": "string"}).null_fields == () + assert sch.schema({"a": "null", "b": "string"}).null_fields == ("a",) + assert sch.schema({"a": "null", "b": "null"}).null_fields == ("a", "b")