Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(examples): use memtables for examples to allow backends that do not support temporary tables to support examples #10094

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ibis/backends/datafusion/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import datafusion as df
import pyarrow as pa
import pyarrow.dataset as ds
import pyarrow_hotfix # noqa: F401
import sqlglot as sg
import sqlglot.expressions as sge
Expand Down Expand Up @@ -372,7 +373,7 @@ def _register(
self.con.deregister_table(table_name)
self.con.register_record_batches(table_name, [[source]])
return self.table(table_name)
elif isinstance(source, pa.dataset.Dataset):
elif isinstance(source, ds.Dataset):
self.con.deregister_table(table_name)
self.con.register_dataset(table_name, source)
return self.table(table_name)
Expand Down
9 changes: 5 additions & 4 deletions ibis/backends/datafusion/tests/test_register.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import pandas as pd
import pyarrow as pa
import pyarrow.dataset as ds
import pytest

import ibis
Expand Down Expand Up @@ -34,22 +33,24 @@ def test_register_pandas(conn):
df = pd.DataFrame({"x": [1, 2, 3]})
with pytest.warns(FutureWarning, match="v9.1"):
conn.register(df, "my_table")
assert conn.table("my_table").x.sum().execute() == 6
assert conn.table("my_table").x.sum().execute() == 6


def test_register_batches(conn):
batch = pa.record_batch([pa.array([1, 2, 3])], names=["x"])
with pytest.warns(FutureWarning, match="v9.1"):
conn.register(batch, "my_table")
assert conn.table("my_table").x.sum().execute() == 6
assert conn.table("my_table").x.sum().execute() == 6


def test_register_dataset(conn):
import pyarrow.dataset as ds

tab = pa.table({"x": [1, 2, 3]})
dataset = ds.InMemoryDataset(tab)
with pytest.warns(FutureWarning, match="v9.1"):
conn.register(dataset, "my_table")
assert conn.table("my_table").x.sum().execute() == 6
assert conn.table("my_table").x.sum().execute() == 6


def test_create_table_with_uppercase_name(conn):
Expand Down
8 changes: 3 additions & 5 deletions ibis/examples/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,9 @@
if pyarrow.types.is_null(field.type):
table = table.set_column(i, field.name, table[i].cast("string"))

# TODO: It should be possible to avoid this memtable call, once all
# backends support passing a `pyarrow.Table` to `create_table`
# directly.
obj = ibis.memtable(table)
return backend.create_table(table_name, obj, temp=True, overwrite=True)
obj = ibis.memtable(table, name=table_name)
backend._register_in_memory_tables(obj)
return backend.table(table_name).cache()

Check warning on line 83 in ibis/examples/__init__.py

View check run for this annotation

Codecov / codecov/patch

ibis/examples/__init__.py#L83

Added line #L83 was not covered by tests
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we're still in the same place here (but more roundabout), since datafusion doesn't implement cache?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LOL, what a mess!



_FETCH_DOCSTRING_TEMPLATE = """\
Expand Down