Skip to content

Commit

Permalink
chore(pyspark): factor out some repeatedly used properties
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud committed Sep 16, 2024
1 parent 9da309c commit 0ede913
Showing 1 changed file with 9 additions and 4 deletions.
13 changes: 9 additions & 4 deletions ibis/backends/pyspark/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,11 +413,16 @@ def _register_udfs(self, expr: ir.Expr) -> None:
self._session.udf.register("unwrap_json_float", unwrap_json_float)

def _register_in_memory_table(self, op: ops.InMemoryTable) -> None:
schema = PySparkSchema.from_ibis(op.schema)
df = self._session.createDataFrame(data=op.data.to_frame(), schema=schema)
session = self._session
df = session.createDataFrame(
data=op.data.to_frame(), schema=PySparkSchema.from_ibis(op.schema)
)
name = op.name

with contextlib.suppress(AnalysisException):
self._session.catalog.dropTempView(op.name)
df.createTempView(op.name)
session.catalog.dropTempView(name)

df.createTempView(name)

def _finalize_memtable(self, name: str) -> None:
self._session.catalog.dropTempView(name)
Expand Down

0 comments on commit 0ede913

Please sign in to comment.