Skip to content

Commit

Permalink
dask fixes
Browse files Browse the repository at this point in the history
Signed-off-by: Anatoly Myachev <[email protected]>
  • Loading branch information
anmyachev committed Oct 23, 2023
1 parent 8c9d0de commit 582355a
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,24 @@ class PandasOnDaskDataframe(PandasDataframe):
"""

_partition_mgr_cls = PandasOnDaskDataframePartitionManager

@classmethod
def reconnect(cls, address, attributes):
try:
from distributed import default_client

default_client()
except ValueError:
from distributed import Client

# setup `default_client` for worker process
_ = Client(address)
obj = cls.__new__(cls)
obj.__dict__.update(attributes)
return obj

def __reduce__(self):
from distributed import default_client

address = default_client().scheduler_info()["address"]
return self.reconnect, (address, self.__dict__)
2 changes: 1 addition & 1 deletion modin/pandas/test/test_series.py
Original file line number Diff line number Diff line change
Expand Up @@ -4796,7 +4796,7 @@ def test_binary_numpy_universal_function_issue_6483():
)


@pytest.mark.xfail(Engine.get() != "Ray", reason="reason")
@pytest.mark.skipif(Engine.get() == "Unidist", reason="Unidist doesn't work for now.")
def test__reduce__():
abbreviations = pd.Series(
["Major League Baseball", "National Basketball Association"],
Expand Down

0 comments on commit 582355a

Please sign in to comment.