Skip to content

Commit

Permalink
Addressing comments
Browse files Browse the repository at this point in the history
  • Loading branch information
mrchtr committed Mar 28, 2024
1 parent 905dbd8 commit 730231f
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 9 deletions.
6 changes: 0 additions & 6 deletions src/fondant/component/data_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,12 +200,6 @@ def _write_dataframe(self, dataframe: dd.DataFrame) -> None:
columns=dataframe.columns,
)

if len(set(location)) > 1:
msg = "Writing to multiple locations is currently not supported."
raise ValueError(
msg,
)

output_location_path = location[0]

# Create directory the dataframe will be written to, since this is not handled by Pandas
Expand Down
5 changes: 2 additions & 3 deletions src/fondant/core/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,16 +299,15 @@ def evolve_manifest_index_and_field_locations( # noqa PLR0913
):
"""Evolve the manifest index and field locations based on the component spec."""
# Update index location as this is always rewritten
# TODO: handle index location - do we have to update/rewrite the index to the new location?
if working_dir:
field = Field.create(
index = Field.create(
name="index",
working_dir=working_dir,
run_id=run_id,
component_id=component_id,
dataset_name=self.dataset_name,
)
evolved_manifest.add_or_update_field(field, overwrite=False)
evolved_manifest.add_or_update_field(index, overwrite=False)

# Remove all previous fields if the component changes the index
if operation_spec.previous_index:
Expand Down

0 comments on commit 730231f

Please sign in to comment.