Skip to content

Commit

Permalink
default downloader
Browse files Browse the repository at this point in the history
  • Loading branch information
ethantang-db committed Oct 29, 2024
1 parent f869b74 commit b004ab3
Showing 1 changed file with 2 additions and 5 deletions.
7 changes: 2 additions & 5 deletions streaming/base/storage/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ def get(cls, remote_dir: Optional[str]) -> 'CloudDownloader':
ValueError: If the remote path is not supported.
"""
if remote_dir is None:
# Return the default downloader, assuming that it exists
return DOWNLOADER_MAPPINGS['']()
return _DEFAULT_DOWNLOADER()

prefix = urllib.parse.urlparse(remote_dir).scheme
if prefix == 'dbfs' and remote_dir.startswith('dbfs:/Volumes'):
Expand Down Expand Up @@ -843,10 +842,8 @@ def _register_cloud_downloader_subclasses() -> dict[str, type[CloudDownloader]]:
for sub_class in sub_classes:
downloader_mappings[sub_class.prefix()] = sub_class

if '' not in downloader_mappings:
raise ValueError('A default downloader with the prefix of an empty string is required.')

return downloader_mappings


DOWNLOADER_MAPPINGS = _register_cloud_downloader_subclasses()
_DEFAULT_DOWNLOADER = LocalDownloader

0 comments on commit b004ab3

Please sign in to comment.