Skip to content

Commit

Permalink
updates object id and object type names
Browse files Browse the repository at this point in the history
  • Loading branch information
@BWMac
BWMac committed Sep 3, 2024
1 parent f368f1f commit 53f247f
Show file tree
Hide file tree
Showing 4 changed files with 761 additions and 656 deletions.
30 changes: 15 additions & 15 deletions synapseclient/core/download/download_functions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,13 +404,13 @@ async def download_by_file_handle(

while retries > 0:
try:
file_handle_result: Dict[
str, str
] = await get_file_handle_for_download_async(
file_handle_id=file_handle_id,
synapse_id=synapse_id,
entity_type=entity_type,
synapse_client=syn,
file_handle_result: Dict[str, str] = (
await get_file_handle_for_download_async(
file_handle_id=file_handle_id,
synapse_id=synapse_id,
entity_type=entity_type,
synapse_client=syn,
)
)
file_handle = file_handle_result["fileHandle"]
concrete_type = file_handle["concreteType"]
Expand Down Expand Up @@ -512,8 +512,8 @@ def download_fn(
lambda: download_from_url(
url=file_handle_result["preSignedURL"],
destination=destination,
object_id=synapse_id,
object_type=entity_type,
entity_id=synapse_id,
file_handle_associate_type=entity_type,
file_handle_id=file_handle["id"],
expected_md5=file_handle.get("contentMd5"),
progress_bar=progress_bar,
Expand Down Expand Up @@ -619,8 +619,8 @@ async def download_from_url_multi_threaded(
def download_from_url(
url: str,
destination: str,
object_id: Optional[str],
object_type: Optional[str],
entity_id: Optional[str],
file_handle_associate_type: Optional[str],
file_handle_id: Optional[str] = None,
expected_md5: Optional[str] = None,
progress_bar: Optional[tqdm] = None,
Expand All @@ -633,9 +633,9 @@ def download_from_url(
Arguments:
url: The source of download
destination: The destination on local file system
object_id: The id of the Synapse object that uses the FileHandle
entity_id: The id of the Synapse object that uses the FileHandle
e.g. "syn123"
object_type: The type of the Synapse object that uses the
file_handle_associate_type: The type of the Synapse object that uses the
FileHandle e.g. "FileEntity". Any of
<https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/file/FileHandleAssociateType.html>
file_handle_id: Optional. If given, the file will be given a temporary name that includes the file
Expand Down Expand Up @@ -758,8 +758,8 @@ def _ftp_report_hook(
if url_is_expired:
response = get_file_handle_for_download(
file_handle_id=file_handle_id,
synapse_id=object_id,
entity_type=object_type,
synapse_id=entity_id,
entity_type=file_handle_associate_type,
synapse_client=client,
)
refreshed_url = response["preSignedURL"]
Expand Down
101 changes: 57 additions & 44 deletions tests/integration/synapseclient/core/test_download.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ async def test_download_check_md5(
await download_from_url(
url=entity_bad_md5.external_url,
destination=tempfile.gettempdir(),
object_id=entity_bad_md5.id,
object_type="FileEntity",
entity_id=entity_bad_md5.id,
file_handle_associate_type="FileEntity",
file_handle_id=entity_bad_md5.data_file_handle_id,
expected_md5="2345a",
)
Expand Down Expand Up @@ -279,8 +279,8 @@ async def test_download_from_url_resume_partial_download(
path = download_from_url(
url=f"{syn.repoEndpoint}/entity/{file.id}/file",
destination=tempfile.gettempdir(),
object_id=file.id,
object_type="FileEntity",
entity_id=file.id,
file_handle_associate_type="FileEntity",
file_handle_id=file.data_file_handle_id,
expected_md5=file.file_handle.content_md5,
)
Expand Down Expand Up @@ -318,8 +318,8 @@ async def test_download_expired_url(
path = download_from_url(
url=expired_url,
destination=tempfile.gettempdir(),
object_id=file.id,
object_type="FileEntity",
entity_id=file.id,
file_handle_associate_type="FileEntity",
file_handle_id=file.data_file_handle_id,
expected_md5=file.file_handle.content_md5,
)
Expand Down Expand Up @@ -442,14 +442,17 @@ async def test_download_from_url_multi_threaded(
os.remove(file_path)
assert not os.path.exists(file_path)

with patch.object(
synapseclient.core.download.download_functions,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
), patch.object(
synapseclient.core.download.download_async,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
with (
patch.object(
synapseclient.core.download.download_functions,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
),
patch.object(
synapseclient.core.download.download_async,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
),
):
# WHEN I download the file with multiple parts
file = await File(id=file.id, path=os.path.dirname(file.path)).get_async()
Expand Down Expand Up @@ -511,21 +514,26 @@ def mock_httpx_send(**kwargs):
# Call the real send function
return client.send(**kwargs)

with patch.object(
synapseclient.core.download.download_functions,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
), patch.object(
synapseclient.core.download.download_async,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
), patch.object(
syn._requests_session_storage,
"send",
mock_httpx_send,
), patch(
"synapseclient.core.download.download_async.DEFAULT_MAX_BACK_OFF_ASYNC",
0.2,
with (
patch.object(
synapseclient.core.download.download_functions,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
),
patch.object(
synapseclient.core.download.download_async,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
),
patch.object(
syn._requests_session_storage,
"send",
mock_httpx_send,
),
patch(
"synapseclient.core.download.download_async.DEFAULT_MAX_BACK_OFF_ASYNC",
0.2,
),
):
# WHEN I download the file with multiple parts
file = await File(id=file.id, path=os.path.dirname(file.path)).get_async()
Expand Down Expand Up @@ -585,21 +593,26 @@ def mock_httpx_send(**kwargs):
# Call the real send function
return client.send(**kwargs)

with patch.object(
synapseclient.core.download.download_functions,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
), patch.object(
synapseclient.core.download.download_async,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
), patch.object(
syn._requests_session_storage,
"send",
mock_httpx_send,
), patch(
"synapseclient.core.download.download_async.DEFAULT_MAX_BACK_OFF_ASYNC",
0.2,
with (
patch.object(
synapseclient.core.download.download_functions,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
),
patch.object(
synapseclient.core.download.download_async,
"SYNAPSE_DEFAULT_DOWNLOAD_PART_SIZE",
new=500,
),
patch.object(
syn._requests_session_storage,
"send",
mock_httpx_send,
),
patch(
"synapseclient.core.download.download_async.DEFAULT_MAX_BACK_OFF_ASYNC",
0.2,
),
):
# WHEN I download the file with multiple parts
file = await File(id=file.id, path=os.path.dirname(file.path)).get_async()
Expand Down
Loading

0 comments on commit 53f247f

Please sign in to comment.