From e63993b231c7f1498260fffb560aa9d6f390f998 Mon Sep 17 00:00:00 2001 From: Dave Johansen Date: Thu, 17 Oct 2024 14:50:02 -0600 Subject: [PATCH] Skip HEAD for multipart check when multipart_threshold is None --- s3transfer/__init__.py | 3 ++- s3transfer/copies.py | 2 +- s3transfer/download.py | 4 ++-- s3transfer/processpool.py | 2 +- s3transfer/upload.py | 6 +++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/s3transfer/__init__.py b/s3transfer/__init__.py index 056492cb..d142a7fe 100644 --- a/s3transfer/__init__.py +++ b/s3transfer/__init__.py @@ -747,6 +747,7 @@ def upload_file( unique_id='s3upload-callback-enable', ) if ( + self._config.multipart_threshold is not None and self._osutil.get_file_size(filename) >= self._config.multipart_threshold ): @@ -803,7 +804,7 @@ def download_file( def _download_file( self, bucket, key, filename, object_size, extra_args, callback ): - if object_size >= self._config.multipart_threshold: + if self._config.multipart_threshold is not None and object_size >= self._config.multipart_threshold: self._ranged_download( bucket, key, filename, object_size, extra_args, callback ) diff --git a/s3transfer/copies.py b/s3transfer/copies.py index c2ae9ce0..2d782d70 100644 --- a/s3transfer/copies.py +++ b/s3transfer/copies.py @@ -130,7 +130,7 @@ def _submit( # If it is greater than threshold do a multipart copy, otherwise # do a regular copy object. - if transfer_future.meta.size < config.multipart_threshold: + if config.multipart_threshold is None or transfer_future.meta.size < config.multipart_threshold: self._submit_copy_request( client, config, osutil, request_executor, transfer_future ) diff --git a/s3transfer/download.py b/s3transfer/download.py index e1a9cf0e..ba8b8649 100644 --- a/s3transfer/download.py +++ b/s3transfer/download.py @@ -346,7 +346,7 @@ def _submit( :param bandwidth_limiter: The bandwidth limiter to use when downloading streams """ - if transfer_future.meta.size is None: + if transfer_future.meta.size is None and config.multipart_threshold is not None: # If a size was not provided figure out the size for the # user. response = client.head_object( @@ -364,7 +364,7 @@ def _submit( # If it is greater than threshold do a ranged download, otherwise # do a regular GetObject download. - if transfer_future.meta.size < config.multipart_threshold: + if config.multipart_threshold is None or transfer_future.meta.size < config.multipart_threshold: self._submit_download_request( client, config, diff --git a/s3transfer/processpool.py b/s3transfer/processpool.py index 318f1e7f..5cf05346 100644 --- a/s3transfer/processpool.py +++ b/s3transfer/processpool.py @@ -816,7 +816,7 @@ def _do_run(self): def _submit_get_object_jobs(self, download_file_request): size = self._get_size(download_file_request) temp_filename = self._allocate_temp_file(download_file_request, size) - if size < self._transfer_config.multipart_threshold: + if self._transfer_config.multiparth_threshold is None or size < self._transfer_config.multipart_threshold: self._submit_single_get_object_job( download_file_request, temp_filename ) diff --git a/s3transfer/upload.py b/s3transfer/upload.py index 0347857b..17f2206e 100644 --- a/s3transfer/upload.py +++ b/s3transfer/upload.py @@ -245,7 +245,7 @@ def provide_transfer_size(self, transfer_future): ) def requires_multipart_upload(self, transfer_future, config): - return transfer_future.meta.size >= config.multipart_threshold + return config.multipart_threshold is not None and transfer_future.meta.size >= config.multipart_threshold def get_put_object_body(self, transfer_future): # Get a file-like object for the given input @@ -393,14 +393,14 @@ def provide_transfer_size(self, transfer_future): def requires_multipart_upload(self, transfer_future, config): # If the user has set the size, we can use that. if transfer_future.meta.size is not None: - return transfer_future.meta.size >= config.multipart_threshold + return config.multipart_threshold is not None and transfer_future.meta.size >= config.multipart_threshold # This is tricky to determine in this case because we can't know how # large the input is. So to figure it out, we read data into memory # up until the threshold and compare how much data was actually read # against the threshold. fileobj = transfer_future.meta.call_args.fileobj - threshold = config.multipart_threshold + threshold = config.multipart_threshold or 8 * 1024 * 1024 self._initial_data = self._read(fileobj, threshold, False) if len(self._initial_data) < threshold: return False