From 3faf885644e01b01cffd945a63e079d6fd446ffa Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 12 Jan 2021 04:17:45 -0800 Subject: [PATCH] bump botocore (#844) --- .gitignore | 4 +++- CHANGES.rst | 5 +++++ Makefile | 8 ++++++-- aiobotocore/__init__.py | 2 +- aiobotocore/client.py | 3 +++ aiobotocore/credentials.py | 14 +++++++++++--- aiobotocore/endpoint.py | 2 +- aiobotocore/response.py | 6 +++--- aiobotocore/signers.py | 8 +++++--- aiobotocore/utils.py | 11 ++++++++++- aiobotocore/waiter.py | 24 +++++++++++++++++++----- setup.py | 6 +++--- tests/botocore/test_credentials.py | 5 +++-- tests/test_patches.py | 16 ++++++++-------- 14 files changed, 81 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index a651b82e..faaa4ac4 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,7 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ -.coverage +.coverage* .cache nosetests.xml coverage.xml @@ -61,3 +61,5 @@ target/ *.iml # rope .ropeproject + +Pipfile.lock \ No newline at end of file diff --git a/CHANGES.rst b/CHANGES.rst index 61e319c9..7483a980 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,10 @@ Changes ------- +1.2.0 (2021-01-11) +^^^^^^^^^^^^^^^^^^ +* bump botocore to 1.19.52 +* use passed in http_session_cls param to create_client (#797) + 1.1.2 (2020-10-07) ^^^^^^^^^^^^^^^^^^ * fix AioPageIterator search method #831 (thanks @joseph-jones) diff --git a/Makefile b/Makefile index 59268087..3a7ca81f 100644 --- a/Makefile +++ b/Makefile @@ -22,12 +22,14 @@ cov cover coverage: flake mototest: docker pull alpine docker pull lambci/lambda:python3.8 - BOTO_CONFIG=/dev/null pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov --log-cli-level=DEBUG aiobotocore tests + BOTO_CONFIG=/dev/null pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov=aiobotocore --cov=tests --log-cli-level=DEBUG aiobotocore tests @echo "open file://`pwd`/htmlcov/index.html" clean: rm -rf `find . -name __pycache__` + rm -rf `find . -name .pytest_cache` + rm -rf `find . -name *.egg-info` rm -f `find . -type f -name '*.py[co]' ` rm -f `find . -type f -name '*~' ` rm -f `find . -type f -name '.*~' ` @@ -35,8 +37,10 @@ clean: rm -f `find . -type f -name '#*#' ` rm -f `find . -type f -name '*.orig' ` rm -f `find . -type f -name '*.rej' ` - rm -f .coverage + rm -f .coverage* rm -rf coverage + rm -rf coverage.xml + rm -rf htmlcov rm -rf build rm -rf cover rm -rf dist diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index c38e82b8..6e766474 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1,4 +1,4 @@ from .session import get_session, AioSession __all__ = ['get_session', 'AioSession'] -__version__ = '1.1.2' +__version__ = '1.2.0' diff --git a/aiobotocore/client.py b/aiobotocore/client.py index 75f8ebdd..c7ecf21c 100644 --- a/aiobotocore/client.py +++ b/aiobotocore/client.py @@ -34,6 +34,9 @@ async def create_client(self, service_name, region_name, is_secure=True, verify, credentials, scoped_config, client_config, endpoint_bridge) service_client = cls(**client_args) self._register_retries(service_client) + self._register_s3_events( + service_client, endpoint_bridge, endpoint_url, client_config, + scoped_config) self._register_s3_events( service_client, endpoint_bridge, endpoint_url, client_config, scoped_config) diff --git a/aiobotocore/credentials.py b/aiobotocore/credentials.py index db47af02..e043c1ed 100644 --- a/aiobotocore/credentials.py +++ b/aiobotocore/credentials.py @@ -46,6 +46,12 @@ def create_credential_resolver(session, cache=None, region_name=None): num_attempts = session.get_config_variable('metadata_service_num_attempts') disable_env_vars = session.instance_variables().get('profile') is not None + imds_config = { + 'ec2_metadata_service_endpoint': session.get_config_variable( + 'ec2_metadata_service_endpoint'), + 'imds_use_ipv6': session.get_config_variable('imds_use_ipv6') + } + if cache is None: cache = {} @@ -55,7 +61,8 @@ def create_credential_resolver(session, cache=None, region_name=None): iam_role_fetcher=AioInstanceMetadataFetcher( timeout=metadata_timeout, num_attempts=num_attempts, - user_agent=session.user_agent()) + user_agent=session.user_agent(), + config=imds_config) ) profile_provider_builder = AioProfileProviderBuilder( @@ -817,6 +824,8 @@ async def load_credentials(self): class AioSSOCredentialFetcher(AioCachedCredentialFetcher): + _UTC_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + def __init__(self, start_url, sso_region, role_name, account_id, client_creator, token_loader=None, cache=None, expiry_window_seconds=None): @@ -826,7 +835,6 @@ def __init__(self, start_url, sso_region, role_name, account_id, self._account_id = account_id self._start_url = start_url self._token_loader = token_loader - super(AioSSOCredentialFetcher, self).__init__( cache, expiry_window_seconds ) @@ -846,7 +854,7 @@ def _parse_timestamp(self, timestamp_ms): # fromtimestamp expects seconds so: milliseconds / 1000 = seconds timestamp_seconds = timestamp_ms / 1000.0 timestamp = datetime.datetime.fromtimestamp(timestamp_seconds, tzutc()) - return _serialize_if_needed(timestamp) + return timestamp.strftime(self._UTC_DATE_FORMAT) async def _get_credentials(self): """Get credentials by calling SSO get role credentials.""" diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index ca312ec0..41f5b149 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -304,7 +304,7 @@ def create_endpoint(self, service_model, region_name, endpoint_url, ssl_context=ssl_context, **connector_args) - aio_session = aiohttp.ClientSession( + aio_session = http_session_cls( connector=connector, timeout=timeout, skip_auto_headers={'CONTENT-TYPE'}, diff --git a/aiobotocore/response.py b/aiobotocore/response.py index 83fbb28e..c901d980 100644 --- a/aiobotocore/response.py +++ b/aiobotocore/response.py @@ -75,7 +75,7 @@ async def __anext__(self): anext = __anext__ - async def iter_lines(self, chunk_size=1024): + async def iter_lines(self, chunk_size=1024, keepends=False): """Return an iterator to yield lines from the raw stream. This is achieved by reading chunk of bytes (of size chunk_size) at a @@ -85,10 +85,10 @@ async def iter_lines(self, chunk_size=1024): async for chunk in self.iter_chunks(chunk_size): lines = (pending + chunk).splitlines(True) for line in lines[:-1]: - yield line.splitlines()[0] + yield line.splitlines(keepends)[0] pending = lines[-1] if pending: - yield pending.splitlines()[0] + yield pending.splitlines(keepends)[0] async def iter_chunks(self, chunk_size=_DEFAULT_CHUNK_SIZE): """Return an iterator to yield chunks of chunk_size bytes from the raw diff --git a/aiobotocore/signers.py b/aiobotocore/signers.py index 99e97c8c..da2784e3 100644 --- a/aiobotocore/signers.py +++ b/aiobotocore/signers.py @@ -46,9 +46,11 @@ async def sign(self, operation_name, request, region_name=None, } if expires_in is not None: kwargs['expires'] = expires_in - if not explicit_region_name and request.context.get( - 'signing', {}).get('region'): - kwargs['region_name'] = request.context['signing']['region'] + signing_context = request.context.get('signing', {}) + if not explicit_region_name and signing_context.get('region'): + kwargs['region_name'] = signing_context['region'] + if signing_context.get('signing_name'): + kwargs['signing_name'] = signing_context['signing_name'] try: auth = await self.get_auth_instance(**kwargs) except UnknownSignatureVersionError as e: diff --git a/aiobotocore/utils.py b/aiobotocore/utils.py index 2d13c74e..8f801254 100644 --- a/aiobotocore/utils.py +++ b/aiobotocore/utils.py @@ -7,7 +7,9 @@ from botocore.utils import ContainerMetadataFetcher, InstanceMetadataFetcher, \ IMDSFetcher, get_environ_proxies, BadIMDSRequestError, S3RegionRedirector, \ ClientError -from botocore.exceptions import MetadataRetrievalError +from botocore.exceptions import ( + InvalidIMDSEndpointError, MetadataRetrievalError, +) import botocore.awsrequest @@ -58,6 +60,13 @@ async def _fetch_metadata_token(self): logger.debug( "Caught retryable HTTP exception while making metadata " "service request to %s: %s", url, e, exc_info=True) + except aiohttp.client_exceptions.ClientConnectorError as e: + if getattr(e, 'errno', None) == 8 or \ + str(getattr(e, 'os_error', None)) == \ + 'Domain name not found': # threaded vs async resolver + raise InvalidIMDSEndpointError(endpoint=url, error=e) + else: + raise return None diff --git a/aiobotocore/waiter.py b/aiobotocore/waiter.py index 9f2b55a5..fa0d0cee 100644 --- a/aiobotocore/waiter.py +++ b/aiobotocore/waiter.py @@ -25,6 +25,7 @@ async def wait(self, **kwargs): config = kwargs.pop('WaiterConfig', {}) sleep_amount = config.get('Delay', self.config.delay) max_attempts = config.get('MaxAttempts', self.config.max_attempts) + last_matched_acceptor = None num_attempts = 0 while True: @@ -32,6 +33,7 @@ async def wait(self, **kwargs): num_attempts += 1 for acceptor in acceptors: if acceptor.matcher_func(response): + last_matched_acceptor = acceptor current_state = acceptor.state break else: @@ -43,24 +45,36 @@ async def wait(self, **kwargs): # can just handle here by raising an exception. raise WaiterError( name=self.name, - reason=response['Error'].get('Message', 'Unknown'), - last_response=response + reason='An error occurred (%s): %s' % ( + response['Error'].get('Code', 'Unknown'), + response['Error'].get('Message', 'Unknown'), + ), + last_response=response, ) if current_state == 'success': logger.debug("Waiting complete, waiter matched the " "success state.") return if current_state == 'failure': + reason = 'Waiter encountered a terminal failure state: %s' % ( + acceptor.explanation + ) raise WaiterError( name=self.name, - reason='Waiter encountered a terminal failure state', + reason=reason, last_response=response, ) if num_attempts >= max_attempts: + if last_matched_acceptor is None: + reason = 'Max attempts exceeded' + else: + reason = 'Max attempts exceeded. Previously accepted state: %s' % ( + acceptor.explanation + ) raise WaiterError( name=self.name, - reason='Max attempts exceeded', - last_response=response + reason=reason, + last_response=response, ) await asyncio.sleep(sleep_amount) diff --git a/setup.py b/setup.py index f4b49300..823b6290 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ # NOTE: When updating botocore make sure to update awscli/boto3 versions below install_requires = [ # pegged to also match items in `extras_require` - 'botocore>=1.17.44,<1.17.45', + 'botocore>=1.19.52,<1.19.53', 'aiohttp>=3.3.1', 'wrapt>=1.10.10', 'aioitertools>=0.5.1', @@ -19,8 +19,8 @@ def read(f): extras_require = { - 'awscli': ['awscli==1.18.121'], - 'boto3': ['boto3==1.14.44'], + 'awscli': ['awscli==1.18.212'], + 'boto3': ['boto3==1.16.52'], } diff --git a/tests/botocore/test_credentials.py b/tests/botocore/test_credentials.py index 27caf81c..eec3deff 100644 --- a/tests/botocore/test_credentials.py +++ b/tests/botocore/test_credentials.py @@ -1084,6 +1084,7 @@ def _f(config_loader: Optional[ConfigValueStore] = None) -> AioSession: 'config_file': 'c', 'metadata_service_timeout': 1, 'metadata_service_num_attempts': 1, + 'imds_use_ipv6': 'false', } def fake_get_component(self, key): @@ -1228,7 +1229,7 @@ async def test_sso_credential_fetcher_can_fetch_credentials( self.assertEqual(credentials['access_key'], 'foo') self.assertEqual(credentials['secret_key'], 'bar') self.assertEqual(credentials['token'], 'baz') - self.assertEqual(credentials['expiry_time'], '2008-09-23T12:43:20UTC') + self.assertEqual(credentials['expiry_time'], '2008-09-23T12:43:20Z') cache_key = '048db75bbe50955c16af7aba6ff9c41a3131bb7e' expected_cached_credentials = { 'ProviderType': 'sso', @@ -1236,7 +1237,7 @@ async def test_sso_credential_fetcher_can_fetch_credentials( 'AccessKeyId': 'foo', 'SecretAccessKey': 'bar', 'SessionToken': 'baz', - 'Expiration': '2008-09-23T12:43:20UTC', + 'Expiration': '2008-09-23T12:43:20Z', } } self.assertEqual(self.cache[cache_key], expected_cached_credentials) diff --git a/tests/test_patches.py b/tests/test_patches.py index 7fb02b57..7f16413d 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -76,7 +76,7 @@ ClientArgsCreator.get_client_args: {'e3a44e6f50159e8e31c3d76f5e8a1110dda495fa'}, # client.py - ClientCreator.create_client: {'ee63a3d60b5917879cb644c1b0aa3fe34538b915'}, + ClientCreator.create_client: {'281fbf7afc4e6282e5c881c7a03717c9e5e4e176'}, ClientCreator._create_client_class: {'5e493d069eedbf314e40e12a7886bbdbcf194335'}, ClientCreator._get_client_args: {'555e1e41f93df7558c8305a60466681e3a267ef3'}, ClientCreator._register_s3_events: {'da3fc62a131d63964c8daa0f52124b092fd8f1b4'}, @@ -116,7 +116,7 @@ RefreshableCredentials.get_frozen_credentials: {'f661c84a8b759786e011f0b1e8a468a0c6294e36'}, SSOCredentialFetcher: - {'e092b115155a06760af6f3c72ccef120979b1201'}, + {'d68353a0d2c291d5742f134d28ae1e1419faa4c6'}, SSOProvider.load: {'f43d79e1520b2a7b7ef85cd537f41e19d4bce806'}, CachedCredentialFetcher._get_credentials: @@ -185,7 +185,7 @@ {'c028b9776383cc566be10999745b6082f458d902'}, BotoProvider.load: {'9351b8565c2c969937963fc1d3fbc8b3b6d8ccc1'}, OriginalEC2Provider.load: {'bde9af019f01acf3848a6eda125338b2c588c1ab'}, - create_credential_resolver: {'5ff7fe49d7636b795a50202ff5c089611f4e27c1'}, + create_credential_resolver: {'c6a08bfc59a4e8f59c8b7a846dbb74db649101fd'}, get_credentials: {'ff0c735a388ac8dd7fe300a32c1e36cdf33c0f56'}, # endpoint.py @@ -226,7 +226,7 @@ RestJSONParser._create_event_stream: {'0564ba55383a71cc1ba3e5be7110549d7e9992f5'}, # response.py - StreamingBody: {'bb4d872649b0c118c9a3d5e44961e1bea92eb79c'}, + StreamingBody: {'b77bd0903f9013bc47c01f91c6d9bfb8a504d106'}, # session.py Session.__init__: {'ccf156a76beda3425fb54363f3b2718dc0445f6d'}, @@ -242,7 +242,7 @@ # signers.py RequestSigner.handler: {'371909df136a0964ef7469a63d25149176c2b442'}, - RequestSigner.sign: {'7df841d3df3f4015763523c1932652aef754287a'}, + RequestSigner.sign: {'a07e4caab222bf9375036b1fafaf021ccb5b2bf3'}, RequestSigner.get_auth: {'4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8'}, RequestSigner.get_auth_instance: {'4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8'}, RequestSigner._choose_signer: {'d1e0e3196ada449d3ae0ec09b8ae9b5868c50d4e'}, @@ -268,9 +268,9 @@ ContainerMetadataFetcher._get_response: {'7e5acdd2cf0167a047e3d5ee1439565a2f79f6a6'}, # Overrided session and dealing with proxy support - IMDSFetcher.__init__: {'690e37140ccdcd67c7a85ce5d36331491a79954e'}, + IMDSFetcher.__init__: {'a0766a5ba7dde9c26f3c51eb38d73f8e6087d492'}, IMDSFetcher._get_request: {'96a0e580cab5a21deb4d2cd7e904aa17d5e1e504'}, - IMDSFetcher._fetch_metadata_token: {'4fdad673b4997b1268c6d9dff09a4b99c1cb5e0d'}, + IMDSFetcher._fetch_metadata_token: {'dcb147f6c7a425ba9e30be8ad4818be4b781305c'}, InstanceMetadataFetcher.retrieve_iam_role_credentials: {'76737f6add82a1b9a0dc590cf10bfac0c7026a2e'}, @@ -285,7 +285,7 @@ # waiter.py NormalizedOperationMethod.__call__: {'79723632d023739aa19c8a899bc2b814b8ab12ff'}, - Waiter.wait: {'5502a89ed740fb5d6238a6f72a3a08efc1a9f43b'}, + Waiter.wait: {'3a4ff0fdfc78b7ec42bfd41f3e1ba3b741f2d2b9'}, create_waiter_with_client: {'c3d12c9a4293105cc8c2ecfc7e69a2152ad564de'}, }