diff --git a/CHANGES.rst b/CHANGES.rst index 6f87f190..288b1d08 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,11 @@ Changes ------- +2.2.0 (2022-03-16) +^^^^^^^^^^^^^^^^^^ +* remove deprecated APIs +* bump to botocore 1.24.21 +* re-enable retry of aiohttp.ClientPayloadError + 2.1.2 (2022-03-03) ^^^^^^^^^^^^^^^^^^ * fix httpsession close call diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index 84baea77..04188a16 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1,15 +1 @@ -# NOTE: These imports are deprecated and will be removed in 2.x -import os - -# Enabling this will enable the old http exception behavior that exposed raw aiohttp -# exceptions and old session variables available via __init__. Disabling will swap to -# botocore exceptions and will not have these imports to match botocore. -# NOTE: without setting this to 0, retries may not work, see #876 -DEPRECATED_1_4_0_APIS = int(os.getenv('AIOBOTOCORE_DEPRECATED_1_4_0_APIS', '0')) - -if DEPRECATED_1_4_0_APIS: - from .session import get_session, AioSession - - __all__ = ['get_session', 'AioSession'] - -__version__ = '2.1.2' +__version__ = '2.2.0' diff --git a/aiobotocore/client.py b/aiobotocore/client.py index 9c2e2017..43fd0431 100644 --- a/aiobotocore/client.py +++ b/aiobotocore/client.py @@ -1,6 +1,7 @@ from botocore.awsrequest import prepare_request_dict from botocore.client import logger, PaginatorDocstring, ClientCreator, \ - BaseClient, ClientEndpointBridge, S3ArnParamHandler, S3EndpointSetter + BaseClient, ClientEndpointBridge, S3ArnParamHandler, S3EndpointSetter, \ + resolve_checksum_context, apply_request_checksum from botocore.discovery import block_endpoint_discovery_required_operations from botocore.exceptions import OperationNotPageableError from botocore.history import get_global_history_recorder @@ -196,6 +197,7 @@ async def _make_api_call(self, operation_name, api_params): } request_dict = await self._convert_to_request_dict( api_params, operation_model, context=request_context) + resolve_checksum_context(request_dict, operation_model, api_params) service_id = self._service_model.service_id.hyphenize() handler, event_response = await self.meta.events.emit_until_response( @@ -208,6 +210,7 @@ async def _make_api_call(self, operation_name, api_params): if event_response is not None: http, parsed_response = event_response else: + apply_request_checksum(request_dict) http, parsed_response = await self._make_request( operation_model, request_dict, request_context) diff --git a/aiobotocore/config.py b/aiobotocore/config.py index d5acdf7c..57db443b 100644 --- a/aiobotocore/config.py +++ b/aiobotocore/config.py @@ -5,7 +5,6 @@ class AioConfig(botocore.client.Config): - def __init__(self, connector_args=None, **kwargs): super().__init__(**kwargs) diff --git a/aiobotocore/configprovider.py b/aiobotocore/configprovider.py new file mode 100644 index 00000000..57fba0e2 --- /dev/null +++ b/aiobotocore/configprovider.py @@ -0,0 +1,37 @@ +from botocore.configprovider import os, SmartDefaultsConfigStoreFactory + + +class AioSmartDefaultsConfigStoreFactory(SmartDefaultsConfigStoreFactory): + async def merge_smart_defaults(self, config_store, mode, region_name): + if mode == 'auto': + mode = await self.resolve_auto_mode(region_name) + default_configs = self._default_config_resolver.get_default_config_values( + mode) + for config_var in default_configs: + config_value = default_configs[config_var] + method = getattr(self, f'_set_{config_var}', None) + if method: + method(config_store, config_value) + + async def resolve_auto_mode(self, region_name): + current_region = None + if os.environ.get('AWS_EXECUTION_ENV'): + default_region = os.environ.get('AWS_DEFAULT_REGION') + current_region = os.environ.get('AWS_REGION', default_region) + if not current_region: + if self._instance_metadata_region: + current_region = self._instance_metadata_region + else: + try: + current_region = \ + await self._imds_region_provider.provide() + self._instance_metadata_region = current_region + except Exception: + pass + + if current_region: + if region_name == current_region: + return 'in-region' + else: + return 'cross-region' + return 'standard' diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 6bd14ad8..b5cfc867 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -4,7 +4,7 @@ import aiohttp.http_exceptions from botocore.endpoint import EndpointCreator, Endpoint, DEFAULT_TIMEOUT, \ MAX_POOL_CONNECTIONS, logger, history_recorder, create_request_object, \ - is_valid_ipv6_endpoint_url, is_valid_endpoint_url + is_valid_ipv6_endpoint_url, is_valid_endpoint_url, handle_checksum_body from botocore.exceptions import ConnectionClosedError from botocore.hooks import first_non_none_response from urllib3.response import HTTPHeaderDict @@ -74,6 +74,8 @@ async def create_request(self, params, operation_model=None): async def _send_request(self, request_dict, operation_model): attempts = 1 + context = request_dict['context'] + self._update_retries_context(context, attempts) request = await self.create_request(request_dict, operation_model) context = request_dict['context'] success_response, exception = await self._get_response( @@ -82,6 +84,9 @@ async def _send_request(self, request_dict, operation_model): request_dict, success_response, exception): attempts += 1 + self._update_retries_context( + context, attempts, success_response + ) # If there is a stream associated with the request, we need # to reset it before attempting to send the request again. # This will ensure that we resend the entire contents of the @@ -109,7 +114,7 @@ async def _get_response(self, request, operation_model, context): # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. success_response, exception = await self._do_get_response( - request, operation_model) + request, operation_model, context) kwargs_to_emit = { 'response_dict': None, 'parsed_response': None, @@ -127,7 +132,7 @@ async def _get_response(self, request, operation_model, context): service_id, operation_model.name), **kwargs_to_emit) return success_response, exception - async def _do_get_response(self, request, operation_model): + async def _do_get_response(self, request, operation_model, context): try: logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { @@ -160,6 +165,9 @@ async def _do_get_response(self, request, operation_model): # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) + handle_checksum_body( + http_response, response_dict, context, operation_model, + ) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ diff --git a/aiobotocore/httpsession.py b/aiobotocore/httpsession.py index 626f676e..128c44a8 100644 --- a/aiobotocore/httpsession.py +++ b/aiobotocore/httpsession.py @@ -14,9 +14,8 @@ MAX_POOL_CONNECTIONS, InvalidProxiesConfigError, SSLError, \ EndpointConnectionError, ProxyConnectionError, ConnectTimeoutError, \ ConnectionClosedError, HTTPClientError, ReadTimeoutError, logger, get_cert_path, \ - ensure_boolean, urlparse + ensure_boolean, urlparse, mask_proxy_url -from aiobotocore import DEPRECATED_1_4_0_APIS from aiobotocore._endpoint_helpers import _text, _IOBaseWrapper, \ ClientResponseProxy @@ -182,43 +181,22 @@ async def send(self, request): return resp except ClientSSLError as e: - if DEPRECATED_1_4_0_APIS: - raise - raise SSLError(endpoint_url=request.url, error=e) except (ClientConnectorError, socket.gaierror) as e: - if DEPRECATED_1_4_0_APIS: - raise - raise EndpointConnectionError(endpoint_url=request.url, error=e) except (ClientProxyConnectionError, ClientHttpProxyError) as e: - if DEPRECATED_1_4_0_APIS: - raise - - raise ProxyConnectionError(proxy_url=proxy_url, error=e) + raise ProxyConnectionError(proxy_url=mask_proxy_url(proxy_url), error=e) except ServerTimeoutError as e: - if DEPRECATED_1_4_0_APIS: - raise - raise ConnectTimeoutError(endpoint_url=request.url, error=e) except asyncio.TimeoutError as e: - if DEPRECATED_1_4_0_APIS: - raise - raise ReadTimeoutError(endpoint_url=request.url, error=e) - except ServerDisconnectedError as e: - if DEPRECATED_1_4_0_APIS: - raise - + except (ServerDisconnectedError, aiohttp.ClientPayloadError) as e: raise ConnectionClosedError( error=e, request=request, endpoint_url=request.url ) except Exception as e: - if DEPRECATED_1_4_0_APIS: - raise - message = 'Exception received when sending urllib3 HTTP request' logger.debug(message, exc_info=True) raise HTTPClientError(error=e) diff --git a/aiobotocore/parsers.py b/aiobotocore/parsers.py index 9f4933f6..dd32f2e8 100644 --- a/aiobotocore/parsers.py +++ b/aiobotocore/parsers.py @@ -86,6 +86,7 @@ async def parse(self, response, shape): headers = response['headers'] response_metadata['HTTPHeaders'] = lowercase_dict(headers) parsed['ResponseMetadata'] = response_metadata + self._add_checksum_response_metadata(response, response_metadata) return parsed diff --git a/aiobotocore/response.py b/aiobotocore/response.py index 6a50a8c9..6f080b6c 100644 --- a/aiobotocore/response.py +++ b/aiobotocore/response.py @@ -1,5 +1,6 @@ import asyncio +import aiohttp import aiohttp.client_exceptions import wrapt from botocore.response import ResponseStreamingError, IncompleteReadError, \ @@ -25,7 +26,7 @@ class StreamingBody(wrapt.ObjectProxy): _DEFAULT_CHUNK_SIZE = 1024 - def __init__(self, raw_stream, content_length): + def __init__(self, raw_stream: aiohttp.StreamReader, content_length: str): super().__init__(raw_stream) self._self_content_length = content_length self._self_amount_read = 0 @@ -39,9 +40,8 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): # NOTE: set_socket_timeout was only for when requests didn't support # read timeouts, so not needed - - def tell(self): - return self._self_amount_read + def readable(self): + return not self.at_eof() async def read(self, amt=None): """Read at most amt bytes from the stream. @@ -65,6 +65,11 @@ async def read(self, amt=None): self._verify_content_length() return chunk + async def readlines(self): + # assuming this is not an iterator + lines = [line async for line in self.iter_lines()] + return lines + def __aiter__(self): """Return an iterator to yield 1k chunks from the raw stream. """ @@ -80,7 +85,7 @@ async def __anext__(self): anext = __anext__ - async def iter_lines(self, chunk_size=1024, keepends=False): + async def iter_lines(self, chunk_size=_DEFAULT_CHUNK_SIZE, keepends=False): """Return an iterator to yield lines from the raw stream. This is achieved by reading chunk of bytes (of size chunk_size) at a @@ -115,6 +120,9 @@ def _verify_content_length(self): actual_bytes=self._self_amount_read, expected_bytes=int(self._self_content_length)) + def tell(self): + return self._self_amount_read + async def get_response(operation_model, http_response): protocol = operation_model.metadata['protocol'] diff --git a/aiobotocore/session.py b/aiobotocore/session.py index ec317e7e..938b3218 100644 --- a/aiobotocore/session.py +++ b/aiobotocore/session.py @@ -1,4 +1,5 @@ -from botocore.session import Session, EVENT_ALIASES, ServiceModel, UnknownServiceError +from botocore.session import Session, EVENT_ALIASES, ServiceModel, \ + UnknownServiceError, copy from botocore import UNSIGNED from botocore import retryhandler, translate @@ -16,7 +17,9 @@ add_generate_presigned_url as boto_add_generate_presigned_url, \ add_generate_presigned_post as boto_add_generate_presigned_post, \ add_generate_db_auth_token as boto_add_generate_db_auth_token +from .configprovider import AioSmartDefaultsConfigStoreFactory from .credentials import create_credential_resolver, AioCredentials +from .utils import AioIMDSRegionProvider _HANDLER_MAPPING = { @@ -61,6 +64,16 @@ def _register_response_parser_factory(self): self._components.register_component('response_parser_factory', AioResponseParserFactory()) + def _register_smart_defaults_factory(self): + def create_smart_defaults_factory(): + default_config_resolver = self._get_internal_component( + 'default_config_resolver') + imds_region_provider = AioIMDSRegionProvider(session=self) + return AioSmartDefaultsConfigStoreFactory( + default_config_resolver, imds_region_provider) + self._internal_components.lazy_register_component( + 'smart_defaults_factory', create_smart_defaults_factory) + def create_client(self, *args, **kwargs): return ClientCreatorContext(self._create_client(*args, **kwargs)) @@ -114,6 +127,13 @@ async def _create_client(self, service_name, region_name=None, endpoint_resolver = self._get_internal_component('endpoint_resolver') exceptions_factory = self._get_internal_component('exceptions_factory') config_store = self.get_component('config_store') + defaults_mode = self._resolve_defaults_mode(config, config_store) + if defaults_mode != 'legacy': + smart_defaults_factory = self._get_internal_component( + 'smart_defaults_factory') + config_store = copy.deepcopy(config_store) + await smart_defaults_factory.merge_smart_defaults( + config_store, defaults_mode, region_name) client_creator = AioClientCreator( loader, endpoint_resolver, self.user_agent(), event_emitter, retryhandler, translate, response_parser_factory, diff --git a/aiobotocore/utils.py b/aiobotocore/utils.py index 8f801254..26810e74 100644 --- a/aiobotocore/utils.py +++ b/aiobotocore/utils.py @@ -6,7 +6,8 @@ import aiohttp.client_exceptions from botocore.utils import ContainerMetadataFetcher, InstanceMetadataFetcher, \ IMDSFetcher, get_environ_proxies, BadIMDSRequestError, S3RegionRedirector, \ - ClientError + ClientError, InstanceMetadataRegionFetcher, IMDSRegionProvider, \ + resolve_imds_endpoint_mode from botocore.exceptions import ( InvalidIMDSEndpointError, MetadataRetrievalError, ) @@ -32,7 +33,7 @@ def __init__(self, *args, session=None, **kwargs): async def _fetch_metadata_token(self): self._assert_enabled() - url = self._base_url + self._TOKEN_PATH + url = self._construct_url(self._TOKEN_PATH) headers = { 'x-aws-ec2-metadata-token-ttl-seconds': self._TOKEN_TTL, } @@ -74,7 +75,7 @@ async def _get_request(self, url_path, retry_func, token=None): self._assert_enabled() if retry_func is None: retry_func = self._default_retry - url = self._base_url + url_path + url = self._construct_url(url_path) headers = {} if token is not None: headers['x-aws-ec2-metadata-token'] = token @@ -142,6 +143,62 @@ async def _get_credentials(self, role_name, token=None): return json.loads(r.text) +class AioIMDSRegionProvider(IMDSRegionProvider): + async def provide(self): + """Provide the region value from IMDS.""" + instance_region = await self._get_instance_metadata_region() + return instance_region + + async def _get_instance_metadata_region(self): + fetcher = self._get_fetcher() + region = await fetcher.retrieve_region() + return region + + def _create_fetcher(self): + metadata_timeout = self._session.get_config_variable( + 'metadata_service_timeout') + metadata_num_attempts = self._session.get_config_variable( + 'metadata_service_num_attempts') + imds_config = { + 'ec2_metadata_service_endpoint': self._session.get_config_variable( + 'ec2_metadata_service_endpoint'), + 'ec2_metadata_service_endpoint_mode': resolve_imds_endpoint_mode( + self._session + ) + } + fetcher = AioInstanceMetadataRegionFetcher( + timeout=metadata_timeout, + num_attempts=metadata_num_attempts, + env=self._environ, + user_agent=self._session.user_agent(), + config=imds_config, + ) + return fetcher + + +class AioInstanceMetadataRegionFetcher(AioIMDSFetcher, InstanceMetadataRegionFetcher): + async def retrieve_region(self): + try: + region = await self._get_region() + return region + except self._RETRIES_EXCEEDED_ERROR_CLS: + logger.debug("Max number of attempts exceeded (%s) when " + "attempting to retrieve data from metadata service.", + self._num_attempts) + return None + + async def _get_region(self): + token = await self._fetch_metadata_token() + response = await self._get_request( + url_path=self._URL_PATH, + retry_func=self._default_retry, + token=token + ) + availability_zone = response.text + region = availability_zone[:-1] + return region + + class AioS3RegionRedirector(S3RegionRedirector): async def redirect_from_error(self, request_dict, response, operation, **kwargs): if response is None: diff --git a/setup.py b/setup.py index 61a07624..38b026df 100644 --- a/setup.py +++ b/setup.py @@ -7,23 +7,22 @@ # NOTE: When updating botocore make sure to update awscli/boto3 versions below install_requires = [ # pegged to also match items in `extras_require` - 'botocore>=1.23.24,<1.23.25', + 'botocore>=1.24.21,<1.24.22', 'aiohttp>=3.3.1', 'wrapt>=1.10.10', 'aioitertools>=0.5.1', ] +extras_require = { + 'awscli': ['awscli>=1.22.76,<1.22.77'], + 'boto3': ['boto3>=1.21.21,<1.21.22'], +} + def read(f): return open(os.path.join(os.path.dirname(__file__), f)).read().strip() -extras_require = { - 'awscli': ['awscli>=1.22.24,<1.22.25'], - 'boto3': ['boto3>=1.20.24,<1.20.25'], -} - - def read_version(): regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") init_py = os.path.join(os.path.dirname(__file__), @@ -37,32 +36,32 @@ def read_version(): 'aiobotocore/__init__.py') -classifiers = [ - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Environment :: Web Environment', - 'Development Status :: 4 - Beta', - 'Framework :: AsyncIO', -] - - setup( name='aiobotocore', version=read_version(), description='Async client for aws services using botocore and aiohttp', long_description='\n\n'.join((read('README.rst'), read('CHANGES.rst'))), long_description_content_type='text/x-rst', - classifiers=classifiers, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Natural Language :: English', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Environment :: Web Environment', + 'Framework :: AsyncIO', + ], author="Nikolay Novik", author_email="nickolainovik@gmail.com", url='https://github.com/aio-libs/aiobotocore', download_url='https://pypi.python.org/pypi/aiobotocore', - license='Apache 2', + license='Apache License 2.0', packages=find_packages(), python_requires='>=3.6', install_requires=install_requires, diff --git a/tests/test_basic_s3.py b/tests/test_basic_s3.py index 2bc4e40a..be3eefec 100644 --- a/tests/test_basic_s3.py +++ b/tests/test_basic_s3.py @@ -43,12 +43,6 @@ async def test_can_make_request_no_verify(s3_client): @pytest.mark.asyncio async def test_fail_proxy_request(aa_fail_proxy_config, s3_client, monkeypatch): # based on test_can_make_request - - monkeypatch.setattr(httpsession, 'DEPRECATED_1_4_0_APIS', 1) - with pytest.raises(httpsession.ClientProxyConnectionError): - await s3_client.list_buckets() - - monkeypatch.setattr(httpsession, 'DEPRECATED_1_4_0_APIS', 0) with pytest.raises(httpsession.EndpointConnectionError): await s3_client.list_buckets() diff --git a/tests/test_patches.py b/tests/test_patches.py index 59953fef..8bd21a15 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -13,6 +13,7 @@ import botocore from botocore.args import ClientArgsCreator from botocore.client import ClientCreator, BaseClient, Config +from botocore.configprovider import SmartDefaultsConfigStoreFactory from botocore.endpoint import convert_to_response_dict, Endpoint, \ EndpointCreator from botocore.paginate import PageIterator, ResultKeyIterator @@ -29,7 +30,8 @@ generate_presigned_post, generate_db_auth_token, add_generate_db_auth_token from botocore.hooks import EventAliaser, HierarchicalEmitter from botocore.utils import ContainerMetadataFetcher, IMDSFetcher, \ - InstanceMetadataFetcher, S3RegionRedirector + InstanceMetadataFetcher, S3RegionRedirector, InstanceMetadataRegionFetcher, \ + IMDSRegionProvider from botocore.credentials import Credentials, RefreshableCredentials, \ CachedCredentialFetcher, AssumeRoleCredentialFetcher, EnvProvider, \ ContainerProvider, InstanceMetadataProvider, ProfileProviderBuilder, \ @@ -94,7 +96,7 @@ ClientCreator._register_v2_adaptive_retries: {'665ecd77d36a5abedffb746d83a44bb0a64c660a'}, - BaseClient._make_api_call: {'0c59329d4c8a55b88250b512b5e69239c42246fb'}, + BaseClient._make_api_call: {'6517c7ead41bf0c70f38bb70666bffd21835ed72'}, BaseClient._make_request: {'033a386f7d1025522bea7f2bbca85edc5c8aafd2'}, BaseClient._convert_to_request_dict: {'0071c2a37c3c696d9b0fba5f54b2985489c76b78'}, BaseClient._emit_api_params: {'2bfadaaa70671b63c50b1beed6d6c66e85813e9b'}, @@ -104,7 +106,7 @@ # config.py Config.merge: {'c3dd8c3ffe0da86953ceba4a35267dfb79c6a2c8'}, - Config: {'05ad5de7db3910654b0ded39874552aaf29c4e1d'}, + Config: {'1fb5fb546abe4970c98560b9f869339322930cdc'}, # credentials.py create_mfa_serial_refresher: {'180b81fc40c91d1cf40de1a28e32ae7d601e1d50'}, @@ -201,13 +203,19 @@ create_credential_resolver: {'177ad331d4b527b9aae765d90e2f17badefeb4a8'}, get_credentials: {'ff0c735a388ac8dd7fe300a32c1e36cdf33c0f56'}, + # configprovider.py + SmartDefaultsConfigStoreFactory.merge_smart_defaults: + {'e1049d34cba3197b4e70dabbbe59e17686fa90f9'}, + SmartDefaultsConfigStoreFactory.resolve_auto_mode: + {'61e749ec045bb0c670bcbc9846b4cfc16cde5718'}, + # endpoint.py convert_to_response_dict: {'2c73c059fa63552115314b079ae8cbf5c4e78da0'}, Endpoint.create_request: {'4ccc14de2fd52f5c60017e55ff8e5b78bbaabcec'}, - Endpoint._send_request: {'5591e6609c138ea7a16298390f4952aac1fbe962'}, - Endpoint._get_response: {'46c3a8cb4ff7672b75193ce5571dbea48aa9da75'}, - Endpoint._do_get_response: {'0bc57fbacf3c49ec5cd243b014d531a38b9b4138'}, + Endpoint._send_request: {'214fa3a0e72f3877cef915c8429d40729775f0cf'}, + Endpoint._get_response: {'6803c16fb6576ea18d9e3d8ffb2e9f3874d9b8ee'}, + Endpoint._do_get_response: {'91370e4a034ec61fa8090fe9442cfafe9b63c6cc'}, Endpoint._needs_retry: {'0f40f52d8c90c6e10b4c9e1c4a5ca00ef2c72850'}, Endpoint._send: {'644c7e5bb88fecaa0b2a204411f8c7e69cc90bf1'}, Endpoint._add_modeled_error_fields: {'1eefcfacbe9a2c3700c61982e565ce6c4cf1ea3a'}, @@ -240,19 +248,22 @@ JSONParser._create_event_stream: {'0564ba55383a71cc1ba3e5be7110549d7e9992f5'}, JSONParser._do_parse: {'9c3d5832e6c55a87630128cc8b9121579ef4a708'}, JSONParser._handle_event_stream: {'3cf7bb1ecff0d72bafd7e7fd6625595b4060abd6'}, - JSONParser.parse: {'46e9e8ecf2ca3a9cdddbb40825cb58fb246b28f1'}, + + # NOTE: if this hits we need to change our ResponseParser impl in JSONParser + JSONParser.parse: {'38231a2fffddfa6e91c56c2a01134459e365beb3'}, + RestJSONParser._create_event_stream: {'0564ba55383a71cc1ba3e5be7110549d7e9992f5'}, create_parser: {'37e9f1c3b60de17f477a9b79eae8e1acaa7c89d7'}, # response.py - StreamingBody: {'0c52037e7b46dc2be5fc08fe572fbb2fe280e0af'}, + StreamingBody: {'a177edffd0c4ec72f1bb4b9e09ea33bc6d37b248'}, get_response: {'f31b478792a5e0502f142daca881b69955e5c11d'}, # session.py Session.__init__: {'ccf156a76beda3425fb54363f3b2718dc0445f6d'}, Session._register_response_parser_factory: {'d6cd5a8b1b473b0ec3b71db5f621acfb12cc412c'}, - Session.create_client: {'801d1b8942f89d3c497d857b280d48628878d1ed'}, + Session.create_client: {'7e0c40c06d3fede4ebbd862f1d6e51118c4a1ff0'}, Session._create_credential_resolver: {'87e98d201c72d06f7fbdb4ebee2dce1c09de0fb2'}, Session.get_credentials: {'c0de970743b6b9dd91b5a71031db8a495fde53e4'}, get_session: {'c47d588f5da9b8bde81ccc26eaef3aee19ddd901'}, @@ -260,6 +271,8 @@ Session.get_service_model: {'1c8f93e6fb9913e859e43aea9bc2546edbea8365'}, Session.get_available_regions: {'bc455d24d98fbc112ff22325ebfd12a6773cb7d4'}, Session.register: {'39791fd2cffcea480f81e77c7daf3974581d9291'}, + Session._register_smart_defaults_factory: + {'24ab10e4751ada800dde24d40d1d105be76a0a14'}, # signers.py RequestSigner.handler: {'371909df136a0964ef7469a63d25149176c2b442'}, @@ -290,8 +303,8 @@ {'7e5acdd2cf0167a047e3d5ee1439565a2f79f6a6'}, # Overrided session and dealing with proxy support IMDSFetcher.__init__: {'a0766a5ba7dde9c26f3c51eb38d73f8e6087d492'}, - IMDSFetcher._get_request: {'96a0e580cab5a21deb4d2cd7e904aa17d5e1e504'}, - IMDSFetcher._fetch_metadata_token: {'dcb147f6c7a425ba9e30be8ad4818be4b781305c'}, + IMDSFetcher._get_request: {'d06ba6890b94c819e79e27ac819454b28f704535'}, + IMDSFetcher._fetch_metadata_token: {'c162c832ec24082cd2054945382d8dc6a1ec5e7b'}, InstanceMetadataFetcher.retrieve_iam_role_credentials: {'76737f6add82a1b9a0dc590cf10bfac0c7026a2e'}, @@ -303,6 +316,16 @@ {'f6f765431145a9bed8e73e6a3dbc7b0d6ae5f738'}, S3RegionRedirector.get_bucket_region: {'b5bbc8b010576668dc2812d657c4b48af79e8f99'}, + InstanceMetadataRegionFetcher.retrieve_region: + {'e916aeb4a28a265224a21006dce1d443cd1207c4'}, + InstanceMetadataRegionFetcher._get_region: + {'73f8c60d21aae765db3a473a527c846e0108291f'}, + IMDSRegionProvider.provide: + {'09d1b70bc1dd7a37cb9ffd437acd71283b9142e9'}, + IMDSRegionProvider._get_instance_metadata_region: + {'4631ced79cff143de5d3fdf03cd69720778f141b'}, + IMDSRegionProvider._create_fetcher: + {'28b711326769d03a282558066058cd85b1cb4568'}, # waiter.py NormalizedOperationMethod.__call__: {'79723632d023739aa19c8a899bc2b814b8ab12ff'}, @@ -314,7 +337,7 @@ inject_presigned_url_ec2: {'37fad2d9c53ca4f1783e32799fa8f70930f44c23'}, # httpsession.py - URLLib3Session: {'021bdd55d5579d387eb7d1e77fa0e59a37cbaa7c'}, + URLLib3Session: {'5adede4ba9d2a80e776bfeb71127656fafff91d7'}, EndpointDiscoveryHandler.discover_endpoint: {'d87eff9008356a6aaa9b7078f23ba7a9ff0c7a60'}, diff --git a/tests/test_session.py b/tests/test_session.py index c56cdb5a..87fefbea 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -44,7 +44,6 @@ async def test_retry(session: AioSession, caplog: LogCaptureFixture, monkeypatch endpoint_url='http://localhost:7878') as client: # this needs the new style exceptions to work - monkeypatch.setattr(httpsession, 'DEPRECATED_1_4_0_APIS', 0) with pytest.raises(httpsession.EndpointConnectionError): await client.get_object(Bucket='foo', Key='bar') diff --git a/tests/test_version.py b/tests/test_version.py index 1c42a089..83d6b661 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,13 +1,31 @@ -import pytest +import ast +from datetime import datetime +from itertools import chain + import docutils.nodes import docutils.parsers.rst import docutils.utils import docutils.frontend -import aiobotocore import re +import operator from pathlib import Path from packaging import version -from datetime import datetime +from typing import NamedTuple, Optional + +import aiobotocore +import requests +import pytest +from pip._internal.req.constructors import install_req_from_line +from pip._internal.req import InstallRequirement +from pip._vendor.packaging.specifiers import SpecifierSet + + +_root_path = Path(__file__).absolute().parent.parent + + +# date can be YYYY-MM-DD or "TBD" +_rst_ver_date_str_re = re.compile( + r'(?P\d+\.\d+\.\d+) \((?P\d{4}-\d{2}-\d{2}|TBD)\)') # from: https://stackoverflow.com/a/48719723/1241593 @@ -21,9 +39,78 @@ def _parse_rst(text: str) -> docutils.nodes.document: return document -# date can be YYYY-MM-DD or "TBD" -_rst_ver_date_str_re = re.compile( - r'(?P\d+\.\d+\.\d+) \((?P\d{4}-\d{2}-\d{2}|TBD)\)') +def _get_assign_target_name(node: ast.Assign): + assert len(node.targets) == 1 + target = node.targets[0] + assert isinstance(target, ast.Name) + return target.id + + +class VersionInfo(NamedTuple): + least_version: str + specifier_set: SpecifierSet + + +def _get_boto_module_versions( + setup_content: str, ensure_plus_one_patch_range: bool = False +): + parsed = ast.parse(setup_content) + top_level_vars = {"install_requires", "requires", "extras_require"} + assignments = dict() + for node in parsed.body: + if isinstance(node, ast.Assign): + target_name = _get_assign_target_name(node) + if target_name not in top_level_vars: + continue + + value = ast.literal_eval(node.value) + assignments[target_name] = value + + module_versions = dict() + + for ver in chain( + assignments.get("install_requires", []), + assignments.get("requires", []), + assignments.get("extras_require", {}).values() + ): + if isinstance(ver, str): + ver: InstallRequirement = install_req_from_line(ver) + elif isinstance(ver, list): + assert len(ver) == 1 + ver: InstallRequirement = install_req_from_line(ver[0]) + else: + assert False, f'Unsupported ver: {ver}' + + module = ver.req.name + if module not in {'botocore', 'awscli', 'boto3'}: + continue + + # NOTE: don't support complex versioning yet as requirements are unknown + gte = lt = eq = None # type: Optional[version.Version] + for spec in ver.req.specifier: + if spec.operator == '>=': + assert gte is None + gte = version.parse(spec.version) + elif spec.operator == '<': + assert lt is None + lt = version.parse(spec.version) + elif spec.operator == '==': + assert eq is None + eq = version.parse(spec.version) + else: + assert False, f'unsupported operator: {spec.operator}' + + if ensure_plus_one_patch_range: + assert len(gte.release) == len(lt.release) == 3, \ + f'{module} gte: {gte} diff len than {lt}' + assert lt.release == tuple(map(operator.add, gte.release, (0, 0, 1))),\ + f'{module} gte: {gte} not one patch off from {lt}' + + module_versions[module] = VersionInfo( + gte.public if gte else None, ver.req.specifier + ) + + return module_versions @pytest.mark.moto @@ -31,7 +118,7 @@ def test_release_versions(): # ensures versions in CHANGES.rst + __init__.py match init_version = version.parse(aiobotocore.__version__) - changes_path = Path(__file__).absolute().parent.parent / 'CHANGES.rst' + changes_path = _root_path / 'CHANGES.rst' with changes_path.open('r') as f: changes_doc = _parse_rst(f.read()) @@ -65,3 +152,26 @@ def test_release_versions(): rst_prev_date = datetime.strptime(rst_prev_date, '%Y-%m-%d').date() assert rst_date > rst_prev_date, 'Current release must be after last release' + + # get aioboto reqs + with (_root_path / 'setup.py').open() as f: + content = f.read() + aioboto_reqs = _get_boto_module_versions(content, True) + + # get awscli reqs + awscli_resp = requests.get( + f"https://raw.githubusercontent.com/aws/aws-cli/" + f"{aioboto_reqs['awscli'].least_version}/setup.py") + awscli_reqs = _get_boto_module_versions(awscli_resp.text) + assert awscli_reqs['botocore'].specifier_set.contains( + aioboto_reqs['botocore'].least_version) + + # get boto3 reqs + boto3_resp = requests.get( + f"https://raw.githubusercontent.com/boto/boto3/" + f"{aioboto_reqs['boto3'].least_version}/setup.py") + boto3_reqs = _get_boto_module_versions(boto3_resp.text) + assert boto3_reqs['botocore'].specifier_set.contains( + aioboto_reqs['botocore'].least_version) + + print()