From 5c00ef014e9deff0bd81aba72e32a2847c354b08 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 15 Nov 2023 09:14:17 +0200 Subject: [PATCH 1/3] Use _epoch_seconds_to_datetime in _parse_timestamp_with_tzinfo --- botocore/utils.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/botocore/utils.py b/botocore/utils.py index 314d30516d..4886827069 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -935,12 +935,18 @@ def _epoch_seconds_to_datetime(value, tzinfo): def _parse_timestamp_with_tzinfo(value, tzinfo): """Parse timestamp with pluggable tzinfo options.""" + # For numeric values attempt fallback to using fromtimestamp-free method. + # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This + # may raise ``OverflowError``, if the timestamp is out of the range of + # values supported by the platform C localtime() function, and ``OSError`` + # on localtime() failure. It's common for this to be restricted to years + # from 1970 through 2038." if isinstance(value, (int, float)): # Possibly an epoch time. - return datetime.datetime.fromtimestamp(value, tzinfo()) + return _epoch_seconds_to_datetime(value, tzinfo) else: try: - return datetime.datetime.fromtimestamp(float(value), tzinfo()) + return _epoch_seconds_to_datetime(float(value), tzinfo) except (TypeError, ValueError): pass try: @@ -974,30 +980,7 @@ def parse_timestamp(value): tzinfo.__name__, exc_info=e, ) - # For numeric values attempt fallback to using fromtimestamp-free method. - # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This - # may raise ``OverflowError``, if the timestamp is out of the range of - # values supported by the platform C localtime() function, and ``OSError`` - # on localtime() failure. It's common for this to be restricted to years - # from 1970 through 2038." - try: - numeric_value = float(value) - except (TypeError, ValueError): - pass - else: - try: - for tzinfo in tzinfo_options: - return _epoch_seconds_to_datetime(numeric_value, tzinfo=tzinfo) - except (OSError, OverflowError) as e: - logger.debug( - 'Unable to parse timestamp using fallback method with "%s" ' - 'timezone info.', - tzinfo.__name__, - exc_info=e, - ) - raise RuntimeError( - f'Unable to calculate correct timezone offset for "{value}"' - ) + raise RuntimeError(f'Unable to parse timestamp {value!r}') def parse_to_aware_datetime(value): From 3c914848e35a8b9900321f3dd0c81ec199bb6f4e Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 15 Nov 2023 09:42:33 +0200 Subject: [PATCH 2/3] Simplify timestamp parsing (and get rid of tzlocal/tzwinlocal use) --- botocore/compat.py | 2 ++ botocore/utils.py | 73 +++++++++++++++++----------------------- tests/unit/test_utils.py | 12 ------- 3 files changed, 32 insertions(+), 55 deletions(-) diff --git a/botocore/compat.py b/botocore/compat.py index 6f79d43e3f..5aa06290a1 100644 --- a/botocore/compat.py +++ b/botocore/compat.py @@ -274,6 +274,8 @@ def _windows_shell_split(s): def get_tzinfo_options(): + # This function is not used internally anymore. + # Due to dateutil/dateutil#197, Windows may fail to parse times in the past # with the system clock. We can alternatively fallback to tzwininfo when # this happens, which will get time info from the Windows registry. diff --git a/botocore/utils.py b/botocore/utils.py index 4886827069..c0d7ed45a6 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -54,7 +54,6 @@ UNSAFE_URL_CHARS, OrderedDict, get_md5, - get_tzinfo_options, json, quote, urlparse, @@ -917,45 +916,27 @@ def percent_encode(input_str, safe=SAFE_CHARS): return quote(input_str, safe=safe) -def _epoch_seconds_to_datetime(value, tzinfo): +_EPOCH_ZERO = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + + +def _epoch_seconds_to_datetime(value): """Parse numerical epoch timestamps (seconds since 1970) into a ``datetime.datetime`` in UTC using ``datetime.timedelta``. This is intended as fallback when ``fromtimestamp`` raises ``OverflowError`` or ``OSError``. :type value: float or int :param value: The Unix timestamps as number. - - :type tzinfo: callable - :param tzinfo: A ``datetime.tzinfo`` class or compatible callable. """ - epoch_zero = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) - epoch_zero_localized = epoch_zero.astimezone(tzinfo()) - return epoch_zero_localized + datetime.timedelta(seconds=value) - - -def _parse_timestamp_with_tzinfo(value, tzinfo): - """Parse timestamp with pluggable tzinfo options.""" - # For numeric values attempt fallback to using fromtimestamp-free method. - # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This - # may raise ``OverflowError``, if the timestamp is out of the range of - # values supported by the platform C localtime() function, and ``OSError`` - # on localtime() failure. It's common for this to be restricted to years - # from 1970 through 2038." - if isinstance(value, (int, float)): - # Possibly an epoch time. - return _epoch_seconds_to_datetime(value, tzinfo) - else: - try: - return _epoch_seconds_to_datetime(float(value), tzinfo) - except (TypeError, ValueError): - pass try: - # In certain cases, a timestamp marked with GMT can be parsed into a - # different time zone, so here we provide a context which will - # enforce that GMT == UTC. - return dateutil.parser.parse(value, tzinfos={'GMT': tzutc()}) - except (TypeError, ValueError) as e: - raise ValueError(f'Invalid timestamp "{value}": {e}') + return datetime.datetime.fromtimestamp(value, tz=tzutc()) + except (OverflowError, OSError): + # For numeric values attempt fallback to using fromtimestamp-free method. + # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This + # may raise ``OverflowError``, if the timestamp is out of the range of + # values supported by the platform C localtime() function, and ``OSError`` + # on localtime() failure. It's common for this to be restricted to years + # from 1970 through 2038." + return _EPOCH_ZERO + datetime.timedelta(seconds=value) def parse_timestamp(value): @@ -970,17 +951,23 @@ def parse_timestamp(value): This will return a ``datetime.datetime`` object. """ - tzinfo_options = get_tzinfo_options() - for tzinfo in tzinfo_options: - try: - return _parse_timestamp_with_tzinfo(value, tzinfo) - except (OSError, OverflowError) as e: - logger.debug( - 'Unable to parse timestamp with "%s" timezone info.', - tzinfo.__name__, - exc_info=e, - ) - raise RuntimeError(f'Unable to parse timestamp {value!r}') + if isinstance(value, (int, float)): + # Possibly an epoch time. + return _epoch_seconds_to_datetime(value) + + # Possibly something we can cast to an epoch time and convert. + try: + return _epoch_seconds_to_datetime(float(value)) + except (TypeError, ValueError): + pass + + try: + # In certain cases, a timestamp marked with GMT can be parsed into a + # different time zone, so here we provide a context which will + # enforce that GMT == UTC. + return dateutil.parser.parse(value, tzinfos={'GMT': tzutc()}) + except (TypeError, ValueError) as e: + raise ValueError(f'Invalid timestamp "{value}": {e}') def parse_to_aware_datetime(value): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 8a92bd6b06..5bf8ac49db 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -505,18 +505,6 @@ def test_parse_invalid_timestamp(self): with self.assertRaises(ValueError): parse_timestamp('invalid date') - def test_parse_timestamp_fails_with_bad_tzinfo(self): - mock_tzinfo = mock.Mock() - mock_tzinfo.__name__ = 'tzinfo' - mock_tzinfo.side_effect = OSError() - mock_get_tzinfo_options = mock.MagicMock(return_value=(mock_tzinfo,)) - - with mock.patch( - 'botocore.utils.get_tzinfo_options', mock_get_tzinfo_options - ): - with self.assertRaises(RuntimeError): - parse_timestamp(0) - @contextmanager def mocked_fromtimestamp_that_raises(self, exception_type): class MockDatetime(datetime.datetime): From a5866baf6ebf77e92d3e92693d88bad63aa688ef Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Wed, 15 Nov 2023 09:43:53 +0200 Subject: [PATCH 3/3] Get rid of get_tzinfo_options() altogether --- botocore/compat.py | 15 --------------- tests/unit/test_compat.py | 10 ---------- 2 files changed, 25 deletions(-) diff --git a/botocore/compat.py b/botocore/compat.py index 5aa06290a1..961477b536 100644 --- a/botocore/compat.py +++ b/botocore/compat.py @@ -28,7 +28,6 @@ from botocore.vendored import six from botocore.exceptions import MD5UnavailableError -from dateutil.tz import tzlocal from urllib3 import exceptions logger = logging.getLogger(__name__) @@ -273,20 +272,6 @@ def _windows_shell_split(s): return components -def get_tzinfo_options(): - # This function is not used internally anymore. - - # Due to dateutil/dateutil#197, Windows may fail to parse times in the past - # with the system clock. We can alternatively fallback to tzwininfo when - # this happens, which will get time info from the Windows registry. - if sys.platform == 'win32': - from dateutil.tz import tzwinlocal - - return (tzlocal, tzwinlocal) - else: - return (tzlocal,) - - # Detect if CRT is available for use try: import awscrt.auth diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py index 6b448ae5e0..063c6eacf5 100644 --- a/tests/unit/test_compat.py +++ b/tests/unit/test_compat.py @@ -19,7 +19,6 @@ compat_shell_split, ensure_bytes, get_md5, - get_tzinfo_options, total_seconds, unquote_str, ) @@ -208,15 +207,6 @@ def assert_raises(self, s, exception_cls, platform): compat_shell_split(s, platform) -class TestTimezoneOperations(unittest.TestCase): - def test_get_tzinfo_options(self): - options = get_tzinfo_options() - self.assertTrue(len(options) > 0) - - for tzinfo in options: - self.assertIsInstance(tzinfo(), datetime.tzinfo) - - class TestCRTIntegration(unittest.TestCase): def test_has_crt_global(self): try: