Merge branch 'master' into master

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [pypy3, 3.5.x, 3.6.x, 3.7.x, 3.8.x, 3.9.x, 3.10.x, 3.11.x]
+        python-version: [3.8.x, 3.9.x, 3.10.x, 3.11.x, 3.12.x]
 
     steps:
     - uses: actions/checkout@v2

CHANGES.md

@@ -1,3 +1,23 @@
+Release 5.33 (release date: 2024-12-10)
+======================================
+Bug Fixes
+------------------
+- Corrected python-requires as <3.13 (#1827)
+
+Other Changes
+------------------
+
+Release 5.32 (release date: 2024-12-04)
+======================================
+New Features
+------------------
+- Added explicit support for Python 3.12
+
+Other Changes
+------------------
+- Dropped Support for Python 3.5, 3.6 and 3.7.
+- Pinned pyOpenSSL to a maximum version of 24.2.1 (#1822).
+
 Release 5.31 (release date: 2024-10-10)
 ======================================
 Bug Fixes

CHECKSUM

@@ -1 +1 @@
-047a02622fbb528fb2e5c13201a33dc2
+c1441eab7ca94a134090e58f578e2e80

VERSION

@@ -1 +1 @@
-5.31
+5.33

gslib/commands/perfdiag.py

@@ -42,7 +42,7 @@
 import six
 from six.moves import cStringIO
 from six.moves import http_client
-from six.moves import xrange
+from six.moves import range
 from six.moves import range
 
 import gslib
@@ -1222,7 +1222,7 @@ def _RunListTests(self):
         [random.choice(string.ascii_lowercase) for _ in range(10)])
     list_prefix = 'gsutil-perfdiag-list-' + random_id + '-'
 
-    for _ in xrange(self.num_objects):
+    for _ in range(self.num_objects):
       fpath = self._MakeTempFile(0,
                                  mem_data=True,
                                  mem_metadata=True,

gslib/commands/signurl.py

@@ -26,6 +26,7 @@
 import copy
 from datetime import datetime
 from datetime import timedelta
+from datetime import timezone
 import getpass
 import json
 import re
@@ -214,7 +215,7 @@
 
 def _NowUTC():
   """Returns the current utc time as a datetime object."""
-  return datetime.utcnow()
+  return datetime.now(tz=timezone.utc).replace(tzinfo=None)
 
 
 def _DurationToTimeDelta(duration):
@@ -697,7 +698,7 @@ def RunCommand(self):
                                 billing_project=billing_project,
                                 string_to_sign_debug=True)
 
-      expiration = calendar.timegm((datetime.utcnow() + delta).utctimetuple())
+      expiration = calendar.timegm((datetime.now(tz=timezone.utc).replace(tzinfo=None) + delta).utctimetuple())
       expiration_dt = datetime.fromtimestamp(expiration)
 
       time_str = expiration_dt.strftime('%Y-%m-%d %H:%M:%S')

gslib/tests/test_compose.py

@@ -19,7 +19,7 @@
 from __future__ import division
 from __future__ import unicode_literals
 
-from six.moves import xrange
+from six.moves import range
 from six.moves import range
 
 from gslib.commands.compose import MAX_COMPOSE_ARITY
@@ -45,7 +45,7 @@ def check_n_ary_compose(self, num_components):
     bucket_uri = self.CreateBucket()
 
     data_list = [
-        ('data-%d,' % i).encode('ascii') for i in xrange(num_components)
+        ('data-%d,' % i).encode('ascii') for i in range(num_components)
     ]
     components = [
         self.CreateObject(bucket_uri=bucket_uri, contents=data).uri

gslib/tests/test_cp.py

@@ -128,7 +128,7 @@
 import six
 from six.moves import http_client
 from six.moves import range
-from six.moves import xrange
+from six.moves import range
 
 if six.PY3:
   long = int  # pylint: disable=redefined-builtin,invalid-name
@@ -3829,7 +3829,7 @@ def test_cp_resumable_download_gzip(self):
     object_uri = self.CreateObject()
     random.seed(0)
     contents = str([
-        random.choice(string.ascii_letters) for _ in xrange(self.halt_size)
+        random.choice(string.ascii_letters) for _ in range(self.halt_size)
     ]).encode('ascii')
     random.seed()  # Reset the seed for any other tests.
     fpath1 = self.CreateTempFile(file_name='unzipped.txt', contents=contents)

gslib/tests/test_retention.py

@@ -273,7 +273,7 @@ def test_get_retention_unlocked(self):
         ['retention', 'get', suri(bucket_uri)], return_stdout=True)
     if self._use_gcloud_storage:
       # Sometimes the field is absent if isLocked is false.
-      self.assertNotRegexpMatches(stdout, r'isLocked \: true')
+      self.assertNotRegex(stdout, r'isLocked \: true')
       self.assertRegex(stdout, r'retentionPeriod\: \'86400\'')
       self.assertRegex(stdout, r'effectiveTime\: \'.*\'')
     else:

gslib/tests/test_rsync.py

@@ -2929,7 +2929,7 @@ def _test_dir_to_bucket_regex_negative_lookahead(self, flag, includes):
                                            'b.txt'), ('data3', 'data4', 'c.txt')
     ])
     self.RunGsUtil(
-        ['rsync', '-r', flag, '^(?!.*\.txt$).*', tmpdir,
+        ['rsync', '-r', flag, r'^(?!.*\.txt$).*', tmpdir,
          suri(bucket_uri)],
         return_stderr=True)
     listing = TailSet(tmpdir, self.FlatListDir(tmpdir))

gslib/tests/test_shim_util.py

@@ -1185,7 +1185,7 @@ def test_gets_format_flag_newline_on_non_windows(self):
 
   def test_gets_format_flag_escaped_newline_on_windows(self):
     with mock.patch.object(system_util, 'IS_WINDOWS', new=True):
-      self.assertEqual(shim_util.get_format_flag_newline(), '^\^n')
+      self.assertEqual(shim_util.get_format_flag_newline(), r'^\^n')
 
 
 class TestBotoTranslation(testcase.GsUtilUnitTestCase):

gslib/utils/copy_helper.py

@@ -45,7 +45,7 @@
 import traceback
 
 import six
-from six.moves import xrange
+from six.moves import range
 from six.moves import range
 
 from apitools.base.protorpclite import protojson
@@ -4138,13 +4138,13 @@ def Initialize(self, source_url, destination_url):
     # Always use the source_url as the key for the item. This is unique.
     self.Set(source_url, 'source_uri', source_url)
     self.Set(source_url, 'destination_uri', destination_url)
-    self.Set(source_url, 'start_time', datetime.datetime.utcnow())
+    self.Set(source_url, 'start_time', datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None))
 
   def SetResult(self, source_url, bytes_transferred, result, description=''):
     self.Set(source_url, 'bytes', bytes_transferred)
     self.Set(source_url, 'result', result)
     self.Set(source_url, 'description', description)
-    self.Set(source_url, 'end_time', datetime.datetime.utcnow())
+    self.Set(source_url, 'end_time', datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None))
     self._WriteRowToManifestFile(source_url)
     self._RemoveItemFromManifest(source_url)
 
@@ -4270,7 +4270,7 @@ def ResolveWildcardsInPathBeforeFinalDir(src_url_path_sans_final_dir,
   wildcarded_src_obj_path = StorageUrlFromString(
       src_url_path_sans_final_dir).object_name.split('/')
   expanded_src_obj_path = exp_src_url.object_name.split('/')
-  for path_segment_index in xrange(len(wildcarded_src_obj_path)):
+  for path_segment_index in range(len(wildcarded_src_obj_path)):
     if ContainsWildcard(wildcarded_src_obj_path[path_segment_index]):
       # The expanded path is guaranteed to be have at least as many path
       # segments as the wildcarded path.

gslib/utils/signurl_helper.py

@@ -16,6 +16,7 @@
 
 import base64
 from datetime import datetime
+from datetime import timezone
 import hashlib
 
 from gslib.utils.constants import UTF8
@@ -33,7 +34,7 @@
 
 
 def _NowUTC():
-  return datetime.utcnow()
+  return datetime.now(tz=timezone.utc).replace(tzinfo=None)
 
 
 def CreatePayload(client_id,

gsutil.py

@@ -27,12 +27,12 @@
 # TODO: gsutil-beta: Distribute a pylint rc file.
 
 ver = sys.version_info
-if (ver.major == 2 and ver.minor < 7) or (ver.major == 3 and (ver.minor < 5 or ver.minor > 11)):
+if ver.major != 3 or ver.minor < 8 or ver.minor > 12:
   sys.exit(
-    "Error: gsutil requires Python version 2.7 or 3.5-3.11, but a different version is installed.\n"
+    "Error: gsutil requires Python version 3.8-3.12, but a different version is installed.\n"
     "You are currently running Python {}.{}\n"
     "Follow the steps below to resolve this issue:\n"
-    "\t1. Switch to Python 3.5-3.11 using your Python version manager or install an appropriate version.\n"
+    "\t1. Switch to Python 3.8-3.12 using your Python version manager or install an appropriate version.\n"
     "\t2. If you are unsure how to manage Python versions, visit [https://cloud.google.com/storage/docs/gsutil_install#specifications] for detailed instructions.".format(ver.major, ver.minor)
   )
 
@@ -116,7 +116,7 @@ def _fix_google_module():
     ('chardet', ''),  # requests dependency
     ('certifi', ''),  # requests dependency
     ('idna', ''),  # requests dependency
-    ('requests', ''),  # google auth dependency
+    ('requests', 'src'),  # google auth dependency
     ('google-auth-library-python', ''),
     ('google-auth-library-python-httplib2', ''), #Package name: google-auth-httplib2
 ]

setup.py

@@ -36,7 +36,7 @@
 """
 
 requires = [
-    'argcomplete>=1.9.4',
+    'argcomplete>=3.5.1',
     'crcmod>=1.7',
     'fasteners>=0.14.1',
     'gcs-oauth2-boto-plugin>=3.2',
@@ -47,7 +47,7 @@
     # 3.0.5 is the last version that supports Python 3.3 or lower.
     'mock>=2.0.0, <=3.0.5; python_version < "3.3"',
     'monotonic>=1.4',
-    'pyOpenSSL>=0.13',
+    'pyOpenSSL>=0.13, <=24.2.1',
     'retry_decorator>=1.0.0',
     'six>=1.16.0',
     # aiohttp is the extra dependency that contains requests lib.
@@ -126,16 +126,16 @@ def make_release_tree(self, base_dir, files):
         'Natural Language :: English',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Topic :: System :: Filesystems',
         'Topic :: Utilities',
     ],
-    # Gsutil supports Python 3.5+
-    python_requires='!=2.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4',
+    # Gsutil supports Python 3.8 to 3.12
+    python_requires='>=3.8, <3.13',
     platforms='any',
     packages=find_packages(
         exclude=[

test/ci/kokoro/linux/py312_json.cfg

@@ -0,0 +1,61 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build_file: "/src/gsutil/test/ci/kokoro/run_integ_tests.sh"
+timeout_mins: 60
+
+# Get access keys from Keystore
+# go/kokoro-keystore
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 74008
+      keyname: "gsutil_kokoro_service_key"
+    }
+    keystore_resource {
+      keystore_config_id: 74008
+      keyname: "mtls_test_account_refresh_token"
+    }
+    keystore_resource {
+      keystore_config_id: 74008
+      keyname: "mtls_test_account_client_id"
+    }
+    keystore_resource {
+      keystore_config_id: 74008
+      keyname: "mtls_test_account_client_secret"
+    }
+    keystore_resource {
+      keystore_config_id: 74008
+      keyname: "mtls_test_cert"
+    }
+  }
+}
+
+# Environment variables to specify interpreter version.
+# go/kokoro-env-vars
+env_vars {
+  key: "PYMAJOR"
+  value: "3"
+}
+
+env_vars {
+  key: "PYMINOR"
+  value: "12"
+}
+
+env_vars {
+  key: "API"
+  value: "json"
+}

test/ci/kokoro/linux/py312_xml.cfg

@@ -0,0 +1,45 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build_file: "/src/gsutil/test/ci/kokoro/run_integ_tests.sh"
+timeout_mins: 60
+
+# Get access keys from Keystore
+# go/kokoro-keystore
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 74008
+      keyname: "gsutil_kokoro_service_key"
+    }
+  }
+}
+
+# Environment variables to specify interpreter version.
+# go/kokoro-env-vars
+env_vars {
+  key: "PYMAJOR"
+  value: "3"
+}
+
+env_vars {
+  key: "PYMINOR"
+  value: "12"
+}
+
+env_vars {
+  key: "API"
+  value: "xml"
+}