From ca43f343c28a90fbefd698efaa2a36f4c19414f0 Mon Sep 17 00:00:00 2001
From: Brad Klein <bklein@cray.com>
Date: Wed, 5 Jul 2023 13:46:23 -0600
Subject: [PATCH] Change cray artifacts to return more than 1000 files
 (CASMPET-6168)

---
 cray/modules/artifacts/cli.py | 10 +++++++++-
 pyproject.toml                |  4 ++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/cray/modules/artifacts/cli.py b/cray/modules/artifacts/cli.py
index ccd5cfc..e0ba158 100644
--- a/cray/modules/artifacts/cli.py
+++ b/cray/modules/artifacts/cli.py
@@ -111,7 +111,15 @@ def list_objects(ctx, bucket):
     # Load config to make sure it exists.
     s3client = get_s3_client()
     try:
-        files = s3client.list_objects(Bucket=bucket).get('Contents', [])
+        files = []
+        paginator = s3client.get_paginator('list_objects')
+        pages = paginator.paginate(Bucket=bucket)
+
+        for page in pages:
+            if 'Contents' in page:
+                for obj in page['Contents']:
+                    files.append(obj)
+
     except (s3client.exceptions.NoSuchBucket, ClientError) as err:
         sys.exit(str(err))
     return {
diff --git a/pyproject.toml b/pyproject.toml
index f45b02d..ec94259 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,6 +46,10 @@ dependencies = [
     'boto3<1.24',
     'botocore<1.27',
     # DO NOT UPDATE the above boto3 and botocore - Breaks cray artifacts bucket list
+    # DO NOT UPDATE the below urllib3 - Breaks cray artifacts list <bucket> pagination
+    # https://github.com/boto/botocore/issues/2926
+    'urllib3>=1.25.4,<1.27',
+    # DO NOT UPDATE the above urllib3 - Breaks cray artifacts list <bucket> pagination
     'click==7.1.2',
     'oauthlib~=3.2',
     'requests-oauthlib~=1.3',