Added some 5xx error code handling for function (WordPress#5014)

* added some 5xx error code handling for function * Fixed trailing whitespace error. * trying to fix the ruff issue * Ran with ruff * Revert "Ran with ruff" This reverts commit 0c644b4. * Lint --------- Co-authored-by: Staci Cooper <[email protected]>
Danil49 · Oct 29, 2024 · df1596e · df1596e
1 parent 157a2a2
commit df1596e
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/catalog/dags/providers/provider_api_scripts/stocksnap.py b/catalog/dags/providers/provider_api_scripts/stocksnap.py
@@ -15,6 +15,9 @@
 import json
 import logging
 
+import backoff
+from requests.exceptions import HTTPError
+
 from common.licenses import get_license_info
 from common.loader import provider_details as prov
 from providers.provider_api_scripts.provider_data_ingester import ProviderDataIngester
@@ -157,6 +160,17 @@ def _get_title(item):
             img_title = " ".join(tags)
             return img_title.title()
 
+    # Add a backoff on every request
+    # for 5XX error codes.
+    # See: https://github.com/WordPress/openverse/issues/4878
+    @backoff.on_exception(
+        backoff.expo,
+        HTTPError,
+        # 30 minutes
+        max_time=60 * 30,
+        # Only retry for 5xx errors
+        giveup=lambda e: e.response.status_code not in {502, 503, 504},
+    )
     def _get_filesize(self, image_url):
         """Get the size of the image in bytes."""
         resp = self.delayed_requester.head(image_url)