Add request retries for 5XX errors on Science Museum & Brooklyn Museum (

#4715)

catalog/dags/providers/provider_api_scripts/brooklyn_museum.py

@@ -1,7 +1,9 @@
 import logging
 
+import backoff
 import lxml.html as html
 from airflow.models import Variable
+from requests import HTTPError
 
 from common.licenses import LicenseInfo, get_license_info
 from common.loader import provider_details as prov
@@ -21,6 +23,17 @@ def __init__(self, *args, **kwargs):
         self.api_key = Variable.get("API_KEY_BROOKLYN_MUSEUM")
         self.headers = {"api_key": self.api_key}
 
+    # Brooklyn Museum's API tends to be flaky, so we add a backoff on every request
+    # for 5XX error codes.
+    # See: https://github.com/WordPress/openverse/issues/4712
+    get_response_json = backoff.on_exception(
+        backoff.expo,
+        HTTPError,
+        max_time=60 * 2,
+        # Only retry on 5XX errors
+        giveup=lambda e: e.response.status_code not in {502, 503, 504},
+    )(ProviderDataIngester.get_response_json)
+
     def get_next_query_params(self, prev_query_params: dict | None) -> dict:
         if not prev_query_params:
             return {

catalog/dags/providers/provider_api_scripts/science_museum.py

@@ -14,6 +14,9 @@
 import re
 from datetime import date
 
+import backoff
+from requests import HTTPError
+
 from common import slack
 from common.licenses import LicenseInfo, get_license_info
 from common.loader import provider_details as prov
@@ -44,6 +47,17 @@ def __init__(self, *args, **kwargs):
         # easily detect when the page limit is reached
         self.page_number = 0
 
+    # Science Museum's API tends to be flaky, so we add a backoff on every request
+    # for 5XX error codes.
+    # See: https://github.com/WordPress/openverse/issues/4710
+    get_response_json = backoff.on_exception(
+        backoff.expo,
+        HTTPError,
+        max_time=60 * 2,
+        # Only retry on 5XX errors
+        giveup=lambda e: e.response.status_code not in {502, 503, 504},
+    )(ProviderDataIngester.get_response_json)
+
     @staticmethod
     def _get_year_ranges(final_year: int) -> list[tuple[int, int]]:
         """