Catch Wikidata API errors and retry

Retry API error calls with exponential backoff.

Send mail to admin if errors continue after retries.

Includes a test.

geocode/mail.py

@@ -0,0 +1,28 @@
+"""Send mail to admin."""
+
+import smtplib
+from email.mime.text import MIMEText
+from email.utils import formatdate, make_msgid
+
+import flask
+
+
+def send_to_admin(subject: str, body: str) -> None:
+    """Send an e-mail."""
+    app = flask.current_app
+    mail_from = app.config["MAIL_FROM"]
+    msg = MIMEText(body, "plain", "UTF-8")
+
+    msg["Subject"] = subject
+    msg["To"] = ", ".join(app.config["ADMINS"])
+    msg["From"] = f'{app.config["MAIL_FROM_NAME"]} <{app.config["MAIL_FROM"]}>'
+    msg["Date"] = formatdate()
+    msg["Message-ID"] = make_msgid()
+
+    # extra mail headers from config
+    for header_name, value in app.config.get("MAIL_HEADERS", {}).items():
+        msg[header_name] = value
+
+    s = smtplib.SMTP(app.config["SMTP_HOST"])
+    s.sendmail(mail_from, app.config["ADMINS"], msg.as_string())
+    s.quit()

geocode/wikidata.py

@@ -3,16 +3,27 @@
 import typing
 import urllib.parse
 
+import backoff
+import backoff.types
 import requests
 from flask import render_template
+from requests.exceptions import JSONDecodeError, RequestException
 
-from . import headers
+from . import headers, mail
 
 wikidata_query_api_url = "https://query.wikidata.org/bigdata/namespace/wdq/sparql"
 wd_entity = "http://www.wikidata.org/entity/Q"
 commons_cat_start = "https://commons.wikimedia.org/wiki/Category:"
 
 
+def giveup(details: backoff.types.Details) -> None:
+    """Display API call fail debug info."""
+    last_exception = details["exception"]  # type: ignore
+    if last_exception and isinstance(last_exception, APIResponseError):
+        body = f"Error making Wikidata API call\n\n{last_exception.response.text}"
+        mail.send_to_admin("Geocode error", body)
+
+
 class QueryError(Exception):
     """Query error."""
 
@@ -22,13 +33,31 @@ class QueryError(Exception):
         self.r = r
 
 
+class APIResponseError(Exception):
+    """Custom exception for API errors with response text."""
+
+    def __init__(self, message: str, response: requests.Response):
+        """Init."""
+        super().__init__(message)
+        self.response = response
+
+
+@backoff.on_exception(
+    backoff.expo,
+    (RequestException, APIResponseError),
+    max_tries=5,
+    on_giveup=giveup,
+)
 def api_call(params: dict[str, str | int]) -> dict[str, typing.Any]:
     """Wikidata API call."""
     api_params: dict[str, str | int] = {"format": "json", "formatversion": 2, **params}
-    r = requests.get(
-        "https://www.wikidata.org/w/api.php", params=api_params, headers=headers
-    )
-    return typing.cast(dict[str, typing.Any], r.json())
+    try:
+        r = requests.get(
+            "https://www.wikidata.org/w/api.php", params=api_params, headers=headers
+        )
+        return typing.cast(dict[str, typing.Any], r.json())
+    except JSONDecodeError:
+        raise APIResponseError("Failed to decode JSON", r)
 
 
 def get_entity(qid: str) -> dict[str, typing.Any] | None: