dpriskorn/owl-map
1
0
Fork 0
forked from edward/owl-map
Code Pull requests Activity

Catch MediaWiki database timeout errors and retry

Browse source
This commit is contained in:
Edward Betts 2024-06-06 13:48:18 +00:00
parent 269c6bce54
commit 13ecf4526d
1 changed files with 19 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
update.py
View file

@ -6,6 +6,8 @@ import json
import typing import typing
from time import sleep from time import sleep
import requests.exceptions
from matcher import model, wikidata, wikidata_api from matcher import model, wikidata, wikidata_api
from matcher.database import init_db, session from matcher.database import init_db, session
@ -81,7 +83,14 @@ def handle_edit(change: Change) -> None:
print(f"{ts}: no need to update {qid}") print(f"{ts}: no need to update {qid}")
return return
entity = wikidata_api.get_entity(qid) for attempt in range(100):
try:
entity = wikidata_api.get_entity(qid)
except requests.exceptions.ConnectionError:
print("connection error, retrying.")
sleep(10)
else:
break
entity_qid = entity.pop("id") entity_qid = entity.pop("id")
if entity_qid != qid: if entity_qid != qid:
print(f"{ts}: item {qid} replaced with redirect") print(f"{ts}: item {qid} replaced with redirect")
@ -123,6 +132,15 @@ def update_database() -> None:
r = wikidata_api.get_recent_changes(rcstart=start, rccontinue=rccontinue) r = wikidata_api.get_recent_changes(rcstart=start, rccontinue=rccontinue)
reply = r.json() reply = r.json()
if (
"error" in reply
and reply["error"]["code"] == "internal_api_error_DBQueryTimeoutError"
):
print(reply)
sleep(10)
continue
if "query" not in reply:
print(reply)
for change in reply["query"]["recentchanges"]: for change in reply["query"]["recentchanges"]:
rctype = change["type"] rctype = change["type"]
timestamp = change["timestamp"] timestamp = change["timestamp"]