Better handling of read timeout from APIs

This commit is contained in:
Edward Betts 2020-01-08 15:39:25 +00:00
parent 9bbbb45fbb
commit 93df781115

48
app.py
View file

@ -15,6 +15,7 @@ from sqlalchemy.orm import aliased
from sqlalchemy.sql.expression import desc from sqlalchemy.sql.expression import desc
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
import requests.exceptions
import inspect import inspect
import itertools import itertools
import hashlib import hashlib
@ -489,23 +490,26 @@ def get_labels_db(keys):
missing.add(qid) missing.add(qid)
page_size = 50 page_size = 50
for num, cur in enumerate(utils.chunk(missing, page_size)): try:
for entity in mediawiki.get_entities(cur): for cur in utils.chunk(missing, page_size):
if 'redirects' in entity: for entity in mediawiki.get_entities(cur):
continue if 'redirects' in entity:
continue
qid = entity['id'] qid = entity['id']
modified = datetime.strptime(entity['modified'], "%Y-%m-%dT%H:%M:%SZ") modified = datetime.strptime(entity['modified'], "%Y-%m-%dT%H:%M:%SZ")
# FIXME: check if the item is an artwork and set is_artwork correctly # FIXME: check if the item is an artwork and set is_artwork correctly
item = Item(item_id=qid[1:], item = Item(item_id=qid[1:],
entity=entity, entity=entity,
lastrevid=entity['lastrevid'], lastrevid=entity['lastrevid'],
modified=modified, modified=modified,
is_artwork=False) is_artwork=False)
database.session.add(item) database.session.add(item)
labels[qid] = item.label labels[qid] = item.label
database.session.commit() database.session.commit()
except requests.exceptions.ReadTimeout:
pass
return labels return labels
@ -734,11 +738,15 @@ def get_image_detail_with_cache(items, cache_name, thumbwidth=None, refresh=Fals
thumbwidth = app.config['THUMBWIDTH'] thumbwidth = app.config['THUMBWIDTH']
filename = f'cache/{cache_name}_images.json' filename = f'cache/{cache_name}_images.json'
if not refresh and os.path.exists(filename): cache_exists = os.path.exists(filename)
if not refresh and cache_exists:
detail = json.load(open(filename)) detail = json.load(open(filename))
else: else:
detail = commons.image_detail(filenames, thumbwidth=thumbwidth) try:
json.dump(detail, open(filename, 'w'), indent=2) detail = commons.image_detail(filenames, thumbwidth=thumbwidth)
json.dump(detail, open(filename, 'w'), indent=2)
except requests.exceptions.ReadTimeout:
detail = json.load(open(filename)) if cache_exists else {}
return detail return detail
@ -814,7 +822,7 @@ def get_db_facets(params):
for values in facet_list.values(): for values in facet_list.values():
for v in values: for v in values:
v['label'] = labels[v['qid']] v['label'] = labels.get(v['qid'])
return facet_list return facet_list
@ -861,7 +869,7 @@ def browse_page():
if not cache_refreshed and image_filename not in detail: if not cache_refreshed and image_filename not in detail:
detail = get_image_detail_with_cache(items, cache_name, refresh=True) detail = get_image_detail_with_cache(items, cache_name, refresh=True)
cache_refreshed = True cache_refreshed = True
item.image = detail[image_filename] item.image = detail.get(image_filename)
return render_template('find_more.html', return render_template('find_more.html',
page=page, page=page,