edward/depicts
1
0
Fork 0
Code Issues 8 Pull requests Packages Projects Releases Wiki Activity

Better handling of read timeout from APIs

Browse source
This commit is contained in:
Edward Betts 2020-01-08 15:39:25 +00:00
parent 9bbbb45fbb
commit 93df781115
1 changed files with 28 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
app.py
View file

@ -15,6 +15,7 @@ from sqlalchemy.orm import aliased
from sqlalchemy.sql.expression import desc from sqlalchemy.sql.expression import desc
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
import requests.exceptions
import inspect import inspect
import itertools import itertools
import hashlib import hashlib
@ -489,23 +490,26 @@ def get_labels_db(keys):
missing.add(qid) missing.add(qid)
page_size = 50 page_size = 50
for num, cur in enumerate(utils.chunk(missing, page_size)): try:
for entity in mediawiki.get_entities(cur): for cur in utils.chunk(missing, page_size):
if 'redirects' in entity: for entity in mediawiki.get_entities(cur):
continue if 'redirects' in entity:
continue
qid = entity['id'] qid = entity['id']
modified = datetime.strptime(entity['modified'], "%Y-%m-%dT%H:%M:%SZ") modified = datetime.strptime(entity['modified'], "%Y-%m-%dT%H:%M:%SZ")
# FIXME: check if the item is an artwork and set is_artwork correctly # FIXME: check if the item is an artwork and set is_artwork correctly
item = Item(item_id=qid[1:], item = Item(item_id=qid[1:],
entity=entity, entity=entity,
lastrevid=entity['lastrevid'], lastrevid=entity['lastrevid'],
modified=modified, modified=modified,
is_artwork=False) is_artwork=False)
database.session.add(item) database.session.add(item)
labels[qid] = item.label labels[qid] = item.label
database.session.commit() database.session.commit()
except requests.exceptions.ReadTimeout:
pass
return labels return labels
@ -734,11 +738,15 @@ def get_image_detail_with_cache(items, cache_name, thumbwidth=None, refresh=Fals
thumbwidth = app.config['THUMBWIDTH'] thumbwidth = app.config['THUMBWIDTH']
filename = f'cache/{cache_name}_images.json' filename = f'cache/{cache_name}_images.json'
if not refresh and os.path.exists(filename): cache_exists = os.path.exists(filename)
if not refresh and cache_exists:
detail = json.load(open(filename)) detail = json.load(open(filename))
else: else:
detail = commons.image_detail(filenames, thumbwidth=thumbwidth) try:
json.dump(detail, open(filename, 'w'), indent=2) detail = commons.image_detail(filenames, thumbwidth=thumbwidth)
json.dump(detail, open(filename, 'w'), indent=2)
except requests.exceptions.ReadTimeout:
detail = json.load(open(filename)) if cache_exists else {}
return detail return detail
@ -814,7 +822,7 @@ def get_db_facets(params):
for values in facet_list.values(): for values in facet_list.values():
for v in values: for v in values:
v['label'] = labels[v['qid']] v['label'] = labels.get(v['qid'])
return facet_list return facet_list
@ -861,7 +869,7 @@ def browse_page():
if not cache_refreshed and image_filename not in detail: if not cache_refreshed and image_filename not in detail:
detail = get_image_detail_with_cache(items, cache_name, refresh=True) detail = get_image_detail_with_cache(items, cache_name, refresh=True)
cache_refreshed = True cache_refreshed = True
item.image = detail[image_filename] item.image = detail.get(image_filename)
return render_template('find_more.html', return render_template('find_more.html',
page=page, page=page,