Better handling of read timeout from APIs
This commit is contained in:
parent
9bbbb45fbb
commit
93df781115
16
app.py
16
app.py
|
@ -15,6 +15,7 @@ from sqlalchemy.orm import aliased
|
||||||
from sqlalchemy.sql.expression import desc
|
from sqlalchemy.sql.expression import desc
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
import requests.exceptions
|
||||||
import inspect
|
import inspect
|
||||||
import itertools
|
import itertools
|
||||||
import hashlib
|
import hashlib
|
||||||
|
@ -489,7 +490,8 @@ def get_labels_db(keys):
|
||||||
missing.add(qid)
|
missing.add(qid)
|
||||||
|
|
||||||
page_size = 50
|
page_size = 50
|
||||||
for num, cur in enumerate(utils.chunk(missing, page_size)):
|
try:
|
||||||
|
for cur in utils.chunk(missing, page_size):
|
||||||
for entity in mediawiki.get_entities(cur):
|
for entity in mediawiki.get_entities(cur):
|
||||||
if 'redirects' in entity:
|
if 'redirects' in entity:
|
||||||
continue
|
continue
|
||||||
|
@ -506,6 +508,8 @@ def get_labels_db(keys):
|
||||||
database.session.add(item)
|
database.session.add(item)
|
||||||
labels[qid] = item.label
|
labels[qid] = item.label
|
||||||
database.session.commit()
|
database.session.commit()
|
||||||
|
except requests.exceptions.ReadTimeout:
|
||||||
|
pass
|
||||||
|
|
||||||
return labels
|
return labels
|
||||||
|
|
||||||
|
@ -734,11 +738,15 @@ def get_image_detail_with_cache(items, cache_name, thumbwidth=None, refresh=Fals
|
||||||
thumbwidth = app.config['THUMBWIDTH']
|
thumbwidth = app.config['THUMBWIDTH']
|
||||||
|
|
||||||
filename = f'cache/{cache_name}_images.json'
|
filename = f'cache/{cache_name}_images.json'
|
||||||
if not refresh and os.path.exists(filename):
|
cache_exists = os.path.exists(filename)
|
||||||
|
if not refresh and cache_exists:
|
||||||
detail = json.load(open(filename))
|
detail = json.load(open(filename))
|
||||||
else:
|
else:
|
||||||
|
try:
|
||||||
detail = commons.image_detail(filenames, thumbwidth=thumbwidth)
|
detail = commons.image_detail(filenames, thumbwidth=thumbwidth)
|
||||||
json.dump(detail, open(filename, 'w'), indent=2)
|
json.dump(detail, open(filename, 'w'), indent=2)
|
||||||
|
except requests.exceptions.ReadTimeout:
|
||||||
|
detail = json.load(open(filename)) if cache_exists else {}
|
||||||
|
|
||||||
return detail
|
return detail
|
||||||
|
|
||||||
|
@ -814,7 +822,7 @@ def get_db_facets(params):
|
||||||
|
|
||||||
for values in facet_list.values():
|
for values in facet_list.values():
|
||||||
for v in values:
|
for v in values:
|
||||||
v['label'] = labels[v['qid']]
|
v['label'] = labels.get(v['qid'])
|
||||||
|
|
||||||
return facet_list
|
return facet_list
|
||||||
|
|
||||||
|
@ -861,7 +869,7 @@ def browse_page():
|
||||||
if not cache_refreshed and image_filename not in detail:
|
if not cache_refreshed and image_filename not in detail:
|
||||||
detail = get_image_detail_with_cache(items, cache_name, refresh=True)
|
detail = get_image_detail_with_cache(items, cache_name, refresh=True)
|
||||||
cache_refreshed = True
|
cache_refreshed = True
|
||||||
item.image = detail[image_filename]
|
item.image = detail.get(image_filename)
|
||||||
|
|
||||||
return render_template('find_more.html',
|
return render_template('find_more.html',
|
||||||
page=page,
|
page=page,
|
||||||
|
|
Loading…
Reference in a new issue