From 93df781115976a9684a96b18bd7097699c0f1eab Mon Sep 17 00:00:00 2001
From: Edward Betts <edward@4angle.com>
Date: Wed, 8 Jan 2020 15:39:25 +0000
Subject: [PATCH] Better handling of read timeout from APIs

---
 app.py | 48 ++++++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/app.py b/app.py
index adb6ebc..494437c 100755
--- a/app.py
+++ b/app.py
@@ -15,6 +15,7 @@ from sqlalchemy.orm import aliased
 from sqlalchemy.sql.expression import desc
 from collections import defaultdict
 from datetime import datetime
+import requests.exceptions
 import inspect
 import itertools
 import hashlib
@@ -489,23 +490,26 @@ def get_labels_db(keys):
             missing.add(qid)
 
     page_size = 50
-    for num, cur in enumerate(utils.chunk(missing, page_size)):
-        for entity in mediawiki.get_entities(cur):
-            if 'redirects' in entity:
-                continue
+    try:
+        for cur in utils.chunk(missing, page_size):
+            for entity in mediawiki.get_entities(cur):
+                if 'redirects' in entity:
+                    continue
 
-            qid = entity['id']
+                qid = entity['id']
 
-            modified = datetime.strptime(entity['modified'], "%Y-%m-%dT%H:%M:%SZ")
-            # FIXME: check if the item is an artwork and set is_artwork correctly
-            item = Item(item_id=qid[1:],
-                        entity=entity,
-                        lastrevid=entity['lastrevid'],
-                        modified=modified,
-                        is_artwork=False)
-            database.session.add(item)
-            labels[qid] = item.label
-        database.session.commit()
+                modified = datetime.strptime(entity['modified'], "%Y-%m-%dT%H:%M:%SZ")
+                # FIXME: check if the item is an artwork and set is_artwork correctly
+                item = Item(item_id=qid[1:],
+                            entity=entity,
+                            lastrevid=entity['lastrevid'],
+                            modified=modified,
+                            is_artwork=False)
+                database.session.add(item)
+                labels[qid] = item.label
+            database.session.commit()
+    except requests.exceptions.ReadTimeout:
+        pass
 
     return labels
 
@@ -734,11 +738,15 @@ def get_image_detail_with_cache(items, cache_name, thumbwidth=None, refresh=Fals
         thumbwidth = app.config['THUMBWIDTH']
 
     filename = f'cache/{cache_name}_images.json'
-    if not refresh and os.path.exists(filename):
+    cache_exists = os.path.exists(filename)
+    if not refresh and cache_exists:
         detail = json.load(open(filename))
     else:
-        detail = commons.image_detail(filenames, thumbwidth=thumbwidth)
-        json.dump(detail, open(filename, 'w'), indent=2)
+        try:
+            detail = commons.image_detail(filenames, thumbwidth=thumbwidth)
+            json.dump(detail, open(filename, 'w'), indent=2)
+        except requests.exceptions.ReadTimeout:
+            detail = json.load(open(filename)) if cache_exists else {}
 
     return detail
 
@@ -814,7 +822,7 @@ def get_db_facets(params):
 
     for values in facet_list.values():
         for v in values:
-            v['label'] = labels[v['qid']]
+            v['label'] = labels.get(v['qid'])
 
     return facet_list
 
@@ -861,7 +869,7 @@ def browse_page():
         if not cache_refreshed and image_filename not in detail:
             detail = get_image_detail_with_cache(items, cache_name, refresh=True)
             cache_refreshed = True
-        item.image = detail[image_filename]
+        item.image = detail.get(image_filename)
 
     return render_template('find_more.html',
                            page=page,