From 253c75f46cff55eec505dd4809a796259f1ef5d0 Mon Sep 17 00:00:00 2001
From: Edward Betts <edward@4angle.com>
Date: Mon, 10 May 2021 14:59:08 +0200
Subject: [PATCH] Load missing wikidata items

---
 static/js/map.js | 79 ++++++++++++++++++++++++++++++++++++++----------
 web_view.py      | 37 +++++++++++++++++++++++
 2 files changed, 100 insertions(+), 16 deletions(-)

diff --git a/static/js/map.js b/static/js/map.js
index 69c46b8..d3c34c1 100644
--- a/static/js/map.js
+++ b/static/js/map.js
@@ -22,6 +22,11 @@ var load_text = document.getElementById("load-text");
 var isa_card = document.getElementById("isa-card");
 var checkbox_list = document.getElementsByClassName("isa-checkbox");
 var isa_labels = {};
+var items_url = "/api/1/items";
+var osm_objects_url = "/api/1/osm";
+var missing_url = "/api/1/missing";
+
+var isa_count = {};
 
 map.zoomControl.setPosition("topright");
 
@@ -95,7 +100,7 @@ function add_to_feature_group(qid, thing) {
   return group;
 }
 
-function update_wikidata() {
+function update_wikidata(check_for_missing = true) {
   if (
     Object.keys(wikidata_items).length === 0 ||
     Object.keys(osm_objects).length === 0
@@ -104,6 +109,34 @@ function update_wikidata() {
     return;
   }
 
+  if (check_for_missing) {
+    var missing_qids = [];
+    for (const qid in osm_objects) {
+      var item = wikidata_items[qid];
+      if (!item) missing_qids.push(qid);
+    }
+    console.log(missing_qids);
+
+    if (missing_qids.length) {
+      var params = { qids: missing_qids.join(",") };
+      axios.get(missing_url, { params: params }).then((response) => {
+        console.log(response.data.items);
+
+        response.data.isa_count.forEach((isa) => {
+          isa_labels[isa.qid] = isa.label;
+          if (isa_count[isa.qid] === undefined) {
+            isa_count[isa.qid] = isa;
+          } else {
+            isa_count[isa.qid].count += 1;
+          }
+        });
+
+        process_wikidata_items(response.data.items);
+        update_wikidata(false);
+      });
+    }
+  }
+
   for (const qid in osm_objects) {
     var osm_list = osm_objects[qid];
 
@@ -132,6 +165,13 @@ function update_wikidata() {
     });
   }
 
+  var isa_count_values = Object.values(isa_count);
+  isa_count_values.sort((a, b) => {
+    b.count - a.count;
+  });
+  console.log(isa_count_values);
+
+  set_isa_list(isa_count_values);
   load_complete();
 }
 
@@ -185,7 +225,6 @@ function set_isa_list(isa_count) {
   var isa_list = document.getElementById("isa-list");
   isa_list.innerHTML = "";
   isa_count.forEach((isa) => {
-    isa_labels[isa.qid] = isa.label;
     var isa_id = `isa-${isa.qid}`;
     var e = document.createElement("div");
     e.setAttribute("class", "isa-item");
@@ -251,6 +290,20 @@ function add_wikidata_marker(item, marker_data) {
   marker_data.marker = marker;
 }
 
+function process_wikidata_items(load_items) {
+  load_items.forEach((item) => {
+    var qid = item.qid;
+    if (item.qid in wikidata_items) return;
+    item.markers.forEach((marker_data) =>
+      add_wikidata_marker(item, marker_data)
+    );
+    wikidata_items[item.qid] = item;
+
+    if (items[qid] === undefined) items[qid] = {};
+    items[qid].isa_list = item.isa_list;
+  });
+}
+
 function load_wikidata_items() {
   var checkbox_list = document.getElementsByClassName("isa-checkbox");
 
@@ -265,28 +318,22 @@ function load_wikidata_items() {
   console.log("map moved", bounds.toBBoxString());
 
   var params = { bounds: bounds.toBBoxString() };
-  var items_url = "/api/1/items";
 
   axios.get(items_url, { params: params }).then((response) => {
-    set_isa_list(response.data.isa_count);
-    var load_items = response.data.items;
-    load_items.forEach((item) => {
-      var qid = item.qid;
-      if (item.qid in wikidata_items) return;
-      item.markers.forEach((marker_data) =>
-        add_wikidata_marker(item, marker_data)
-      );
-      wikidata_items[item.qid] = item;
-
-      if (items[qid] === undefined) items[qid] = {};
-      items[qid].isa_list = item.isa_list;
+    response.data.isa_count.forEach((isa) => {
+      isa_count[isa.qid] = isa;
+      isa_labels[isa.qid] = isa.label;
     });
 
+    // set_isa_list(response.data.isa_count);
+
+    process_wikidata_items(response.data.items);
+
     wikidata_loaded = true;
     isa_card.classList.remove("visually-hidden");
     update_wikidata();
   });
-  var osm_objects_url = "/api/1/osm";
+
   axios.get(osm_objects_url, { params: params }).then((response) => {
     console.log(`${response.data.duration} seconds`);
     response.data.objects.forEach((osm) => {
diff --git a/web_view.py b/web_view.py
index d69c541..6819dc5 100755
--- a/web_view.py
+++ b/web_view.py
@@ -373,6 +373,43 @@ def api_osm_objects():
     return jsonify(success=True, objects=objects, duration=t1)
 
 
+@app.route("/api/1/missing")
+def api_missing_wikidata_items():
+    qids_arg = request.args.get("qids")
+    qids = qids_arg.split(",")
+    if not qids or not qids[0]:
+        return jsonify(success=True, items=[], isa_count=[])
+
+    db_items = []
+    for qid in qids:
+        item = model.Item.query.get(qid[1:])
+        if not item:
+            item = get_and_save_item(qid)
+        db_items.append(item)
+    items = get_markers(db_items)
+    counts = get_isa_count(db_items)
+    isa_ids = [qid[1:] for qid, count in counts]
+    isa_items = {
+        isa.qid: isa for isa in model.Item.query.filter(model.Item.item_id.in_(isa_ids))
+    }
+
+    isa_count = []
+    for qid, count in counts:
+        item = isa_items.get(qid)
+        if not item:
+            item = get_and_save_item(qid)
+
+        label = item.label() if item else "[missing]"
+        isa = {
+            "qid": qid,
+            "count": count,
+            "label": label,
+        }
+        isa_count.append(isa)
+
+    return jsonify(success=True, items=items, isa_count=isa_count)
+
+
 @app.route("/api/1/search")
 def api_search():
     q = request.args["q"]