From 253c75f46cff55eec505dd4809a796259f1ef5d0 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Mon, 10 May 2021 14:59:08 +0200 Subject: [PATCH] Load missing wikidata items --- static/js/map.js | 79 ++++++++++++++++++++++++++++++++++++++---------- web_view.py | 37 +++++++++++++++++++++++ 2 files changed, 100 insertions(+), 16 deletions(-) diff --git a/static/js/map.js b/static/js/map.js index 69c46b8..d3c34c1 100644 --- a/static/js/map.js +++ b/static/js/map.js @@ -22,6 +22,11 @@ var load_text = document.getElementById("load-text"); var isa_card = document.getElementById("isa-card"); var checkbox_list = document.getElementsByClassName("isa-checkbox"); var isa_labels = {}; +var items_url = "/api/1/items"; +var osm_objects_url = "/api/1/osm"; +var missing_url = "/api/1/missing"; + +var isa_count = {}; map.zoomControl.setPosition("topright"); @@ -95,7 +100,7 @@ function add_to_feature_group(qid, thing) { return group; } -function update_wikidata() { +function update_wikidata(check_for_missing = true) { if ( Object.keys(wikidata_items).length === 0 || Object.keys(osm_objects).length === 0 @@ -104,6 +109,34 @@ function update_wikidata() { return; } + if (check_for_missing) { + var missing_qids = []; + for (const qid in osm_objects) { + var item = wikidata_items[qid]; + if (!item) missing_qids.push(qid); + } + console.log(missing_qids); + + if (missing_qids.length) { + var params = { qids: missing_qids.join(",") }; + axios.get(missing_url, { params: params }).then((response) => { + console.log(response.data.items); + + response.data.isa_count.forEach((isa) => { + isa_labels[isa.qid] = isa.label; + if (isa_count[isa.qid] === undefined) { + isa_count[isa.qid] = isa; + } else { + isa_count[isa.qid].count += 1; + } + }); + + process_wikidata_items(response.data.items); + update_wikidata(false); + }); + } + } + for (const qid in osm_objects) { var osm_list = osm_objects[qid]; @@ -132,6 +165,13 @@ function update_wikidata() { }); } + var isa_count_values = Object.values(isa_count); + isa_count_values.sort((a, b) => { + b.count - a.count; + }); + console.log(isa_count_values); + + set_isa_list(isa_count_values); load_complete(); } @@ -185,7 +225,6 @@ function set_isa_list(isa_count) { var isa_list = document.getElementById("isa-list"); isa_list.innerHTML = ""; isa_count.forEach((isa) => { - isa_labels[isa.qid] = isa.label; var isa_id = `isa-${isa.qid}`; var e = document.createElement("div"); e.setAttribute("class", "isa-item"); @@ -251,6 +290,20 @@ function add_wikidata_marker(item, marker_data) { marker_data.marker = marker; } +function process_wikidata_items(load_items) { + load_items.forEach((item) => { + var qid = item.qid; + if (item.qid in wikidata_items) return; + item.markers.forEach((marker_data) => + add_wikidata_marker(item, marker_data) + ); + wikidata_items[item.qid] = item; + + if (items[qid] === undefined) items[qid] = {}; + items[qid].isa_list = item.isa_list; + }); +} + function load_wikidata_items() { var checkbox_list = document.getElementsByClassName("isa-checkbox"); @@ -265,28 +318,22 @@ function load_wikidata_items() { console.log("map moved", bounds.toBBoxString()); var params = { bounds: bounds.toBBoxString() }; - var items_url = "/api/1/items"; axios.get(items_url, { params: params }).then((response) => { - set_isa_list(response.data.isa_count); - var load_items = response.data.items; - load_items.forEach((item) => { - var qid = item.qid; - if (item.qid in wikidata_items) return; - item.markers.forEach((marker_data) => - add_wikidata_marker(item, marker_data) - ); - wikidata_items[item.qid] = item; - - if (items[qid] === undefined) items[qid] = {}; - items[qid].isa_list = item.isa_list; + response.data.isa_count.forEach((isa) => { + isa_count[isa.qid] = isa; + isa_labels[isa.qid] = isa.label; }); + // set_isa_list(response.data.isa_count); + + process_wikidata_items(response.data.items); + wikidata_loaded = true; isa_card.classList.remove("visually-hidden"); update_wikidata(); }); - var osm_objects_url = "/api/1/osm"; + axios.get(osm_objects_url, { params: params }).then((response) => { console.log(`${response.data.duration} seconds`); response.data.objects.forEach((osm) => { diff --git a/web_view.py b/web_view.py index d69c541..6819dc5 100755 --- a/web_view.py +++ b/web_view.py @@ -373,6 +373,43 @@ def api_osm_objects(): return jsonify(success=True, objects=objects, duration=t1) +@app.route("/api/1/missing") +def api_missing_wikidata_items(): + qids_arg = request.args.get("qids") + qids = qids_arg.split(",") + if not qids or not qids[0]: + return jsonify(success=True, items=[], isa_count=[]) + + db_items = [] + for qid in qids: + item = model.Item.query.get(qid[1:]) + if not item: + item = get_and_save_item(qid) + db_items.append(item) + items = get_markers(db_items) + counts = get_isa_count(db_items) + isa_ids = [qid[1:] for qid, count in counts] + isa_items = { + isa.qid: isa for isa in model.Item.query.filter(model.Item.item_id.in_(isa_ids)) + } + + isa_count = [] + for qid, count in counts: + item = isa_items.get(qid) + if not item: + item = get_and_save_item(qid) + + label = item.label() if item else "[missing]" + isa = { + "qid": qid, + "count": count, + "label": label, + } + isa_count.append(isa) + + return jsonify(success=True, items=items, isa_count=isa_count) + + @app.route("/api/1/search") def api_search(): q = request.args["q"]