Load missing wikidata items

This commit is contained in:
Edward Betts 2021-05-10 14:59:08 +02:00
parent 70dad3cdac
commit 253c75f46c
2 changed files with 100 additions and 16 deletions

View file

@ -22,6 +22,11 @@ var load_text = document.getElementById("load-text");
var isa_card = document.getElementById("isa-card"); var isa_card = document.getElementById("isa-card");
var checkbox_list = document.getElementsByClassName("isa-checkbox"); var checkbox_list = document.getElementsByClassName("isa-checkbox");
var isa_labels = {}; var isa_labels = {};
var items_url = "/api/1/items";
var osm_objects_url = "/api/1/osm";
var missing_url = "/api/1/missing";
var isa_count = {};
map.zoomControl.setPosition("topright"); map.zoomControl.setPosition("topright");
@ -95,7 +100,7 @@ function add_to_feature_group(qid, thing) {
return group; return group;
} }
function update_wikidata() { function update_wikidata(check_for_missing = true) {
if ( if (
Object.keys(wikidata_items).length === 0 || Object.keys(wikidata_items).length === 0 ||
Object.keys(osm_objects).length === 0 Object.keys(osm_objects).length === 0
@ -104,6 +109,34 @@ function update_wikidata() {
return; return;
} }
if (check_for_missing) {
var missing_qids = [];
for (const qid in osm_objects) {
var item = wikidata_items[qid];
if (!item) missing_qids.push(qid);
}
console.log(missing_qids);
if (missing_qids.length) {
var params = { qids: missing_qids.join(",") };
axios.get(missing_url, { params: params }).then((response) => {
console.log(response.data.items);
response.data.isa_count.forEach((isa) => {
isa_labels[isa.qid] = isa.label;
if (isa_count[isa.qid] === undefined) {
isa_count[isa.qid] = isa;
} else {
isa_count[isa.qid].count += 1;
}
});
process_wikidata_items(response.data.items);
update_wikidata(false);
});
}
}
for (const qid in osm_objects) { for (const qid in osm_objects) {
var osm_list = osm_objects[qid]; var osm_list = osm_objects[qid];
@ -132,6 +165,13 @@ function update_wikidata() {
}); });
} }
var isa_count_values = Object.values(isa_count);
isa_count_values.sort((a, b) => {
b.count - a.count;
});
console.log(isa_count_values);
set_isa_list(isa_count_values);
load_complete(); load_complete();
} }
@ -185,7 +225,6 @@ function set_isa_list(isa_count) {
var isa_list = document.getElementById("isa-list"); var isa_list = document.getElementById("isa-list");
isa_list.innerHTML = ""; isa_list.innerHTML = "";
isa_count.forEach((isa) => { isa_count.forEach((isa) => {
isa_labels[isa.qid] = isa.label;
var isa_id = `isa-${isa.qid}`; var isa_id = `isa-${isa.qid}`;
var e = document.createElement("div"); var e = document.createElement("div");
e.setAttribute("class", "isa-item"); e.setAttribute("class", "isa-item");
@ -251,6 +290,20 @@ function add_wikidata_marker(item, marker_data) {
marker_data.marker = marker; marker_data.marker = marker;
} }
function process_wikidata_items(load_items) {
load_items.forEach((item) => {
var qid = item.qid;
if (item.qid in wikidata_items) return;
item.markers.forEach((marker_data) =>
add_wikidata_marker(item, marker_data)
);
wikidata_items[item.qid] = item;
if (items[qid] === undefined) items[qid] = {};
items[qid].isa_list = item.isa_list;
});
}
function load_wikidata_items() { function load_wikidata_items() {
var checkbox_list = document.getElementsByClassName("isa-checkbox"); var checkbox_list = document.getElementsByClassName("isa-checkbox");
@ -265,28 +318,22 @@ function load_wikidata_items() {
console.log("map moved", bounds.toBBoxString()); console.log("map moved", bounds.toBBoxString());
var params = { bounds: bounds.toBBoxString() }; var params = { bounds: bounds.toBBoxString() };
var items_url = "/api/1/items";
axios.get(items_url, { params: params }).then((response) => { axios.get(items_url, { params: params }).then((response) => {
set_isa_list(response.data.isa_count); response.data.isa_count.forEach((isa) => {
var load_items = response.data.items; isa_count[isa.qid] = isa;
load_items.forEach((item) => { isa_labels[isa.qid] = isa.label;
var qid = item.qid;
if (item.qid in wikidata_items) return;
item.markers.forEach((marker_data) =>
add_wikidata_marker(item, marker_data)
);
wikidata_items[item.qid] = item;
if (items[qid] === undefined) items[qid] = {};
items[qid].isa_list = item.isa_list;
}); });
// set_isa_list(response.data.isa_count);
process_wikidata_items(response.data.items);
wikidata_loaded = true; wikidata_loaded = true;
isa_card.classList.remove("visually-hidden"); isa_card.classList.remove("visually-hidden");
update_wikidata(); update_wikidata();
}); });
var osm_objects_url = "/api/1/osm";
axios.get(osm_objects_url, { params: params }).then((response) => { axios.get(osm_objects_url, { params: params }).then((response) => {
console.log(`${response.data.duration} seconds`); console.log(`${response.data.duration} seconds`);
response.data.objects.forEach((osm) => { response.data.objects.forEach((osm) => {

View file

@ -373,6 +373,43 @@ def api_osm_objects():
return jsonify(success=True, objects=objects, duration=t1) return jsonify(success=True, objects=objects, duration=t1)
@app.route("/api/1/missing")
def api_missing_wikidata_items():
qids_arg = request.args.get("qids")
qids = qids_arg.split(",")
if not qids or not qids[0]:
return jsonify(success=True, items=[], isa_count=[])
db_items = []
for qid in qids:
item = model.Item.query.get(qid[1:])
if not item:
item = get_and_save_item(qid)
db_items.append(item)
items = get_markers(db_items)
counts = get_isa_count(db_items)
isa_ids = [qid[1:] for qid, count in counts]
isa_items = {
isa.qid: isa for isa in model.Item.query.filter(model.Item.item_id.in_(isa_ids))
}
isa_count = []
for qid, count in counts:
item = isa_items.get(qid)
if not item:
item = get_and_save_item(qid)
label = item.label() if item else "[missing]"
isa = {
"qid": qid,
"count": count,
"label": label,
}
isa_count.append(isa)
return jsonify(success=True, items=items, isa_count=isa_count)
@app.route("/api/1/search") @app.route("/api/1/search")
def api_search(): def api_search():
q = request.args["q"] q = request.args["q"]