Various improvements and bug fixes

This commit is contained in:
Edward Betts 2021-06-15 00:09:05 +02:00
parent d407224f35
commit 6567a554d6

View file

@ -170,12 +170,14 @@ def get_item_street_addresses(item):
def get_markers(all_items): def get_markers(all_items):
items = [] items = []
for item in all_items: for item in all_items:
if not item:
continue
locations = [list(i.get_lat_lon()) for i in item.locations] locations = [list(i.get_lat_lon()) for i in item.locations]
image_filenames = item.get_claim("P18") image_filenames = item.get_claim("P18")
street_address = get_item_street_addresses(item) street_address = get_item_street_addresses(item)
item = { d = {
"qid": item.qid, "qid": item.qid,
"label": item.label(), "label": item.label(),
"description": item.description(), "description": item.description(),
@ -184,7 +186,11 @@ def get_markers(all_items):
"street_address": street_address, "street_address": street_address,
"isa_list": [v["id"] for v in item.get_claim("P31") if v], "isa_list": [v["id"] for v in item.get_claim("P31") if v],
} }
items.append(item)
if aliases := item.get_aliases():
d["aliases"] = aliases
items.append(d)
return items return items
@ -345,6 +351,8 @@ def search_page():
def get_isa_count(items): def get_isa_count(items):
isa_count = Counter() isa_count = Counter()
for item in items: for item in items:
if not item:
continue
isa_list = item.get_claim("P31") isa_list = item.get_claim("P31")
for isa in isa_list: for isa in isa_list:
if not isa: if not isa:
@ -357,9 +365,11 @@ def get_isa_count(items):
def get_and_save_item(qid): def get_and_save_item(qid):
entity = wikidata_api.get_entity(qid) entity = wikidata_api.get_entity(qid)
if entity["id"] != qid: entity_qid = entity["id"]
print(f'redirect {qid} -> {entity["id"]}') if entity_qid != qid:
return print(f'redirect {qid} -> {entity_qid}')
item = model.Item.query.get(entity_qid[1:])
return item
coords = wikidata.get_entity_coords(entity["claims"]) coords = wikidata.get_entity_coords(entity["claims"])
@ -417,7 +427,9 @@ def api_wikidata_items():
t1 = time() - t0 t1 = time() - t0
print(f"wikidata: {t1} seconds") print(f"wikidata: {t1} seconds")
return jsonify(success=True, items=items, isa_count=isa_count, duration=t1) response = jsonify(success=True, items=items, isa_count=isa_count, duration=t1)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
@app.route("/api/1/osm") @app.route("/api/1/osm")
@ -427,11 +439,13 @@ def api_osm_objects():
objects = get_osm_with_wikidata_tag(bounds) objects = get_osm_with_wikidata_tag(bounds)
t1 = time() - t0 t1 = time() - t0
print(f"OSM: {t1} seconds") print(f"OSM: {t1} seconds")
return jsonify(success=True, objects=objects, duration=t1) response = jsonify(success=True, objects=objects, duration=t1)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
edu = ['Tag:amenity=college', 'Tag:amenity=university', 'Tag:amenity=school', edu = ['Tag:amenity=college', 'Tag:amenity=university', 'Tag:amenity=school',
'Tag:office=educational_institution'] 'Tag:office=educational_institution', 'Tag:building=university']
tall = ['Key:height', 'Key:building:levels'] tall = ['Key:height', 'Key:building:levels']
extra_keys = { extra_keys = {
@ -449,6 +463,7 @@ extra_keys = {
'Q5167149': edu, # cooking school 'Q5167149': edu, # cooking school
'Q7894959': edu, # University Technical College 'Q7894959': edu, # University Technical College
'Q47530379': edu, # agricultural college 'Q47530379': edu, # agricultural college
'Q38723': edu, # higher education institution
'Q11303': tall, # skyscraper 'Q11303': tall, # skyscraper
'Q18142': tall, # high-rise building 'Q18142': tall, # high-rise building
'Q33673393': tall, # multi-storey building 'Q33673393': tall, # multi-storey building
@ -591,6 +606,10 @@ extra_keys = {
'Tag:landuse=recreation_ground'], 'Tag:landuse=recreation_ground'],
'Q4830453': ['Key:office', 'Q4830453': ['Key:office',
'Tag:building=office'], # business 'Tag:building=office'], # business
'Q43229': ['Key:office',
'Tag:building=office'], # organization
'Q17084016': ['Tag:office=association',
'Tag:office=ngo'], # nonprofit corporation
} }
skip_isa = { skip_isa = {
@ -664,7 +683,6 @@ skip_isa = {
3563237, # economic unit 3563237, # economic unit
2198779, # unit 2198779, # unit
7184903, # abstract object 7184903, # abstract object
43229, # organization
16334295, # group of humans 16334295, # group of humans
16334298, # group of living things 16334298, # group of living things
61961344, # group of physical objects 61961344, # group of physical objects
@ -727,6 +745,7 @@ skip_isa = {
19603939, # building component 19603939, # building component
55638, # tertiary sector of the economy 55638, # tertiary sector of the economy
3958441, # economic sector 3958441, # economic sector
7406919, # service
} }
skip_tags = {"Key:addr:street"} skip_tags = {"Key:addr:street"}
@ -782,8 +801,10 @@ def api_get_item_tags(item_id):
item = model.Item.query.get(item_id) item = model.Item.query.get(item_id)
osm_list = get_item_tags(item) osm_list = get_item_tags(item)
t1 = time() - t0 t1 = time() - t0
return jsonify(success=True, qid=item.qid, tag_or_key_list=osm_list, duration=t1)
response = jsonify(success=True, qid=item.qid, tag_or_key_list=osm_list, duration=t1)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
def get_tag_filter(cls, tag_list): def get_tag_filter(cls, tag_list):
tag_filter = [] tag_filter = []
@ -827,10 +848,15 @@ def get_nearby(bbox, item, max_distance=200):
if not tag_list: if not tag_list:
return [] return []
item_is_street = item.is_street()
for loc in item.locations: for loc in item.locations:
lat, lon = loc.get_lat_lon() lat, lon = loc.get_lat_lon()
point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326) point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326)
for cls in model.Point, model.Line, model.Polygon: for cls in model.Point, model.Line, model.Polygon:
if item_is_street and cls == model.Point:
continue
tag_filter = get_tag_filter(cls, tag_list) tag_filter = get_tag_filter(cls, tag_list)
dist = func.ST_Distance(point, cls.way.cast(Geography(srid=4326))) dist = func.ST_Distance(point, cls.way.cast(Geography(srid=4326)))
@ -841,6 +867,10 @@ def get_nearby(bbox, item, max_distance=200):
or_(*tag_filter)) or_(*tag_filter))
.order_by(point.distance_centroid(cls.way))) .order_by(point.distance_centroid(cls.way)))
if item_is_street:
q = q.filter(cls.tags.has_key("name"),
cls.tags["highway"] != 'bus_stop')
if "Key:amenity" in tag_list: if "Key:amenity" in tag_list:
q = q.filter(cls.tags["amenity"] != "bicycle_parking", q = q.filter(cls.tags["amenity"] != "bicycle_parking",
cls.tags["amenity"] != "bicycle_repair_station", cls.tags["amenity"] != "bicycle_repair_station",
@ -960,7 +990,9 @@ def api_find_osm_candidates(item_id):
nearby.append(cur) nearby.append(cur)
t1 = time() - t0 t1 = time() - t0
return jsonify(success=True, qid=item.qid, nearby=nearby, duration=t1) response = jsonify(success=True, qid=item.qid, nearby=nearby, duration=t1)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
@app.route("/api/1/missing") @app.route("/api/1/missing")
@ -1000,7 +1032,9 @@ def api_missing_wikidata_items():
} }
isa_count.append(isa) isa_count.append(isa)
return jsonify(success=True, items=items, isa_count=isa_count) response = jsonify(success=True, items=items, isa_count=isa_count)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
@app.route("/api/1/search") @app.route("/api/1/search")
@ -1011,8 +1045,32 @@ def api_search():
if "geotext" in hit: if "geotext" in hit:
del hit["geotext"] del hit["geotext"]
hit["name"] = nominatim.get_hit_name(hit) hit["name"] = nominatim.get_hit_name(hit)
hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}"
return jsonify(hits=hits) response = jsonify(hits=hits)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
@app.route("/refresh/Q<int:item_id>")
def refresh_item(item_id):
assert not model.Item.query.get(item_id)
qid = f'Q{item_id}'
entity = wikidata_api.get_entity(qid)
entity_qid = entity.pop("id")
assert qid == entity_qid
coords = wikidata.get_entity_coords(entity["claims"])
assert coords
obj = {k: v for k, v in entity.items() if k in entity_keys}
item = model.Item(item_id=item_id, **obj)
print(item)
item.locations = model.location_objects(coords)
database.session.add(item)
database.session.commit()
return 'done'
if __name__ == "__main__": if __name__ == "__main__":