#!/usr/bin/python3 from flask import (Flask, render_template, request, jsonify, redirect, url_for, g, flash, session) from sqlalchemy import func, or_ from sqlalchemy.orm import selectinload from matcher import nominatim, model, database, commons, wikidata, wikidata_api, osm_oauth from collections import Counter from time import time from geoalchemy2 import Geography from requests_oauthlib import OAuth1Session import flask_login import os import json import GeoIP import re srid = 4326 re_point = re.compile(r'^POINT\((.+) (.+)\)$') app = Flask(__name__) app.debug = True app.config.from_object('config.default') login_manager = flask_login.LoginManager(app) login_manager.login_view = 'login_route' osm_api_base = 'https://api.openstreetmap.org/api/0.6' DB_URL = "postgresql:///matcher" database.init_db(DB_URL) entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"} re_qid = re.compile(r'^Q\d+$') property_map = [ ("P238", ["iata"], "IATA airport code"), ("P239", ["icao"], "ICAO airport code"), ("P240", ["faa", "ref"], "FAA airport code"), # ('P281', ['addr:postcode', 'postal_code'], 'postal code'), ("P296", ["ref", "ref:train", "railway:ref"], "station code"), ("P300", ["ISO3166-2"], "ISO 3166-2 code"), ("P359", ["ref:rce"], "Rijksmonument ID"), ("P590", ["ref:gnis", "GNISID", "gnis:id", "gnis:feature_id"], "USGS GNIS ID"), ("P649", ["ref:nrhp"], "NRHP reference number"), ("P722", ["uic_ref"], "UIC station code"), ("P782", ["ref"], "LAU (local administrative unit)"), ("P836", ["ref:gss"], "UK Government Statistical Service code"), ("P856", ["website", "contact:website", "url"], "website"), ("P882", ["nist:fips_code"], "FIPS 6-4 (US counties)"), ("P901", ["ref:fips"], "FIPS 10-4 (countries and regions)"), # A UIC id can be a IBNR, but not every IBNR is an UIC id ("P954", ["uic_ref"], "IBNR ID"), ("P981", ["ref:woonplaatscode"], "BAG code for Dutch residencies"), ("P1216", ["HE_ref"], "National Heritage List for England number"), ("P2253", ["ref:edubase"], "EDUBase URN"), ("P2815", ["esr:user", "ref", "ref:train"], "ESR station code"), ("P3425", ["ref", "ref:SIC"], "Natura 2000 site ID"), ("P3562", ["seamark:light:reference"], "Admiralty number"), ( "P4755", ["ref", "ref:train", "ref:crs", "crs", "nat_ref"], "UK railway station code", ), ("P4803", ["ref", "ref:train"], "Amtrak station code"), ("P6082", ["nycdoitt:bin"], "NYC Building Identification Number"), ("P5086", ["ref"], "FIPS 5-2 alpha code (US states)"), ("P5087", ["ref:fips"], "FIPS 5-2 numeric code (US states)"), ("P5208", ["ref:bag"], "BAG building ID for Dutch buildings"), ] @app.teardown_appcontext def shutdown_session(exception=None): database.session.remove() def check_for_tagged_qids(qids): tagged = set() for qid in qids: for cls in model.Point, model.Polygon, model.Line: q = cls.query.filter(cls.tags["wikidata"] == qid) if q.count(): tagged.add(qid) break return tagged def check_for_tagged_qid(qid): return any( database.session.query( cls.query.filter( cls.tags.has_key("wikidata"), cls.tags["wikidata"] == qid ).exists() ).scalar() for cls in (model.Point, model.Polygon, model.Line) ) def make_envelope(bbox): west, south, east, north = [float(i) for i in bbox.split(",")] return func.ST_MakeEnvelope(west, south, east, north, srid) def get_osm_with_wikidata_tag(bbox): db_bbox = make_envelope(bbox) tagged = [] seen = set() for cls in (model.Point, model.Polygon, model.Line): q = cls.query.filter( cls.tags.has_key("wikidata"), func.ST_Intersects(db_bbox, cls.way), func.ST_Area(cls.way) < 20 * func.ST_Area(db_bbox), ) for osm in q: if osm.identifier in seen: continue seen.add(osm.identifier) name = osm.name if not name: if "addr:housename" in osm.tags: name = osm.tags["addr:housename"] else: name = "[no label]" tagged.append( { "identifier": osm.identifier, "id": osm.id, "type": osm.type, "url": osm.osm_url, "geojson": osm.geojson(), "centroid": list(osm.get_centroid()), "name": name, "wikidata": osm.tags["wikidata"], } ) return tagged def get_items_in_bbox(bbox): db_bbox = make_envelope(bbox) q = ( model.Item.query.join(model.ItemLocation) .filter(func.ST_Covers(db_bbox, model.ItemLocation.location)) .options(selectinload(model.Item.locations)) ) return q def get_item_street_addresses(item): street_address = [addr["text"] for addr in item.get_claim("P6375")] if street_address or "P669" not in item.claims: return street_address for claim in item.claims["P669"]: qualifiers = claim.get("qualifiers") if not qualifiers or 'P670' not in qualifiers: continue number = qualifiers["P670"][0]["datavalue"]["value"] street_item = get_item(claim["mainsnak"]["datavalue"]["value"]["numeric-id"]) street = street_item.label() for q in qualifiers["P670"]: number = q["datavalue"]["value"] address = (f"{number} {street}" if g.street_number_first else f"{street} {number}") street_address.append(address) return street_address def get_markers(all_items): items = [] for item in all_items: if not item: continue locations = [list(i.get_lat_lon()) for i in item.locations] image_filenames = item.get_claim("P18") street_address = get_item_street_addresses(item) d = { "qid": item.qid, "label": item.label(), "description": item.description(), "markers": locations, "image_list": image_filenames, "street_address": street_address, "isa_list": [v["id"] for v in item.get_claim("P31") if v], } if aliases := item.get_aliases(): d["aliases"] = aliases items.append(d) return items def get_user_location(): gi = GeoIP.open("/home/edward/lib/data/GeoIPCity.dat", GeoIP.GEOIP_STANDARD) remote_ip = request.remote_addr gir = gi.record_by_addr(remote_ip) if not gir: return lat, lon = gir["latitude"], gir["longitude"] return (lat, lon) @app.route("/") def redirect_from_root(): return redirect(url_for("map_start_page")) @app.route("/index") def index_page(): return render_template("index.html") @app.route("/identifier") def identifier_index(): return render_template("identifier_index.html", property_map=property_map) @app.route("/commons/") def get_commons_image(filename): detail = commons.image_detail([filename], thumbheight=600, thumbwidth=600) image = detail[filename] return redirect(image["thumburl"]) @app.route("/identifier/") def identifier_page(pid): per_page = 10 page = int(request.args.get("page", 1)) property_dict = {pid: (osm_keys, label) for pid, osm_keys, label in property_map} osm_keys, label = property_dict[pid] wd = model.Item.query.filter(model.Item.claims.has_key(pid)) total = wd.count() start = per_page * (page - 1) items = wd.all()[start : per_page * page] qids = [item.qid for item in items] print(qids) # pred = None # values = set() # for item in items: # values |= set(item.get_claim(pid)) # # for key in osm_keys: # if key == 'ref': # continue # if pred is None: # pred = model.Point.tags[key].in_(values) # else: # pred |= model.Point.tags[key].in_(values) # osm_points = {} for qid in qids: osm_points[qid] = model.Point.query.filter( model.Point.tags["wikidata"] == qid ).all() osm_total = len(osm_points) return render_template( "identifier_page.html", pid=pid, osm_keys=osm_keys, label=label, items=items, total=total, osm_total=osm_total, osm_points=osm_points, ) @app.route("/old_map///") def old_map_location(zoom, lat, lng): t = int(time()) return render_template("map.html", zoom=zoom, lat=lat, lng=lng, time=t) @app.route("/map") def map_start_page(): location = get_user_location() lat, lon = location return redirect(url_for( 'map_location', zoom=16, lat=f'{lat:.5f}', lon=f'{lon:.5f}', )) @app.route("/map///") def map_location(zoom, lat, lon): user = flask_login.current_user username = user.username if user.is_authenticated else None return render_template("map.html", zoom=zoom, lat=lat, lon=lon, username=username) @app.route("/old_map") def old_map_start_page(): t = int(time()) location = get_user_location() if not location: return render_template("map.html", zoom=16, lat=None, lng=None, time=t) lat, lng = location return render_template("map.html", zoom=16, lat=lat, lng=lng, time=t) @app.route("/search/map") def search_map_page(): user_lat, user_lon = get_user_location() or (None, None) q = request.args.get("q") if not q: return render_template("map.html", user_lat=user_lat, user_lon=user_lon) hits = nominatim.lookup(q) for hit in hits: if "geotext" in hit: del hit["geotext"] bbox = [hit["boundingbox"] for hit in hits] return render_template( "search_map.html", hits=hits, bbox_list=bbox, user_lat=user_lat, user_lon=user_lon, ) @app.route("/search") def search_page(): q = request.args.get("q") if not q: return render_template("search.html", hits=None, bbox_list=None) hits = nominatim.lookup(q) for hit in hits: if "geotext" in hit: del hit["geotext"] bbox = [hit["boundingbox"] for hit in hits] return render_template("search.html", hits=hits, bbox_list=bbox) def get_isa_count(items): isa_count = Counter() for item in items: if not item: continue isa_list = item.get_claim("P31") for isa in isa_list: if not isa: print("missing IsA:", item.qid) continue isa_count[isa["id"]] += 1 return isa_count.most_common() def get_and_save_item(qid): entity = wikidata_api.get_entity(qid) entity_qid = entity["id"] if entity_qid != qid: print(f'redirect {qid} -> {entity_qid}') item = model.Item.query.get(entity_qid[1:]) return item coords = wikidata.get_entity_coords(entity["claims"]) item_id = int(qid[1:]) obj = {k: v for k, v in entity.items() if k in entity_keys} try: item = model.Item(item_id=item_id, **obj) except TypeError: print(qid) print(f'{entity["pageid"]=} {entity["ns"]=} {entity["type"]=}') print(entity.keys()) raise item.locations = model.location_objects(coords) database.session.add(item) database.session.commit() return item def get_bbox_centroid(bbox): bbox = make_envelope(bbox) centroid = database.session.query(func.ST_AsText(func.ST_Centroid(bbox))).scalar() lon, lat = re_point.match(centroid).groups() return lat, lon @app.route("/api/1/items") def api_wikidata_items(): bounds = request.args.get("bounds") g.street_number_first = is_street_number_first(*get_bbox_centroid(bounds)) t0 = time() q = get_items_in_bbox(bounds) db_items = q.all() items = get_markers(db_items) counts = get_isa_count(db_items) isa_ids = [qid[1:] for qid, count in counts] isa_items = { isa.qid: isa for isa in model.Item.query.filter(model.Item.item_id.in_(isa_ids)) } isa_count = [] for qid, count in counts: item = isa_items.get(qid) if not item: item = get_and_save_item(qid) label = item.label() if item else "[missing]" isa = { "qid": qid, "count": count, "label": label, } isa_count.append(isa) t1 = time() - t0 print(f"wikidata: {t1} seconds") response = jsonify(success=True, items=items, isa_count=isa_count, duration=t1) response.headers["Access-Control-Allow-Origin"] = "*" return response @app.route("/api/1/osm") def api_osm_objects(): bounds = request.args.get("bounds") t0 = time() objects = get_osm_with_wikidata_tag(bounds) t1 = time() - t0 print(f"OSM: {t1} seconds") response = jsonify(success=True, objects=objects, duration=t1) response.headers["Access-Control-Allow-Origin"] = "*" return response edu = ['Tag:amenity=college', 'Tag:amenity=university', 'Tag:amenity=school', 'Tag:office=educational_institution', 'Tag:building=university'] tall = ['Key:height', 'Key:building:levels'] extra_keys = { 'Q3914': ['Tag:building=school', 'Tag:building=college', 'Tag:amenity=college', 'Tag:office=educational_institution'], # school 'Q322563': edu, # vocational school 'Q383092': edu, # film school 'Q1021290': edu, # music school 'Q1244442': edu, # school building 'Q1469420': edu, # adult education centre 'Q2143781': edu, # drama school 'Q2385804': edu, # educational institution 'Q5167149': edu, # cooking school 'Q7894959': edu, # University Technical College 'Q47530379': edu, # agricultural college 'Q38723': edu, # higher education institution 'Q11303': tall, # skyscraper 'Q18142': tall, # high-rise building 'Q33673393': tall, # multi-storey building 'Q641226': ['Tag:leisure=stadium'], # arena 'Q2301048': ['Tag:aeroway=helipad'], # special airfield 'Q622425': ['Tag:amenity=pub', 'Tag:amenity=music_venue'], # nightclub 'Q187456': ['Tag:amenity=pub', 'Tag:amenity=nightclub'], # bar 'Q16917': ['Tag:amenity=clinic', 'Tag:building=clinic'], # hospital 'Q330284': ['Tag:amenity=market'], # marketplace 'Q5307737': ['Tag:amenity=pub', 'Tag:amenity=bar'], # drinking establishment 'Q875157': ['Tag:tourism=resort'], # resort 'Q174782': ['Tag:leisure=park', 'Tag:highway=pedestrian', 'Tag:foot=yes', 'Tag:area=yes', 'Tag:amenity=market', 'Tag:leisure=common'], # square 'Q34627': ['Tag:religion=jewish'], # synagogue 'Q16970': ['Tag:religion=christian'], # church 'Q32815': ['Tag:religion=islam'], # mosque 'Q811979': ['Key:building'], # architectural structure 'Q11691': ['Key:building'], # stock exchange 'Q1329623': ['Tag:amenity=arts_centre', # cultural centre 'Tag:amenity=community_centre'], 'Q856584': ['Tag:amenity=library'], # library building 'Q11315': ['Tag:landuse=retail'], # shopping mall 'Q39658032': ['Tag:landuse=retail'], # open air shopping centre 'Q277760': ['Tag:historic=folly', 'Tag:historic=city_gate'], # gatehouse 'Q180174': ['Tag:historic=folly'], # folly 'Q15243209': ['Tag:leisure=park', 'Tag:boundary=national_park'], # historic district 'Q3010369': ['Tag:historic=monument'], # opening ceremony 'Q123705': ['Tag:place=suburb'], # neighbourhood 'Q256020': ['Tag:amenity=pub'], # inn 'Q41253': ['Tag:amenity=theatre'], # movie theater 'Q17350442': ['Tag:amenity=theatre'], # venue 'Q156362': ['Tag:amenity=winery'], # winery 'Q14092': ['Tag:leisure=fitness_centre', 'Tag:leisure=sports_centre'], # gymnasium 'Q27686': ['Tag:tourism=hostel', # hotel 'Tag:tourism=guest_house', 'Tag:building=hotel'], 'Q11707': ['Tag:amenity=cafe', 'Tag:amenity=fast_food', 'Tag:shop=deli', 'Tag:shop=bakery', 'Key:cuisine'], # restaurant 'Q2360219': ['Tag:amenity=embassy'], # permanent mission 'Q27995042': ['Tag:protection_title=Wilderness Area'], # wilderness area 'Q838948': ['Tag:historic=memorial', 'Tag:historic=monument'], # work of art 'Q23413': ['Tag:place=locality'], # castle 'Q28045079': ['Tag:historic=archaeological_site', 'Tag:site_type=fortification', 'Tag:embankment=yes'], # contour fort 'Q744099': ['Tag:historic=archaeological_site', 'Tag:site_type=fortification', 'Tag:embankment=yes'], # hillfort 'Q515': ['Tag:border_type=city'], # city 'Q1254933': ['Tag:amenity=university'], # astronomical observatory 'Q1976594': ['Tag:landuse=industrial'], # science park 'Q190928': ['Tag:landuse=industrial'], # shipyard 'Q4663385': ['Tag:historic=train_station', # former railway station 'Tag:railway=historic_station'], 'Q11997323': ['Tag:emergency=lifeboat_station'], # lifeboat station 'Q16884952': ['Tag:castle_type=stately', 'Tag:building=country_house'], # country house 'Q1343246': ['Tag:castle_type=stately', 'Tag:building=country_house'], # English country house 'Q4919932': ['Tag:castle_type=stately'], # stately home 'Q1763828': ['Tag:amenity=community_centre'], # multi-purpose hall 'Q3469910': ['Tag:amenity=community_centre'], # performing arts center 'Q57660343': ['Tag:amenity=community_centre'], # performing arts building 'Q163740': ['Tag:amenity=community_centre', # nonprofit organization 'Tag:amenity=social_facility', 'Key:social_facility'], 'Q41176': ['Key:building:levels'], # building 'Q44494': ['Tag:historic=mill'], # mill 'Q56822897': ['Tag:historic=mill'], # mill building 'Q2175765': ['Tag:public_transport=stop_area'], # tram stop 'Q179700': ['Tag:memorial=statue', # statue 'Tag:memorial:type=statue', 'Tag:historic=memorial'], 'Q1076486': ['Tag:landuse=recreation_ground'], # sports venue 'Q988108': ['Tag:amenity=community_centre', # club 'Tag:community_centre=club_home'], 'Q55004558': ['Tag:service=yard', 'Tag:landuse=railway'], # car barn 'Q19563580': ['Tag:landuse=railway'], # rail yard 'Q134447': ['Tag:generator:source=nuclear'], # nuclear power plant 'Q1258086': ['Tag:leisure=park', 'Tag:boundary=national_park'], # National Historic Site 'Q32350958': ['Tag:leisure=bingo'], # Bingo hall 'Q53060': ['Tag:historic=gate', # gate 'Tag:tourism=attraction'], 'Q3947': ['Tag:tourism=hotel', # house 'Tag:building=hotel', 'Tag:tourism=guest_house'], 'Q847017': ['Tag:leisure=sports_centre'], # sports club 'Q820477': ['Tag:landuse=quarry', 'Tag:gnis:feature_type=Mine'], # mine 'Q77115': ['Tag:leisure=sports_centre'], # community center 'Q35535': ['Tag:amenity=police'], # police 'Q16560': ['Tag:tourism=attraction', # palace 'Tag:historic=yes'], 'Q131734': ['Tag:amenity=pub', # brewery 'Tag:industrial=brewery'], 'Q828909': ['Tag:landuse=commercial', 'Tag:landuse=industrial', 'Tag:historic=dockyard'], # wharf 'Q10283556': ['Tag:landuse=railway'], # motive power depot 'Q18674739': ['Tag:leisure=stadium'], # event venue 'Q20672229': ['Tag:historic=archaeological_site'], # friary 'Q207694': ['Tag:museum=art'], # art museum 'Q22698': ['Tag:leisure=dog_park', 'Tag:amenity=market', 'Tag:place=square', 'Tag:leisure=common'], # park 'Q738570': ['Tag:place=suburb'], # central business district 'Q1133961': ['Tag:place=suburb'], # commercial district 'Q935277': ['Tag:gnis:ftype=Playa', 'Tag:natural=sand'], # salt pan 'Q14253637': ['Tag:gnis:ftype=Playa', 'Tag:natural=sand'], # dry lake 'Q63099748': ['Tag:tourism=hotel', # hotel building 'Tag:building=hotel', 'Tag:tourism=guest_house'], 'Q2997369': ['Tag:leisure=park', 'Tag:highway=pedestrian', 'Tag:foot=yes', 'Tag:area=yes', 'Tag:amenity=market', 'Tag:leisure=common'], # plaza 'Q130003': ['Tag:landuse=winter_sports', # ski resort 'Tag:site=piste', 'Tag:leisure=resort', 'Tag:landuse=recreation_ground'], 'Q4830453': ['Key:office', 'Tag:building=office'], # business 'Q43229': ['Key:office', 'Tag:building=office'], # organization 'Q17084016': ['Tag:office=association', 'Tag:office=ngo'], # nonprofit corporation } skip_isa = { 13226383, 16686448, 2221906, 2133296, # space (architecture) 56052926, # building division 15989253, # part 9350592, # telecommunications infrastructure 121359, # infrastructure 28877, # goods 2897903, # goods and services 2995644, # result 733541, # consequence 408386, # inference 3249551, # process 20937557, # series 16887380, # group 28813620, # set 99527517, # collection entity 1150070, # change 1190554, # occurrence 26907166, # temporal entity 2425052, # electrical appliance 931447, # electrical load 210729, # electrical element 3749263, # electrical device 16798631, # equipment 66310125, # nonbiological component 22811462, # type of manufactured good 21146257, # type 16889133, # class 1310239, # component 337060, # perceptible object 581105, # consumer electronics 2858615, # electronic machine 1183543, # device 39546, # tool 35825432, # converter 11019, # machine 8205328, # artificial physical object 223557, # physical object 35459920, # three-dimensional object 488383, # object 35120, # entity 1454986, # physical system 30060700, # scientific object 58778, # system 6671777, # structure 4406616, # concrete object 2555640, # cell (architecture) 78642244, # closed space 1902617, # verblijfsruimte 180516, # room ['Key:room'] 17334923, # location 27096213, # geographic entity 58416391, # spatial entity 58415929, # spatio-temporal entity 811979, # architectural structure 811430, # human-made geographic feature 35145743, # human-made landform 27096235, # artificial geographic entity 618123, # geographical feature 386724, # work 15401930, # product 102074988, # artificial physical structure 15710813, # physical structure 1299240, # interior space 4830453, # business 3563237, # economic unit 2198779, # unit 7184903, # abstract object 16334295, # group of humans 16334298, # group of living things 61961344, # group of physical objects 98119401, # group or class of physical objects 106559804, # person or organization 24229398, # agent 23958946, # individual entity 4830453, # business 2695280, # technique 21162272, # means 4026292, # action 1914636, # activity 372222, # human-readable medium 494756, # data 42848, # data 1166770, # depiction 11024, # communication 6031064, # information exchange 52948, # interaction 23009552, # exchange 23009675, # transfer 22294683, # biological process involved in intraspecies interaction between organisms 628858, # workplace 1228250, # line 211548, # locus 36161, # set 864377, # multiset 246672, # mathematical object 5469988, # formalization 4393498, # representation 930933, # relation 217594, # class 294440, # public space 7551384, # social space 83493482, # thanking 83492918, # acknowledgement 628523, # message 11028, # information 189970, # social status 11424100, # status 4897819, # role 1207505, # quality 937228, # property 11862829, # academic discipline 1047113, # specialty 9081, # knowledge 104127086, # memory 12488383, # content 2434238, # heritage 23893363, # heritage 82821, # tradition 251777, # custom 1299714, # habit 36529775, # habit 7302601, # recognition 854429, # portal 391414, # architectural element 3955017, # intermediate good 2424752, # product 19603939, # building component 55638, # tertiary sector of the economy 3958441, # economic sector 7406919, # service } skip_tags = {"Key:addr:street"} def get_item(item_id): item = model.Item.query.get(item_id) if item: return item item = get_and_save_item(f"Q{item_id}") database.session.add(item) database.session.commit() return item def get_items(item_ids): items = [] for item_id in item_ids: item = model.Item.query.get(item_id) if not item: if not get_and_save_item(f"Q{item_id}"): continue item = model.Item.query.get(item_id) items.append(item) return items def get_item_tags(item): isa_items = [] isa_list = [v["numeric-id"] for v in item.get_claim("P31")] isa_items = get_items(isa_list) osm_list = set() seen = set(isa_list) | skip_isa while isa_items: isa = isa_items.pop() if not isa: continue osm = [v for v in isa.get_claim("P1282") if v not in skip_tags] if isa.qid in extra_keys: osm += extra_keys[isa.qid] osm_list.update(osm) subclass_of = [v["numeric-id"] for v in (isa.get_claim("P279") or []) if v] isa_list = [isa_id for isa_id in subclass_of if isa_id not in seen] seen.update(isa_list) isa_items += get_items(isa_list) return sorted(osm_list) @app.route("/api/1/item/Q/tags") def api_get_item_tags(item_id): t0 = time() item = model.Item.query.get(item_id) osm_list = get_item_tags(item) t1 = time() - t0 response = jsonify(success=True, qid=item.qid, tag_or_key_list=osm_list, duration=t1) response.headers["Access-Control-Allow-Origin"] = "*" return response def get_tag_filter(cls, tag_list): tag_filter = [] for tag_or_key in tag_list: if tag_or_key.startswith("Key:"): tag_filter.append(cls.tags.has_key(tag_or_key[4:])) if tag_or_key.startswith("Tag:"): k, _, v = tag_or_key.partition("=") tag_filter.append(cls.tags[k[4:]] == v) return tag_filter def get_country_iso3166_1(lat, lon): point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), srid) alpha2_codes = set() q = model.Polygon.query.filter(func.ST_Covers(model.Polygon.way, point), model.Polygon.admin_level == "2") for country in q: alpha2 = country.tags.get("ISO3166-1") if not alpha2: continue alpha2_codes.add(alpha2) g.alpha2_codes = alpha2_codes return alpha2_codes def is_street_number_first(lat, lon): if lat is None or lon is None: return True alpha2 = get_country_iso3166_1(lat, lon) alpha2_number_first = {'GB', 'IE', 'US', 'MX', 'CA', 'FR', 'AU', 'NZ', 'ZA'} return bool(alpha2_number_first & alpha2) def get_nearby(bbox, item, max_distance=200): db_bbox = make_envelope(bbox) osm_objects = {} distances = {} tag_list = get_item_tags(item) if not tag_list: return [] item_is_street = item.is_street() for loc in item.locations: lat, lon = loc.get_lat_lon() point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326) for cls in model.Point, model.Line, model.Polygon: if item_is_street and cls == model.Point: continue tag_filter = get_tag_filter(cls, tag_list) dist = func.ST_Distance(point, cls.way.cast(Geography(srid=4326))) q = (cls.query.add_columns(dist.label('distance')) .filter( func.ST_Intersects(db_bbox, cls.way), func.ST_Area(cls.way) < 20 * func.ST_Area(db_bbox), or_(*tag_filter)) .order_by(point.distance_centroid(cls.way))) if item_is_street: q = q.filter(cls.tags.has_key("name"), cls.tags["highway"] != 'bus_stop') if "Key:amenity" in tag_list: q = q.filter(cls.tags["amenity"] != "bicycle_parking", cls.tags["amenity"] != "bicycle_repair_station", cls.tags["amenity"] != "atm", cls.tags["amenity"] != "recycling") q = q.limit(20) # print(q.statement.compile(compile_kwargs={"literal_binds": True})) for i, dist in q: if dist > max_distance: continue osm_objects.setdefault(i.identifier, i) if i.identifier not in distances or dist < distances[i.identifier]: distances[i.identifier] = dist nearby = [(osm_objects[identifier], dist) for identifier, dist in sorted(distances.items(), key=lambda i:i[1])] return nearby[:10] def find_preset_file(k, v, ending): ts_dir = app.config["ID_TAGGING_SCHEMA_DIR"] preset_dir = os.path.join(ts_dir, "data", "presets") filename = os.path.join(preset_dir, k, v + ".json") if os.path.exists(filename): return { "tag_or_key": f"Tag:{k}={v}", "preset": f"{k}/{v}", "filename": filename, } filename = os.path.join(preset_dir, k, f"{v}_{ending}.json") if os.path.exists(filename): return { "tag_or_key": f"Tag:{k}={v}", "preset": f"{k}/{v}", "filename": filename, } filename = os.path.join(preset_dir, k, "_" + v + ".json") if os.path.exists(filename): return { "tag_or_key": f"Tag:{k}={v}", "preset": f"{k}/{v}", "filename": filename, } filename = os.path.join(preset_dir, k + ".json") if os.path.exists(filename): return { "tag_or_key": f"Key:{k}", "preset": k, "filename": filename, } def get_presets_from_tags(osm): if 'GB' in g.alpha2_codes: ts_dir = app.config["ID_TAGGING_SCHEMA_DIR"] filename = os.path.join(ts_dir, "dist", "translations", "en-GB.json") translations = json.load(open(filename))["en-GB"]["presets"]["presets"] else: translations = {} found = [] endings = {model.Point: "point", model.Line: "line", model.Polygon: "area"} ending = endings[type(osm)] for k, v in osm.tags.items(): if k == 'amenity' and v == 'clock' and osm.tags.get('display') == 'sundial': tag_or_key = f"Tag:{k}={v}" found.append({"tag_or_key": tag_or_key, "name": "Sundial"}) continue match = find_preset_file(k, v, ending) if not match: continue preset = match["preset"] if preset in translations: match["name"] = translations[preset]["name"] else: match["name"] = json.load(open(match["filename"]))["name"] del match["filename"] found.append(match) return found def get_address_nodes_within_building(building, bbox): db_bbox = make_envelope(bbox) ewkt = building.as_EWKT q = model.Point.query.filter( func.ST_Intersects(db_bbox, model.Point.way), func.ST_Covers(func.ST_GeomFromEWKT(ewkt), model.Point.way), model.Point.tags.has_key("addr:street"), model.Point.tags.has_key("addr:housenumber"), ) return [node.tags for node in q] def get_part_of(thing, bbox): db_bbox = make_envelope(bbox) ewkt = thing.as_EWKT q = model.Polygon.query.filter( func.ST_Intersects(db_bbox, model.Polygon.way), func.ST_Covers(model.Polygon.way, func.ST_GeomFromEWKT(ewkt)), or_( model.Polygon.tags.has_key("landuse"), model.Polygon.tags.has_key("amenity"), ), model.Polygon.tags.has_key("name"), ) return [polygon.tags for polygon in q] def address_from_tags(tags): keys = ["street", "housenumber"] if g.street_number_first: keys.reverse() return " ".join(tags["addr:" + k] for k in keys) @app.route("/api/1/item/Q/candidates") def api_find_osm_candidates(item_id): bounds = request.args.get("bounds") g.street_number_first = is_street_number_first(*get_bbox_centroid(bounds)) t0 = time() item = model.Item.query.get(item_id) nearby = [] for osm, dist in get_nearby(bounds, item): tags = osm.tags name = osm.name or tags.get("addr:housename") or tags.get("inscription") if not name and "addr:housenumber" in tags and "addr:street" in tags: name = address_from_tags(tags) if isinstance(osm, model.Polygon) and "building" in osm.tags: address_nodes = get_address_nodes_within_building(osm, bounds) address_list = [address_from_tags(addr) for addr in address_nodes] else: address_list = [] cur = { "identifier": osm.identifier, "distance": dist, "name": name, "tags": tags, "geojson": osm.geojson(), "presets": get_presets_from_tags(osm), "address_list": address_list, } if hasattr(osm, 'area'): cur["area"] = osm.area part_of = [i["name"] for i in get_part_of(osm, bounds) if i["name"] != name] if part_of: cur["part_of"] = part_of nearby.append(cur) t1 = time() - t0 response = jsonify(success=True, qid=item.qid, nearby=nearby, duration=t1) response.headers["Access-Control-Allow-Origin"] = "*" return response @app.route("/api/1/missing") def api_missing_wikidata_items(): qids_arg = request.args.get("qids") if not qids_arg: return jsonify(success=False, error="required parameter 'qids' is missing", items=[], isa_count=[]) qids = [] for qid in qids_arg.upper().split(","): qid = qid.strip() m = re_qid.match(qid) if not m: continue qids.append(qid) if not qids: return jsonify(success=True, items=[], isa_count=[]) lat, lon = request.args.get("lat"), request.args.get("lon") g.street_number_first = is_street_number_first(lat, lon) db_items = [] for qid in qids: item = model.Item.query.get(qid[1:]) if not item: item = get_and_save_item(qid) db_items.append(item) items = get_markers(db_items) counts = get_isa_count(db_items) isa_ids = [qid[1:] for qid, count in counts] isa_items = { isa.qid: isa for isa in model.Item.query.filter(model.Item.item_id.in_(isa_ids)) } isa_count = [] for qid, count in counts: item = isa_items.get(qid) if not item: item = get_and_save_item(qid) label = item.label() if item else "[missing]" isa = { "qid": qid, "count": count, "label": label, } isa_count.append(isa) response = jsonify(success=True, items=items, isa_count=isa_count) response.headers["Access-Control-Allow-Origin"] = "*" return response @app.route("/api/1/search") def api_search(): q = request.args["q"] hits = nominatim.lookup(q) for hit in hits: if "geotext" in hit: del hit["geotext"] hit["name"] = nominatim.get_hit_name(hit) hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}" response = jsonify(hits=hits) response.headers["Access-Control-Allow-Origin"] = "*" return response @app.route("/refresh/Q") def refresh_item(item_id): assert not model.Item.query.get(item_id) qid = f'Q{item_id}' entity = wikidata_api.get_entity(qid) entity_qid = entity.pop("id") assert qid == entity_qid coords = wikidata.get_entity_coords(entity["claims"]) assert coords obj = {k: v for k, v in entity.items() if k in entity_keys} item = model.Item(item_id=item_id, **obj) print(item) item.locations = model.location_objects(coords) database.session.add(item) database.session.commit() return 'done' @app.route('/login') def login_openstreetmap(): return redirect(url_for('start_oauth', next=request.args.get('next'))) @app.route('/logout') def logout(): next_url = request.args.get('next') or url_for('index') flask_login.logout_user() flash('you are logged out') return redirect(next_url) @app.route('/done/') def done(): flash('login successful') return redirect(url_for('index')) @app.route('/oauth/start') def start_oauth(): next_page = request.args.get('next') if next_page: session['next'] = next_page client_key = app.config['CLIENT_KEY'] client_secret = app.config['CLIENT_SECRET'] request_token_url = 'https://www.openstreetmap.org/oauth/request_token' callback = url_for('oauth_callback', _external=True) oauth = OAuth1Session(client_key, client_secret=client_secret, callback_uri=callback) fetch_response = oauth.fetch_request_token(request_token_url) session['owner_key'] = fetch_response.get('oauth_token') session['owner_secret'] = fetch_response.get('oauth_token_secret') base_authorization_url = 'https://www.openstreetmap.org/oauth/authorize' authorization_url = oauth.authorization_url(base_authorization_url, oauth_consumer_key=client_key) return redirect(authorization_url) @login_manager.user_loader def load_user(user_id): return model.User.query.get(user_id) @app.route("/oauth/callback", methods=["GET"]) def oauth_callback(): client_key = app.config['CLIENT_KEY'] client_secret = app.config['CLIENT_SECRET'] oauth = OAuth1Session(client_key, client_secret=client_secret, resource_owner_key=session['owner_key'], resource_owner_secret=session['owner_secret']) oauth_response = oauth.parse_authorization_response(request.url) verifier = oauth_response.get('oauth_verifier') access_token_url = 'https://www.openstreetmap.org/oauth/access_token' oauth = OAuth1Session(client_key, client_secret=client_secret, resource_owner_key=session['owner_key'], resource_owner_secret=session['owner_secret'], verifier=verifier) oauth_tokens = oauth.fetch_access_token(access_token_url) session['owner_key'] = oauth_tokens.get('oauth_token') session['owner_secret'] = oauth_tokens.get('oauth_token_secret') r = oauth.get(osm_api_base + '/user/details') info = osm_oauth.parse_userinfo_call(r.content) user = model.User.query.filter_by(osm_id=info['id']).one_or_none() if user: user.osm_oauth_token = oauth_tokens.get('oauth_token') user.osm_oauth_token_secret = oauth_tokens.get('oauth_token_secret') else: user = model.User( username=info['username'], description=info['description'], img=info['img'], osm_id=info['id'], osm_account_created=info['account_created'], ) database.session.add(user) database.session.commit() flask_login.login_user(user) next_page = session.get('next') or url_for('index_page') return redirect(next_page) if __name__ == "__main__": app.run(host="0.0.0.0")