From 54b280655f98eff7ae7c9fdb67e369a312c265ab Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Sat, 17 Apr 2021 18:02:53 +0200 Subject: [PATCH 01/10] Moving code around --- geocode/database.py | 25 +++++++++++++++++++++++ geocode/model.py | 34 ++++++++++++++++++++++++++++++ geocode/scotland.py | 12 +++++++++++ lookup.py | 50 +++++++++++++-------------------------------- 4 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 geocode/database.py create mode 100644 geocode/model.py create mode 100644 geocode/scotland.py diff --git a/geocode/database.py b/geocode/database.py new file mode 100644 index 0000000..dc667bf --- /dev/null +++ b/geocode/database.py @@ -0,0 +1,25 @@ +from sqlalchemy import create_engine, func +from sqlalchemy.orm import scoped_session, sessionmaker + +session = scoped_session(sessionmaker()) + + +def init_db(db_url): + session.configure(bind=get_engine(db_url)) + + +def get_engine(db_url, echo=False): + return create_engine(db_url, pool_recycle=3600, echo=echo) + + +def init_app(app, echo=False): + db_url = app.config["DB_URL"] + session.configure(bind=get_engine(db_url, echo=echo)) + + @app.teardown_appcontext + def shutdown_session(exception=None): + session.remove() + + +def now_utc(): + return func.timezone("utc", func.now()) diff --git a/geocode/model.py b/geocode/model.py new file mode 100644 index 0000000..5800b24 --- /dev/null +++ b/geocode/model.py @@ -0,0 +1,34 @@ +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.schema import Column +from sqlalchemy.types import Integer, Float, Numeric, String +from sqlalchemy.dialects import postgresql +from geoalchemy2 import Geometry +from .database import session + +Base = declarative_base() +Base.query = session.query_property() + +class Polygon(Base): + __tablename__ = "planet_osm_polygon" + + osm_id = Column(Integer, primary_key=True, autoincrement=False) + + way_area = Column(Float) + tags = Column(postgresql.HSTORE) + way = Column(Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False) + +class Scotland(Base): + __tablename__ = "scotland" + + gid = Column(Integer, primary_key=True) + shape_leng = Column(Numeric) + shape_area = Column(Numeric) + code = Column(String(3)) + c91code1 = Column(String(5)) + c91code2 = Column(String(5)) + c91code3 = Column(String(5)) + c91code4 = Column(String(5)) + name = Column(String(50)) + + geom = Column(Geometry("MULTIPOLYGON", srid=27700)) + diff --git a/geocode/scotland.py b/geocode/scotland.py new file mode 100644 index 0000000..60c9b43 --- /dev/null +++ b/geocode/scotland.py @@ -0,0 +1,12 @@ +from flask import current_app +import psycopg2 + +def get_scotland_code(lat, lon): + conn = psycopg2.connect(**current_app.config["DB_PARAMS"]) + cur = conn.cursor() + + point = f"ST_Transform(ST_SetSRID(ST_MakePoint({lon}, {lat}), 4326), 27700)" + cur.execute(f"select code, name from scotland where st_contains(geom, {point});") + row = cur.fetchone() + conn.close() + return row[0] if row else None diff --git a/lookup.py b/lookup.py index e255e56..a3445f7 100755 --- a/lookup.py +++ b/lookup.py @@ -2,11 +2,9 @@ from flask import Flask, render_template, request, jsonify, redirect, url_for import geocode -import geocode.wikidata -import geocode.overpass +from geocode import wikidata, overpass, scotland, database, model import urllib.parse import random -import psycopg2 from geopy.distance import distance # select gid, code, name from scotland where st_contains(geom, ST_Transform(ST_SetSRID(ST_MakePoint(-4.177, 55.7644), 4326), 27700)); @@ -19,6 +17,7 @@ city_of_london_qid = "Q23311" app = Flask(__name__) app.config.from_object("config.default") +database.init_app(app) def get_random_lat_lon(): @@ -70,7 +69,7 @@ def build_dict(hit, lat, lon): def do_lookup(elements, lat, lon): try: hit = osm_lookup(elements, lat, lon) - except geocode.wikidata.QueryError as e: + except wikidata.QueryError as e: return { "query": e.query, "error": e.r.text, @@ -80,26 +79,6 @@ def do_lookup(elements, lat, lon): return build_dict(hit, lat, lon) -def get_scotland_code(lat, lon): - conn = psycopg2.connect(**app.config["DB_PARAMS"]) - cur = conn.cursor() - - point = f"ST_Transform(ST_SetSRID(ST_MakePoint({lon}, {lat}), 4326), 27700)" - cur.execute(f"select code, name from scotland where st_contains(geom, {point});") - row = cur.fetchone() - - # expand search, disabled for now 2020-04-20 - if not row: - cur.execute( - f"select code, name from scotland where ST_DWithin(geom, {point}, 100);" - ) - row = cur.fetchone() - - conn.close() - if row: - return row[0] - - def wdqs_geosearch_query(lat, lon): if isinstance(lat, float): lat = f"{lat:f}" @@ -107,7 +86,7 @@ def wdqs_geosearch_query(lat, lon): lon = f"{lon:f}" query = render_template("sparql/geosearch.sparql", lat=lat, lon=lon) - return geocode.wikidata.wdqs(query) + return wikidata.wdqs(query) def wdqs_geosearch(lat, lon): @@ -142,7 +121,7 @@ def wdqs_geosearch(lat, lon): def lat_lon_to_wikidata(lat, lon): - scotland_code = get_scotland_code(lat, lon) + scotland_code = scotland.get_scotland_code(lat, lon) if scotland_code: rows = lookup_scottish_parish_in_wikidata(scotland_code) @@ -152,7 +131,7 @@ def lat_lon_to_wikidata(lat, lon): return {"elements": elements, "result": result} - elements = geocode.overpass.get_osm_elements(lat, lon) + elements = overpass.get_osm_elements(lat, lon) result = do_lookup(elements, lat, lon) # special case because the City of London is admin_level=6 in OSM @@ -173,22 +152,21 @@ def lat_lon_to_wikidata(lat, lon): return {"elements": elements, "result": result} - def lookup_scottish_parish_in_wikidata(code): query = render_template("sparql/scottish_parish.sparql", code=code) - return geocode.wikidata.wdqs(query) + return wikidata.wdqs(query) def lookup_gss_in_wikidata(gss): query = render_template("sparql/lookup_gss.sparql", gss=gss) - return geocode.wikidata.wdqs(query) + return wikidata.wdqs(query) def lookup_wikidata_by_name(name, lat, lon): query = render_template( "sparql/lookup_by_name.sparql", name=repr(name), lat=str(lat), lon=str(lon) ) - return geocode.wikidata.wdqs(query) + return wikidata.wdqs(query) def unescape_title(t): @@ -228,7 +206,7 @@ def osm_lookup(elements, lat, lon): continue if "wikidata" in tags: qid = tags["wikidata"] - commons = geocode.wikidata.qid_to_commons_category(qid) + commons = wikidata.qid_to_commons_category(qid) if commons: return { "wikidata": qid, @@ -262,7 +240,7 @@ def osm_lookup(elements, lat, lon): qid = has_wikidata_tag[0]["wikidata"] return { "wikidata": qid, - "commons_cat": geocode.qid_to_commons_category(qid), + "commons_cat": wikidata.qid_to_commons_category(qid), "admin_level": admin_level, } @@ -287,7 +265,7 @@ def index(): def random_location(): lat, lon = get_random_lat_lon() - elements = geocode.overpass.get_osm_elements(lat, lon) + elements = overpass.get_osm_elements(lat, lon) result = do_lookup(elements, lat, lon) return render_template( @@ -300,7 +278,7 @@ def wikidata_tag(): lat = float(request.args.get("lat")) lon = float(request.args.get("lon")) - scotland_code = get_scotland_code(lat, lon) + scotland_code = scotland.get_scotland_code(lat, lon) if scotland_code: rows = lookup_scottish_parish_in_wikidata(scotland_code) @@ -308,7 +286,7 @@ def wikidata_tag(): elements = [] result = build_dict(hit, lat, lon) else: - elements = geocode.overpass.get_osm_elements(lat, lon) + elements = overpass.get_osm_elements(lat, lon) result = do_lookup(elements, lat, lon) return render_template( From 44241751b265a05553742080046de1f8b60646bd Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Sat, 17 Apr 2021 18:31:58 +0200 Subject: [PATCH 02/10] Query database instead of overpass --- geocode/model.py | 16 ++++++++++++++++ geocode/overpass.py | 36 ------------------------------------ lookup.py | 24 ++++++------------------ templates/detail.html | 2 +- 4 files changed, 23 insertions(+), 55 deletions(-) delete mode 100644 geocode/overpass.py diff --git a/geocode/model.py b/geocode/model.py index 5800b24..16efdaa 100644 --- a/geocode/model.py +++ b/geocode/model.py @@ -2,6 +2,10 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.schema import Column from sqlalchemy.types import Integer, Float, Numeric, String from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import column_property +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy import func + from geoalchemy2 import Geometry from .database import session @@ -12,10 +16,22 @@ class Polygon(Base): __tablename__ = "planet_osm_polygon" osm_id = Column(Integer, primary_key=True, autoincrement=False) + admin_level = Column(String) way_area = Column(Float) tags = Column(postgresql.HSTORE) way = Column(Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False) + area = column_property(func.ST_Area(way)) + + @hybrid_property + def area_in_sq_km(self): + return self.area / (1000 * 1000) + + @classmethod + def coords_within(cls, lat, lon): + point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326) + return cls.query.filter(cls.admin_level.isnot(None), + func.ST_Within(point, cls.way)) class Scotland(Base): __tablename__ = "scotland" diff --git a/geocode/overpass.py b/geocode/overpass.py deleted file mode 100644 index 860ea2a..0000000 --- a/geocode/overpass.py +++ /dev/null @@ -1,36 +0,0 @@ -from flask import current_app -from . import headers -import os -import json -import requests - -OVERPASS_URL = "https://lz4.overpass-api.de" - - -def run_query(oql): - return requests.post( - OVERPASS_URL + "/api/interpreter", data=oql.encode("utf-8"), headers=headers - ) - - -def is_in_lat_lon(lat, lon): - oql = f""" -[out:json][timeout:25]; -is_in({lat},{lon})->.a; -(way(pivot.a); rel(pivot.a);); -out bb tags qt;""" - - return run_query(oql) - - -def get_osm_elements(lat, lon): - filename = f"cache/{lat}_{lon}.json" - use_cache = current_app.config["USE_CACHE"] - - if use_cache and os.path.exists(filename): - return json.load(open(filename))["elements"] - - r = is_in_lat_lon(lat, lon) - if use_cache: - open(filename, "wb").write(r.content) - return r.json()["elements"] diff --git a/lookup.py b/lookup.py index a3445f7..e33fd0a 100755 --- a/lookup.py +++ b/lookup.py @@ -2,10 +2,9 @@ from flask import Flask, render_template, request, jsonify, redirect, url_for import geocode -from geocode import wikidata, overpass, scotland, database, model +from geocode import wikidata, scotland, database, model import urllib.parse import random -from geopy.distance import distance # select gid, code, name from scotland where st_contains(geom, ST_Transform(ST_SetSRID(ST_MakePoint(-4.177, 55.7644), 4326), 27700)); @@ -32,15 +31,6 @@ def get_random_lat_lon(): return lat, lon -def bounding_box_area(element): - bbox = element["bounds"] - - x = distance((bbox["maxlat"], bbox["minlon"]), (bbox["maxlat"], bbox["maxlon"])) - y = distance((bbox["minlat"], bbox["maxlon"]), (bbox["maxlat"], bbox["minlon"])) - - return x.km * y.km - - def wd_to_qid(wd): # expecting {"type": "url", "value": "https://www.wikidata.org/wiki/Q30"} if wd["type"] == "uri": @@ -131,7 +121,7 @@ def lat_lon_to_wikidata(lat, lon): return {"elements": elements, "result": result} - elements = overpass.get_osm_elements(lat, lon) + elements = model.Polygon.coords_within(lat, lon) result = do_lookup(elements, lat, lon) # special case because the City of London is admin_level=6 in OSM @@ -190,12 +180,10 @@ def get_commons_cat_from_gss(gss): def osm_lookup(elements, lat, lon): - elements.sort(key=lambda e: bounding_box_area(e)) + elements = sorted(elements, key=lambda e: e.area) for e in elements: - if "tags" not in e: - continue - tags = e["tags"] + tags = e.tags admin_level_tag = tags.get("admin_level") admin_level = ( int(admin_level_tag) @@ -265,7 +253,7 @@ def index(): def random_location(): lat, lon = get_random_lat_lon() - elements = overpass.get_osm_elements(lat, lon) + elements = model.Polygon.coords_within(lat, lon) result = do_lookup(elements, lat, lon) return render_template( @@ -286,7 +274,7 @@ def wikidata_tag(): elements = [] result = build_dict(hit, lat, lon) else: - elements = overpass.get_osm_elements(lat, lon) + elements = model.Polygon.coords_within(lat, lon) result = do_lookup(elements, lat, lon) return render_template( diff --git a/templates/detail.html b/templates/detail.html index b372a5c..941d8bf 100644 --- a/templates/detail.html +++ b/templates/detail.html @@ -32,7 +32,7 @@ {% for element in elements %} {% set tags = element.tags %} - 
{{ element | pprint }}
+ 
{{ element.tags | pprint }}
{% endfor %} From c5a66abb25d29bad406d85eb4dfe3a23453d22e2 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Sat, 17 Apr 2021 19:29:09 +0200 Subject: [PATCH 03/10] Move code around --- geocode/wikidata.py | 106 +++++++++++++++++++++++++++++++++ lookup.py | 139 +++++--------------------------------------- 2 files changed, 120 insertions(+), 125 deletions(-) diff --git a/geocode/wikidata.py b/geocode/wikidata.py index 33ea0d2..226ecab 100644 --- a/geocode/wikidata.py +++ b/geocode/wikidata.py @@ -1,8 +1,12 @@ +from flask import render_template import requests import simplejson from . import headers +import urllib.parse wikidata_query_api_url = "https://query.wikidata.org/bigdata/namespace/wdq/sparql" +wd_entity = "http://www.wikidata.org/entity/Q" +commons_cat_start = "https://commons.wikimedia.org/wiki/Category:" class QueryError(Exception): @@ -49,3 +53,105 @@ def wdqs(query): return r.json()["results"]["bindings"] except simplejson.errors.JSONDecodeError: raise QueryError(query, r) + + +def wd_to_qid(wd): + # expecting {"type": "url", "value": "https://www.wikidata.org/wiki/Q30"} + if wd["type"] == "uri": + return wd_uri_to_qid(wd["value"]) + + +def wd_uri_to_qid(value): + assert value.startswith(wd_entity) + return value[len(wd_entity) - 1 :] + +def geosearch_query(lat, lon): + if isinstance(lat, float): + lat = f"{lat:f}" + if isinstance(lon, float): + lon = f"{lon:f}" + + query = render_template("sparql/geosearch.sparql", lat=lat, lon=lon) + return wdqs(query) + + +def geosearch(lat, lon): + default_max_dist = 1 + rows = geosearch_query(lat, lon) + max_dist = { + "Q188509": 1, # suburb + "Q3957": 2, # town + "Q532": 1, # village + "Q5084": 1, # hamlet + "Q515": 2, # city + "Q1549591": 3, # big city + } + for row in rows: + isa = wd_uri_to_qid(row["isa"]["value"]) + + if ( + "commonsCat" not in row + and "commonsSiteLink" not in row + and isa not in max_dist + ): + continue + + distance = float(row["distance"]["value"]) + if distance > max_dist.get(isa, default_max_dist): + continue + + if "commonsCat" not in row and "commonsSiteLink" not in row: + break + + return row + + +def lookup_scottish_parish_in_wikidata(code): + query = render_template("sparql/scottish_parish.sparql", code=code) + return wdqs(query) + + +def lookup_gss_in_wikidata(gss): + query = render_template("sparql/lookup_gss.sparql", gss=gss) + return wdqs(query) + + +def lookup_wikidata_by_name(name, lat, lon): + query = render_template( + "sparql/lookup_by_name.sparql", name=repr(name), lat=str(lat), lon=str(lon) + ) + return wdqs(query) + + +def unescape_title(t): + return urllib.parse.unquote(t.replace("_", " ")) + + +def commons_from_rows(rows): + for row in rows: + if "commonsCat" in row: + qid = wd_to_qid(row["item"]) + return {"wikidata": qid, "commons_cat": row["commonsCat"]["value"]} + if "commonsSiteLink" in row: + site_link = row["commonsSiteLink"]["value"] + qid = wd_to_qid(row["item"]) + cat = unescape_title(site_link[len(commons_cat_start) :]) + return {"wikidata": qid, "commons_cat": cat} + + +def get_commons_cat_from_gss(gss): + return commons_from_rows(lookup_gss_in_wikidata(gss)) + + +def build_dict(hit, lat, lon): + coords = {"lat": lat, "lon": lon} + if hit is None: + return dict(commons_cat=None, missing=True, coords=coords) + commons_cat = hit["commons_cat"] + url = commons_cat_start + urllib.parse.quote(commons_cat.replace(" ", "_")) + return dict( + commons_cat={"title": commons_cat, "url": url}, + coords=coords, + admin_level=hit.get("admin_level"), + wikidata=hit["wikidata"], + ) diff --git a/lookup.py b/lookup.py index e33fd0a..b70ab7b 100755 --- a/lookup.py +++ b/lookup.py @@ -1,19 +1,11 @@ #!/usr/bin/python3 from flask import Flask, render_template, request, jsonify, redirect, url_for -import geocode from geocode import wikidata, scotland, database, model -import urllib.parse +import geocode import random -# select gid, code, name from scotland where st_contains(geom, ST_Transform(ST_SetSRID(ST_MakePoint(-4.177, 55.7644), 4326), 27700)); - -commons_cat_start = "https://commons.wikimedia.org/wiki/Category:" - -wd_entity = "http://www.wikidata.org/entity/Q" city_of_london_qid = "Q23311" - - app = Flask(__name__) app.config.from_object("config.default") database.init_app(app) @@ -31,31 +23,6 @@ def get_random_lat_lon(): return lat, lon -def wd_to_qid(wd): - # expecting {"type": "url", "value": "https://www.wikidata.org/wiki/Q30"} - if wd["type"] == "uri": - return wd_uri_to_qid(wd["value"]) - - -def wd_uri_to_qid(value): - assert value.startswith(wd_entity) - return value[len(wd_entity) - 1 :] - - -def build_dict(hit, lat, lon): - coords = {"lat": lat, "lon": lon} - if hit is None: - return dict(commons_cat=None, missing=True, coords=coords) - commons_cat = hit["commons_cat"] - url = commons_cat_start + urllib.parse.quote(commons_cat.replace(" ", "_")) - return dict( - commons_cat={"title": commons_cat, "url": url}, - coords=coords, - admin_level=hit.get("admin_level"), - wikidata=hit["wikidata"], - ) - - def do_lookup(elements, lat, lon): try: hit = osm_lookup(elements, lat, lon) @@ -66,58 +33,17 @@ def do_lookup(elements, lat, lon): "query_url": "https://query.wikidata.org/#" + e.query, } - return build_dict(hit, lat, lon) - - -def wdqs_geosearch_query(lat, lon): - if isinstance(lat, float): - lat = f"{lat:f}" - if isinstance(lon, float): - lon = f"{lon:f}" - - query = render_template("sparql/geosearch.sparql", lat=lat, lon=lon) - return wikidata.wdqs(query) - - -def wdqs_geosearch(lat, lon): - default_max_dist = 1 - rows = wdqs_geosearch_query(lat, lon) - max_dist = { - "Q188509": 1, # suburb - "Q3957": 2, # town - "Q532": 1, # village - "Q5084": 1, # hamlet - "Q515": 2, # city - "Q1549591": 3, # big city - } - for row in rows: - isa = wd_uri_to_qid(row["isa"]["value"]) - - if ( - "commonsCat" not in row - and "commonsSiteLink" not in row - and isa not in max_dist - ): - continue - - distance = float(row["distance"]["value"]) - if distance > max_dist.get(isa, default_max_dist): - continue - - if "commonsCat" not in row and "commonsSiteLink" not in row: - break - - return row + return wikidata.build_dict(hit, lat, lon) def lat_lon_to_wikidata(lat, lon): scotland_code = scotland.get_scotland_code(lat, lon) if scotland_code: - rows = lookup_scottish_parish_in_wikidata(scotland_code) - hit = commons_from_rows(rows) + rows = wikidata.lookup_scottish_parish_in_wikidata(scotland_code) + hit = wikidata.commons_from_rows(rows) elements = [] - result = build_dict(hit, lat, lon) + result = wikidata.build_dict(hit, lat, lon) return {"elements": elements, "result": result} @@ -133,52 +59,15 @@ def lat_lon_to_wikidata(lat, lon): if not admin_level or admin_level >= 7: return {"elements": elements, "result": result} - row = wdqs_geosearch(lat, lon) + row = wikidata.geosearch(lat, lon) if row: - hit = commons_from_rows([row]) + hit = wikidata.commons_from_rows([row]) elements = [] - result = build_dict(hit, lat, lon) + result = wikidata.build_dict(hit, lat, lon) return {"elements": elements, "result": result} -def lookup_scottish_parish_in_wikidata(code): - query = render_template("sparql/scottish_parish.sparql", code=code) - return wikidata.wdqs(query) - - -def lookup_gss_in_wikidata(gss): - query = render_template("sparql/lookup_gss.sparql", gss=gss) - return wikidata.wdqs(query) - - -def lookup_wikidata_by_name(name, lat, lon): - query = render_template( - "sparql/lookup_by_name.sparql", name=repr(name), lat=str(lat), lon=str(lon) - ) - return wikidata.wdqs(query) - - -def unescape_title(t): - return urllib.parse.unquote(t.replace("_", " ")) - - -def commons_from_rows(rows): - for row in rows: - if "commonsCat" in row: - qid = wd_to_qid(row["item"]) - return {"wikidata": qid, "commons_cat": row["commonsCat"]["value"]} - if "commonsSiteLink" in row: - site_link = row["commonsSiteLink"]["value"] - qid = wd_to_qid(row["item"]) - cat = unescape_title(site_link[len(commons_cat_start) :]) - return {"wikidata": qid, "commons_cat": cat} - - -def get_commons_cat_from_gss(gss): - return commons_from_rows(lookup_gss_in_wikidata(gss)) - - def osm_lookup(elements, lat, lon): elements = sorted(elements, key=lambda e: e.area) @@ -203,7 +92,7 @@ def osm_lookup(elements, lat, lon): } gss = tags.get("ref:gss") if gss: - ret = get_commons_cat_from_gss(gss) + ret = wikidata.get_commons_cat_from_gss(gss) if ret: ret["admin_level"] = admin_level return ret @@ -213,10 +102,10 @@ def osm_lookup(elements, lat, lon): continue if name.endswith(" CP"): name = name[:-3] - rows = lookup_wikidata_by_name(name, lat, lon) + rows = wikidata.lookup_wikidata_by_name(name, lat, lon) if len(rows) == 1: - ret = commons_from_rows(rows) + ret = wikidata.commons_from_rows(rows) if ret: ret["admin_level"] = admin_level return ret @@ -269,10 +158,10 @@ def wikidata_tag(): scotland_code = scotland.get_scotland_code(lat, lon) if scotland_code: - rows = lookup_scottish_parish_in_wikidata(scotland_code) - hit = commons_from_rows(rows) + rows = wikidata.lookup_scottish_parish_in_wikidata(scotland_code) + hit = wikidata.commons_from_rows(rows) elements = [] - result = build_dict(hit, lat, lon) + result = wikidata.build_dict(hit, lat, lon) else: elements = model.Polygon.coords_within(lat, lon) result = do_lookup(elements, lat, lon) From 529611b42c8eae628d4ad980c5a9c5e5b2a1530e Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Wed, 18 May 2022 14:11:51 +0100 Subject: [PATCH 04/10] Update model --- geocode/model.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/geocode/model.py b/geocode/model.py index 16efdaa..67cc45f 100644 --- a/geocode/model.py +++ b/geocode/model.py @@ -21,7 +21,12 @@ class Polygon(Base): way_area = Column(Float) tags = Column(postgresql.HSTORE) way = Column(Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False) - area = column_property(func.ST_Area(way)) + area = column_property(func.ST_Area(way, False)) + + @property + def osm_url(self): + osm_type = "way" if self.osm_id > 0 else "relation" + return f"https://www.openstreetmap.org/{osm_type}/{abs(self.osm_id)}" @hybrid_property def area_in_sq_km(self): @@ -30,8 +35,9 @@ class Polygon(Base): @classmethod def coords_within(cls, lat, lon): point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326) - return cls.query.filter(cls.admin_level.isnot(None), - func.ST_Within(point, cls.way)) + return (cls.query.filter(cls.admin_level.isnot(None), + func.ST_Within(point, cls.way)) + .order_by(cls.area)) class Scotland(Base): __tablename__ = "scotland" From a5f7ba917dca4df192f44da562fb62f6fb804e09 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Wed, 18 May 2022 14:12:34 +0100 Subject: [PATCH 05/10] Bug fixes --- lookup.py | 19 ++++++++++++++----- templates/query_error.html | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 templates/query_error.html diff --git a/lookup.py b/lookup.py index b70ab7b..7644c6e 100755 --- a/lookup.py +++ b/lookup.py @@ -51,10 +51,10 @@ def lat_lon_to_wikidata(lat, lon): result = do_lookup(elements, lat, lon) # special case because the City of London is admin_level=6 in OSM - if result["wikidata"] == city_of_london_qid: + if result.get("wikidata") == city_of_london_qid: return {"elements": elements, "result": result} - admin_level = result["admin_level"] + admin_level = result.get("admin_level") if not admin_level or admin_level >= 7: return {"elements": elements, "result": result} @@ -69,8 +69,6 @@ def lat_lon_to_wikidata(lat, lon): def osm_lookup(elements, lat, lon): - elements = sorted(elements, key=lambda e: e.area) - for e in elements: tags = e.tags admin_level_tag = tags.get("admin_level") @@ -177,7 +175,18 @@ def detail_page(): lat, lon = [float(request.args.get(param)) for param in ("lat", "lon")] except TypeError: return redirect(url_for("index")) - reply = lat_lon_to_wikidata(lat, lon) + try: + reply = lat_lon_to_wikidata(lat, lon) + except wikidata.QueryError as e: + query, r = e.args + return render_template( + "query_error.html", + lat=lat, + lon=lon, + query=query, + r=r + ) + return render_template("detail.html", lat=lat, lon=lon, **reply) diff --git a/templates/query_error.html b/templates/query_error.html new file mode 100644 index 0000000..9c83655 --- /dev/null +++ b/templates/query_error.html @@ -0,0 +1,28 @@ + + + + + Geocode to commons + + + +

Geocode coordinates to Commons Category

+ +

visit endpoint + +| view in OSM + +| # + +

+ +

query

+ +
{{ query }}
+ +

reply

+ +
{{ r.text }}
+ + + From 5fde13608f2ed43b7b72cb4725b29d77a414de16 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Wed, 18 May 2022 14:12:50 +0100 Subject: [PATCH 06/10] Tidy --- templates/sparql/geosearch.sparql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/sparql/geosearch.sparql b/templates/sparql/geosearch.sparql index 433e072..8cbdb20 100644 --- a/templates/sparql/geosearch.sparql +++ b/templates/sparql/geosearch.sparql @@ -16,4 +16,4 @@ SELECT DISTINCT ?item ?distance ?itemLabel ?isa ?isaLabel ?commonsCat ?commonsSi OPTIONAL { ?commonsSiteLink schema:about ?item; schema:isPartOf . } SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } -} ORDER BY (?distance)""" +} ORDER BY (?distance) From 89549b750f059e37a8b4c9b94ec91cf01a72911f Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Wed, 18 May 2022 14:13:29 +0100 Subject: [PATCH 07/10] Add missing template --- templates/wikidata_tag.html | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 templates/wikidata_tag.html diff --git a/templates/wikidata_tag.html b/templates/wikidata_tag.html new file mode 100644 index 0000000..91aded6 --- /dev/null +++ b/templates/wikidata_tag.html @@ -0,0 +1,51 @@ + + + + + Geocode to commons + + + + +

Geocode coordinates to Commons Category

+ +

visit endpoint + +| view in OSM + +{% if result.commons_cat %} +| Commons category +{% endif %} + +{% if result.wikidata %} + | {{ result.wikidata }} +{% endif %} + +| # + +

+ +
{{ result | pprint }}
+ +{% if result.commons_cat %} +

({{ lat }}, {{ lon }}, {{result.commons_cat.title | pprint }}),

+{% endif %} + + +{% for e in elements %} +{% set tags = e.tags %} + + + + + + + + +{% endfor %} +
{{ tags.name }}{{ tags.admin_level }}{{ tags.boundary }}{{ tags.designation }}{{ '{:,.0f}'.format(e.area_in_sq_km) }} km²{{ 'wikidata' in tags }}
+ + + From faa0b3cb4e2c8ee7f48c17504c280f2dacfae884 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Tue, 10 Oct 2023 07:27:25 +0000 Subject: [PATCH 08/10] Bug fixes and improvements. --- geocode/model.py | 5 +++-- geocode/wikidata.py | 11 ++++++++--- lookup.py | 3 ++- templates/sparql/geosearch.sparql | 1 + 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/geocode/model.py b/geocode/model.py index 67cc45f..8230b0b 100644 --- a/geocode/model.py +++ b/geocode/model.py @@ -4,7 +4,7 @@ from sqlalchemy.types import Integer, Float, Numeric, String from sqlalchemy.dialects import postgresql from sqlalchemy.orm import column_property from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy import func +from sqlalchemy import func, cast from geoalchemy2 import Geometry from .database import session @@ -36,8 +36,9 @@ class Polygon(Base): def coords_within(cls, lat, lon): point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326) return (cls.query.filter(cls.admin_level.isnot(None), + cls.admin_level.regexp_match("^\d+$"), func.ST_Within(point, cls.way)) - .order_by(cls.area)) + .order_by(cls.area, cast(cls.admin_level, Integer).desc())) class Scotland(Base): __tablename__ = "scotland" diff --git a/geocode/wikidata.py b/geocode/wikidata.py index 226ecab..7588816 100644 --- a/geocode/wikidata.py +++ b/geocode/wikidata.py @@ -148,10 +148,15 @@ def build_dict(hit, lat, lon): if hit is None: return dict(commons_cat=None, missing=True, coords=coords) commons_cat = hit["commons_cat"] - url = commons_cat_start + urllib.parse.quote(commons_cat.replace(" ", "_")) - return dict( - commons_cat={"title": commons_cat, "url": url}, + ret = dict( coords=coords, admin_level=hit.get("admin_level"), wikidata=hit["wikidata"], ) + if not commons_cat: + return ret + + url = commons_cat_start + urllib.parse.quote(commons_cat.replace(" ", "_")) + ret["commons_cat"] = {"title": commons_cat, "url": url} + + return ret diff --git a/lookup.py b/lookup.py index 7644c6e..65e6270 100755 --- a/lookup.py +++ b/lookup.py @@ -2,6 +2,7 @@ from flask import Flask, render_template, request, jsonify, redirect, url_for from geocode import wikidata, scotland, database, model +from pprint import pprint import geocode import random @@ -108,7 +109,7 @@ def osm_lookup(elements, lat, lon): ret["admin_level"] = admin_level return ret - has_wikidata_tag = [e["tags"] for e in elements if "wikidata" in e["tags"]] + has_wikidata_tag = [e.tags for e in elements if e.tags.get("wikidata")] if len(has_wikidata_tag) != 1: return diff --git a/templates/sparql/geosearch.sparql b/templates/sparql/geosearch.sparql index 8cbdb20..55ea42f 100644 --- a/templates/sparql/geosearch.sparql +++ b/templates/sparql/geosearch.sparql @@ -12,6 +12,7 @@ SELECT DISTINCT ?item ?distance ?itemLabel ?isa ?isaLabel ?commonsCat ?commonsSi } } MINUS { ?item wdt:P582 ?endTime . } + MINUS { ?item wdt:P31 wd:Q1497375 . } OPTIONAL { ?item wdt:P373 ?commonsCat. } OPTIONAL { ?commonsSiteLink schema:about ?item; schema:isPartOf . } From ab81201908bcad899895a4430d886d7604b21a8d Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Tue, 10 Oct 2023 09:08:26 +0100 Subject: [PATCH 09/10] Remove unused pprint import --- lookup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lookup.py b/lookup.py index 65e6270..c75d1d0 100755 --- a/lookup.py +++ b/lookup.py @@ -2,7 +2,6 @@ from flask import Flask, render_template, request, jsonify, redirect, url_for from geocode import wikidata, scotland, database, model -from pprint import pprint import geocode import random From 4375b80d45e0feebea6a3bca7a5dd710cc0fcdef Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Tue, 10 Oct 2023 09:09:01 +0100 Subject: [PATCH 10/10] Add .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6a7802f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +.mypy_cache/