Compare commits

..

No commits in common. "4375b80d45e0feebea6a3bca7a5dd710cc0fcdef" and "fbcf3d102bfecb8eeb261de5f0995cc3de7b2cde" have entirely different histories.

11 changed files with 213 additions and 328 deletions

2
.gitignore vendored
View file

@ -1,2 +0,0 @@
__pycache__
.mypy_cache/

View file

@ -1,25 +0,0 @@
from sqlalchemy import create_engine, func
from sqlalchemy.orm import scoped_session, sessionmaker
session = scoped_session(sessionmaker())
def init_db(db_url):
session.configure(bind=get_engine(db_url))
def get_engine(db_url, echo=False):
return create_engine(db_url, pool_recycle=3600, echo=echo)
def init_app(app, echo=False):
db_url = app.config["DB_URL"]
session.configure(bind=get_engine(db_url, echo=echo))
@app.teardown_appcontext
def shutdown_session(exception=None):
session.remove()
def now_utc():
return func.timezone("utc", func.now())

View file

@ -1,57 +0,0 @@
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.schema import Column
from sqlalchemy.types import Integer, Float, Numeric, String
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import column_property
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy import func, cast
from geoalchemy2 import Geometry
from .database import session
Base = declarative_base()
Base.query = session.query_property()
class Polygon(Base):
__tablename__ = "planet_osm_polygon"
osm_id = Column(Integer, primary_key=True, autoincrement=False)
admin_level = Column(String)
way_area = Column(Float)
tags = Column(postgresql.HSTORE)
way = Column(Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False)
area = column_property(func.ST_Area(way, False))
@property
def osm_url(self):
osm_type = "way" if self.osm_id > 0 else "relation"
return f"https://www.openstreetmap.org/{osm_type}/{abs(self.osm_id)}"
@hybrid_property
def area_in_sq_km(self):
return self.area / (1000 * 1000)
@classmethod
def coords_within(cls, lat, lon):
point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326)
return (cls.query.filter(cls.admin_level.isnot(None),
cls.admin_level.regexp_match("^\d+$"),
func.ST_Within(point, cls.way))
.order_by(cls.area, cast(cls.admin_level, Integer).desc()))
class Scotland(Base):
__tablename__ = "scotland"
gid = Column(Integer, primary_key=True)
shape_leng = Column(Numeric)
shape_area = Column(Numeric)
code = Column(String(3))
c91code1 = Column(String(5))
c91code2 = Column(String(5))
c91code3 = Column(String(5))
c91code4 = Column(String(5))
name = Column(String(50))
geom = Column(Geometry("MULTIPOLYGON", srid=27700))

36
geocode/overpass.py Normal file
View file

@ -0,0 +1,36 @@
from flask import current_app
from . import headers
import os
import json
import requests
OVERPASS_URL = "https://lz4.overpass-api.de"
def run_query(oql):
return requests.post(
OVERPASS_URL + "/api/interpreter", data=oql.encode("utf-8"), headers=headers
)
def is_in_lat_lon(lat, lon):
oql = f"""
[out:json][timeout:25];
is_in({lat},{lon})->.a;
(way(pivot.a); rel(pivot.a););
out bb tags qt;"""
return run_query(oql)
def get_osm_elements(lat, lon):
filename = f"cache/{lat}_{lon}.json"
use_cache = current_app.config["USE_CACHE"]
if use_cache and os.path.exists(filename):
return json.load(open(filename))["elements"]
r = is_in_lat_lon(lat, lon)
if use_cache:
open(filename, "wb").write(r.content)
return r.json()["elements"]

View file

@ -1,12 +0,0 @@
from flask import current_app
import psycopg2
def get_scotland_code(lat, lon):
conn = psycopg2.connect(**current_app.config["DB_PARAMS"])
cur = conn.cursor()
point = f"ST_Transform(ST_SetSRID(ST_MakePoint({lon}, {lat}), 4326), 27700)"
cur.execute(f"select code, name from scotland where st_contains(geom, {point});")
row = cur.fetchone()
conn.close()
return row[0] if row else None

View file

@ -1,12 +1,8 @@
from flask import render_template
import requests import requests
import simplejson import simplejson
from . import headers from . import headers
import urllib.parse
wikidata_query_api_url = "https://query.wikidata.org/bigdata/namespace/wdq/sparql" wikidata_query_api_url = "https://query.wikidata.org/bigdata/namespace/wdq/sparql"
wd_entity = "http://www.wikidata.org/entity/Q"
commons_cat_start = "https://commons.wikimedia.org/wiki/Category:"
class QueryError(Exception): class QueryError(Exception):
@ -53,110 +49,3 @@ def wdqs(query):
return r.json()["results"]["bindings"] return r.json()["results"]["bindings"]
except simplejson.errors.JSONDecodeError: except simplejson.errors.JSONDecodeError:
raise QueryError(query, r) raise QueryError(query, r)
def wd_to_qid(wd):
# expecting {"type": "url", "value": "https://www.wikidata.org/wiki/Q30"}
if wd["type"] == "uri":
return wd_uri_to_qid(wd["value"])
def wd_uri_to_qid(value):
assert value.startswith(wd_entity)
return value[len(wd_entity) - 1 :]
def geosearch_query(lat, lon):
if isinstance(lat, float):
lat = f"{lat:f}"
if isinstance(lon, float):
lon = f"{lon:f}"
query = render_template("sparql/geosearch.sparql", lat=lat, lon=lon)
return wdqs(query)
def geosearch(lat, lon):
default_max_dist = 1
rows = geosearch_query(lat, lon)
max_dist = {
"Q188509": 1, # suburb
"Q3957": 2, # town
"Q532": 1, # village
"Q5084": 1, # hamlet
"Q515": 2, # city
"Q1549591": 3, # big city
}
for row in rows:
isa = wd_uri_to_qid(row["isa"]["value"])
if (
"commonsCat" not in row
and "commonsSiteLink" not in row
and isa not in max_dist
):
continue
distance = float(row["distance"]["value"])
if distance > max_dist.get(isa, default_max_dist):
continue
if "commonsCat" not in row and "commonsSiteLink" not in row:
break
return row
def lookup_scottish_parish_in_wikidata(code):
query = render_template("sparql/scottish_parish.sparql", code=code)
return wdqs(query)
def lookup_gss_in_wikidata(gss):
query = render_template("sparql/lookup_gss.sparql", gss=gss)
return wdqs(query)
def lookup_wikidata_by_name(name, lat, lon):
query = render_template(
"sparql/lookup_by_name.sparql", name=repr(name), lat=str(lat), lon=str(lon)
)
return wdqs(query)
def unescape_title(t):
return urllib.parse.unquote(t.replace("_", " "))
def commons_from_rows(rows):
for row in rows:
if "commonsCat" in row:
qid = wd_to_qid(row["item"])
return {"wikidata": qid, "commons_cat": row["commonsCat"]["value"]}
if "commonsSiteLink" in row:
site_link = row["commonsSiteLink"]["value"]
qid = wd_to_qid(row["item"])
cat = unescape_title(site_link[len(commons_cat_start) :])
return {"wikidata": qid, "commons_cat": cat}
def get_commons_cat_from_gss(gss):
return commons_from_rows(lookup_gss_in_wikidata(gss))
def build_dict(hit, lat, lon):
coords = {"lat": lat, "lon": lon}
if hit is None:
return dict(commons_cat=None, missing=True, coords=coords)
commons_cat = hit["commons_cat"]
ret = dict(
coords=coords,
admin_level=hit.get("admin_level"),
wikidata=hit["wikidata"],
)
if not commons_cat:
return ret
url = commons_cat_start + urllib.parse.quote(commons_cat.replace(" ", "_"))
ret["commons_cat"] = {"title": commons_cat, "url": url}
return ret

212
lookup.py
View file

@ -1,14 +1,24 @@
#!/usr/bin/python3 #!/usr/bin/python3
from flask import Flask, render_template, request, jsonify, redirect, url_for from flask import Flask, render_template, request, jsonify, redirect, url_for
from geocode import wikidata, scotland, database, model
import geocode import geocode
import geocode.wikidata
import geocode.overpass
import urllib.parse
import random import random
import psycopg2
from geopy.distance import distance
# select gid, code, name from scotland where st_contains(geom, ST_Transform(ST_SetSRID(ST_MakePoint(-4.177, 55.7644), 4326), 27700));
commons_cat_start = "https://commons.wikimedia.org/wiki/Category:"
wd_entity = "http://www.wikidata.org/entity/Q"
city_of_london_qid = "Q23311" city_of_london_qid = "Q23311"
app = Flask(__name__) app = Flask(__name__)
app.config.from_object("config.default") app.config.from_object("config.default")
database.init_app(app)
def get_random_lat_lon(): def get_random_lat_lon():
@ -23,54 +33,191 @@ def get_random_lat_lon():
return lat, lon return lat, lon
def bounding_box_area(element):
bbox = element["bounds"]
x = distance((bbox["maxlat"], bbox["minlon"]), (bbox["maxlat"], bbox["maxlon"]))
y = distance((bbox["minlat"], bbox["maxlon"]), (bbox["maxlat"], bbox["minlon"]))
return x.km * y.km
def wd_to_qid(wd):
# expecting {"type": "url", "value": "https://www.wikidata.org/wiki/Q30"}
if wd["type"] == "uri":
return wd_uri_to_qid(wd["value"])
def wd_uri_to_qid(value):
assert value.startswith(wd_entity)
return value[len(wd_entity) - 1 :]
def build_dict(hit, lat, lon):
coords = {"lat": lat, "lon": lon}
if hit is None:
return dict(commons_cat=None, missing=True, coords=coords)
commons_cat = hit["commons_cat"]
url = commons_cat_start + urllib.parse.quote(commons_cat.replace(" ", "_"))
return dict(
commons_cat={"title": commons_cat, "url": url},
coords=coords,
admin_level=hit.get("admin_level"),
wikidata=hit["wikidata"],
)
def do_lookup(elements, lat, lon): def do_lookup(elements, lat, lon):
try: try:
hit = osm_lookup(elements, lat, lon) hit = osm_lookup(elements, lat, lon)
except wikidata.QueryError as e: except geocode.wikidata.QueryError as e:
return { return {
"query": e.query, "query": e.query,
"error": e.r.text, "error": e.r.text,
"query_url": "https://query.wikidata.org/#" + e.query, "query_url": "https://query.wikidata.org/#" + e.query,
} }
return wikidata.build_dict(hit, lat, lon) return build_dict(hit, lat, lon)
def get_scotland_code(lat, lon):
conn = psycopg2.connect(**app.config["DB_PARAMS"])
cur = conn.cursor()
point = f"ST_Transform(ST_SetSRID(ST_MakePoint({lon}, {lat}), 4326), 27700)"
cur.execute(f"select code, name from scotland where st_contains(geom, {point});")
row = cur.fetchone()
# expand search, disabled for now 2020-04-20
if not row:
cur.execute(
f"select code, name from scotland where ST_DWithin(geom, {point}, 100);"
)
row = cur.fetchone()
conn.close()
if row:
return row[0]
def wdqs_geosearch_query(lat, lon):
if isinstance(lat, float):
lat = f"{lat:f}"
if isinstance(lon, float):
lon = f"{lon:f}"
query = render_template("sparql/geosearch.sparql", lat=lat, lon=lon)
return geocode.wikidata.wdqs(query)
def wdqs_geosearch(lat, lon):
default_max_dist = 1
rows = wdqs_geosearch_query(lat, lon)
max_dist = {
"Q188509": 1, # suburb
"Q3957": 2, # town
"Q532": 1, # village
"Q5084": 1, # hamlet
"Q515": 2, # city
"Q1549591": 3, # big city
}
for row in rows:
isa = wd_uri_to_qid(row["isa"]["value"])
if (
"commonsCat" not in row
and "commonsSiteLink" not in row
and isa not in max_dist
):
continue
distance = float(row["distance"]["value"])
if distance > max_dist.get(isa, default_max_dist):
continue
if "commonsCat" not in row and "commonsSiteLink" not in row:
break
return row
def lat_lon_to_wikidata(lat, lon): def lat_lon_to_wikidata(lat, lon):
scotland_code = scotland.get_scotland_code(lat, lon) scotland_code = get_scotland_code(lat, lon)
if scotland_code: if scotland_code:
rows = wikidata.lookup_scottish_parish_in_wikidata(scotland_code) rows = lookup_scottish_parish_in_wikidata(scotland_code)
hit = wikidata.commons_from_rows(rows) hit = commons_from_rows(rows)
elements = [] elements = []
result = wikidata.build_dict(hit, lat, lon) result = build_dict(hit, lat, lon)
return {"elements": elements, "result": result} return {"elements": elements, "result": result}
elements = model.Polygon.coords_within(lat, lon) elements = geocode.overpass.get_osm_elements(lat, lon)
result = do_lookup(elements, lat, lon) result = do_lookup(elements, lat, lon)
# special case because the City of London is admin_level=6 in OSM # special case because the City of London is admin_level=6 in OSM
if result.get("wikidata") == city_of_london_qid: if result["wikidata"] == city_of_london_qid:
return {"elements": elements, "result": result} return {"elements": elements, "result": result}
admin_level = result.get("admin_level") admin_level = result["admin_level"]
if not admin_level or admin_level >= 7: if not admin_level or admin_level >= 7:
return {"elements": elements, "result": result} return {"elements": elements, "result": result}
row = wikidata.geosearch(lat, lon) row = wdqs_geosearch(lat, lon)
if row: if row:
hit = wikidata.commons_from_rows([row]) hit = commons_from_rows([row])
elements = [] elements = []
result = wikidata.build_dict(hit, lat, lon) result = build_dict(hit, lat, lon)
return {"elements": elements, "result": result} return {"elements": elements, "result": result}
def lookup_scottish_parish_in_wikidata(code):
query = render_template("sparql/scottish_parish.sparql", code=code)
return geocode.wikidata.wdqs(query)
def lookup_gss_in_wikidata(gss):
query = render_template("sparql/lookup_gss.sparql", gss=gss)
return geocode.wikidata.wdqs(query)
def lookup_wikidata_by_name(name, lat, lon):
query = render_template(
"sparql/lookup_by_name.sparql", name=repr(name), lat=str(lat), lon=str(lon)
)
return geocode.wikidata.wdqs(query)
def unescape_title(t):
return urllib.parse.unquote(t.replace("_", " "))
def commons_from_rows(rows):
for row in rows:
if "commonsCat" in row:
qid = wd_to_qid(row["item"])
return {"wikidata": qid, "commons_cat": row["commonsCat"]["value"]}
if "commonsSiteLink" in row:
site_link = row["commonsSiteLink"]["value"]
qid = wd_to_qid(row["item"])
cat = unescape_title(site_link[len(commons_cat_start) :])
return {"wikidata": qid, "commons_cat": cat}
def get_commons_cat_from_gss(gss):
return commons_from_rows(lookup_gss_in_wikidata(gss))
def osm_lookup(elements, lat, lon): def osm_lookup(elements, lat, lon):
elements.sort(key=lambda e: bounding_box_area(e))
for e in elements: for e in elements:
tags = e.tags if "tags" not in e:
continue
tags = e["tags"]
admin_level_tag = tags.get("admin_level") admin_level_tag = tags.get("admin_level")
admin_level = ( admin_level = (
int(admin_level_tag) int(admin_level_tag)
@ -81,7 +228,7 @@ def osm_lookup(elements, lat, lon):
continue continue
if "wikidata" in tags: if "wikidata" in tags:
qid = tags["wikidata"] qid = tags["wikidata"]
commons = wikidata.qid_to_commons_category(qid) commons = geocode.wikidata.qid_to_commons_category(qid)
if commons: if commons:
return { return {
"wikidata": qid, "wikidata": qid,
@ -90,7 +237,7 @@ def osm_lookup(elements, lat, lon):
} }
gss = tags.get("ref:gss") gss = tags.get("ref:gss")
if gss: if gss:
ret = wikidata.get_commons_cat_from_gss(gss) ret = get_commons_cat_from_gss(gss)
if ret: if ret:
ret["admin_level"] = admin_level ret["admin_level"] = admin_level
return ret return ret
@ -100,22 +247,22 @@ def osm_lookup(elements, lat, lon):
continue continue
if name.endswith(" CP"): if name.endswith(" CP"):
name = name[:-3] name = name[:-3]
rows = wikidata.lookup_wikidata_by_name(name, lat, lon) rows = lookup_wikidata_by_name(name, lat, lon)
if len(rows) == 1: if len(rows) == 1:
ret = wikidata.commons_from_rows(rows) ret = commons_from_rows(rows)
if ret: if ret:
ret["admin_level"] = admin_level ret["admin_level"] = admin_level
return ret return ret
has_wikidata_tag = [e.tags for e in elements if e.tags.get("wikidata")] has_wikidata_tag = [e["tags"] for e in elements if "wikidata" in e["tags"]]
if len(has_wikidata_tag) != 1: if len(has_wikidata_tag) != 1:
return return
qid = has_wikidata_tag[0]["wikidata"] qid = has_wikidata_tag[0]["wikidata"]
return { return {
"wikidata": qid, "wikidata": qid,
"commons_cat": wikidata.qid_to_commons_category(qid), "commons_cat": geocode.qid_to_commons_category(qid),
"admin_level": admin_level, "admin_level": admin_level,
} }
@ -140,7 +287,7 @@ def index():
def random_location(): def random_location():
lat, lon = get_random_lat_lon() lat, lon = get_random_lat_lon()
elements = model.Polygon.coords_within(lat, lon) elements = geocode.overpass.get_osm_elements(lat, lon)
result = do_lookup(elements, lat, lon) result = do_lookup(elements, lat, lon)
return render_template( return render_template(
@ -153,15 +300,15 @@ def wikidata_tag():
lat = float(request.args.get("lat")) lat = float(request.args.get("lat"))
lon = float(request.args.get("lon")) lon = float(request.args.get("lon"))
scotland_code = scotland.get_scotland_code(lat, lon) scotland_code = get_scotland_code(lat, lon)
if scotland_code: if scotland_code:
rows = wikidata.lookup_scottish_parish_in_wikidata(scotland_code) rows = lookup_scottish_parish_in_wikidata(scotland_code)
hit = wikidata.commons_from_rows(rows) hit = commons_from_rows(rows)
elements = [] elements = []
result = wikidata.build_dict(hit, lat, lon) result = build_dict(hit, lat, lon)
else: else:
elements = model.Polygon.coords_within(lat, lon) elements = geocode.overpass.get_osm_elements(lat, lon)
result = do_lookup(elements, lat, lon) result = do_lookup(elements, lat, lon)
return render_template( return render_template(
@ -175,18 +322,7 @@ def detail_page():
lat, lon = [float(request.args.get(param)) for param in ("lat", "lon")] lat, lon = [float(request.args.get(param)) for param in ("lat", "lon")]
except TypeError: except TypeError:
return redirect(url_for("index")) return redirect(url_for("index"))
try:
reply = lat_lon_to_wikidata(lat, lon) reply = lat_lon_to_wikidata(lat, lon)
except wikidata.QueryError as e:
query, r = e.args
return render_template(
"query_error.html",
lat=lat,
lon=lon,
query=query,
r=r
)
return render_template("detail.html", lat=lat, lon=lon, **reply) return render_template("detail.html", lat=lat, lon=lon, **reply)

View file

@ -32,7 +32,7 @@
{% for element in elements %} {% for element in elements %}
{% set tags = element.tags %} {% set tags = element.tags %}
<pre>{{ element.tags | pprint }}</pre> <pre>{{ element | pprint }}</pre>
{% endfor %} {% endfor %}

View file

@ -1,28 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Geocode to commons</title>
</head>
<body>
<h1>Geocode coordinates to Commons Category</h1>
<p><a href="{{ url_for('index', lat=lat, lon=lon) }}">visit endpoint</a>
| <a href="https://www.openstreetmap.org/#map=17/{{lat }}/{{ lon }}">view in OSM</a>
| <a href="{{ url_for('detail_page', lat=lat, lon=lon) }}">#</a>
</p>
<h2>query</h2>
<pre>{{ query }}</pre>
<h2>reply</h2>
<pre>{{ r.text }}</pre>
</body>
</html>

View file

@ -12,9 +12,8 @@ SELECT DISTINCT ?item ?distance ?itemLabel ?isa ?isaLabel ?commonsCat ?commonsSi
} }
} }
MINUS { ?item wdt:P582 ?endTime . } MINUS { ?item wdt:P582 ?endTime . }
MINUS { ?item wdt:P31 wd:Q1497375 . }
OPTIONAL { ?item wdt:P373 ?commonsCat. } OPTIONAL { ?item wdt:P373 ?commonsCat. }
OPTIONAL { ?commonsSiteLink schema:about ?item; OPTIONAL { ?commonsSiteLink schema:about ?item;
schema:isPartOf <https://commons.wikimedia.org/>. } schema:isPartOf <https://commons.wikimedia.org/>. }
SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
} ORDER BY (?distance) } ORDER BY (?distance)"""

View file

@ -1,51 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Geocode to commons</title>
<style>
.text-end { text-align: right }
</style>
</head>
<body>
<h1>Geocode coordinates to Commons Category</h1>
<p><a href="{{ url_for('index', lat=lat, lon=lon) }}">visit endpoint</a>
| <a href="https://www.openstreetmap.org/#map=17/{{lat }}/{{ lon }}">view in OSM</a>
{% if result.commons_cat %}
| <a href="{{ result.commons_cat.url }}">Commons category</a>
{% endif %}
{% if result.wikidata %}
| <a href="https://www.wikidata.org/wiki/{{ result.wikidata }}">{{ result.wikidata }}</a>
{% endif %}
| <a href="{{ url_for('detail_page', lat=lat, lon=lon) }}">#</a>
</p>
<pre>{{ result | pprint }}</pre>
{% if result.commons_cat %}
<p>({{ lat }}, {{ lon }}, {{result.commons_cat.title | pprint }}),</p>
{% endif %}
<table>
{% for e in elements %}
{% set tags = e.tags %}
<tr>
<td><a href="{{ e.osm_url }}">{{ tags.name }}</a></td>
<td>{{ tags.admin_level }}</td>
<td>{{ tags.boundary }}</td>
<td>{{ tags.designation }}</td>
<td class="text-end">{{ '{:,.0f}'.format(e.area_in_sq_km) }} km²</td>
<td>{{ 'wikidata' in tags }}</td>
</tr>
{% endfor %}
</table>
</body>
</html>