dpriskorn/owl-map
1
0
Fork 0
forked from edward/owl-map
Code Pull requests Activity

Merge branch 'main' of github.com:EdwardBetts/owl-map

Browse source
This commit is contained in:
Edward Betts 2023-05-14 10:30:15 +00:00
parent 6d9c7fb1bc 88a0b9f897
commit 6ce8b30fcc
14 changed files with 847 additions and 452 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
config/default.py Normal file
View file

@ -0,0 +1,25 @@
"""Sample config."""
ID_TAGGING_SCHEMA_DIR = "/var/lib/data/id-tagging-schema"
ID_PRESET_DIR = "/var/lib/data/id-tagging-schema/data/presets/"
GEOIP_DATA = "/var/lib/data/GeoIP/GeoIPCity.dat"
GEOLITE2 = "/var/lib/data/GeoLite2/GeoLite2-City.mmdb"
CLIENT_KEY = ""
CLIENT_SECRET = ""
SECRET_KEY = ""
DEFAULT_COMMENT = "+wikidata"
ADMIN_NAME = ""
ADMIN_EMAIL = ""
ADMINS = [ADMIN_EMAIL]
SMTP_HOST = "localhost"
MAIL_FROM = "osm-wikidata@localhost"
ERROR_MAIL = True
PROPAGATE_EXCEPTIONS = False

13
matcher/__init__.py Normal file
View file

@ -0,0 +1,13 @@
"""Match OSM and Wikidata items."""
CallParams = dict[str, str | int]
user_agent = (
"osm-wikidata/0.1 (https://github.com/EdwardBetts/osm-wikidata; edward@4angle.com)"
)
def user_agent_headers() -> dict[str, str]:
"""User-Agent headers."""
return {"User-Agent": user_agent}

466
matcher/api.py
View file

@ -1,19 +1,21 @@
from sqlalchemy import func, or_, and_, text import json
import os.path
import re
from collections import Counter, defaultdict
from flask import current_app, g
from sqlalchemy import and_, func, or_, text
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from sqlalchemy.sql import select from sqlalchemy.sql import select
from sqlalchemy.sql.expression import literal, union, cast, column from sqlalchemy.sql.expression import cast, column, literal, union
from sqlalchemy.types import Float from sqlalchemy.types import Float
from sqlalchemy.dialects import postgresql
from matcher.planet import point, line, polygon from matcher import database, model, wikidata, wikidata_api
from matcher import model, database, wikidata_api, wikidata from matcher.planet import line, point, polygon
from collections import Counter, defaultdict
from flask import g, current_app
import re
import os.path
import json
srid = 4326 srid = 4326
re_point = re.compile(r'^POINT\((.+) (.+)\)$') re_point = re.compile(r"^POINT\((.+) (.+)\)$")
entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"} entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"}
tag_prefixes = { tag_prefixes = {
@ -37,9 +39,9 @@ skip_tags = {
"Key:brand", "Key:brand",
} }
def get_country_iso3166_1(lat, lon): def get_country_iso3166_1(lat, lon):
""" """For a given lat/lon return a set of ISO country codes.
For a given lat/lon return a set of ISO country codes.
Also cache the country code in the global object. Also cache the country code in the global object.
@ -48,8 +50,9 @@ def get_country_iso3166_1(lat, lon):
point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), srid) point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), srid)
alpha2_codes = set() alpha2_codes = set()
q = model.Polygon.query.filter(func.ST_Covers(model.Polygon.way, point), q = model.Polygon.query.filter(
model.Polygon.admin_level == "2") func.ST_Covers(model.Polygon.way, point), model.Polygon.admin_level == "2"
)
for country in q: for country in q:
alpha2 = country.tags.get("ISO3166-1") alpha2 = country.tags.get("ISO3166-1")
if not alpha2: if not alpha2:
@ -60,22 +63,23 @@ def get_country_iso3166_1(lat, lon):
return alpha2_codes return alpha2_codes
def is_street_number_first(lat, lon): def is_street_number_first(lat: float, lon: float) -> bool:
"""Is lat/lon within a country that puts number first in a street address."""
if lat is None or lon is None: if lat is None or lon is None:
return True return True
alpha2 = get_country_iso3166_1(lat, lon) alpha2 = get_country_iso3166_1(lat, lon)
# Incomplete list of countries that put street number first. # Incomplete list of countries that put street number first.
alpha2_number_first = { alpha2_number_first = {
'GB', # United Kingdom "GB", # United Kingdom
'IE', # Ireland "IE", # Ireland
'US', # United States "US", # United States
'MX', # Mexico "MX", # Mexico
'CA', # Canada "CA", # Canada
'FR', # France "FR", # France
'AU', # Australia "AU", # Australia
'NZ', # New Zealand "NZ", # New Zealand
'ZA', # South Africa "ZA", # South Africa
} }
return bool(alpha2_number_first & alpha2) return bool(alpha2_number_first & alpha2)
@ -84,22 +88,26 @@ def is_street_number_first(lat, lon):
def make_envelope(bounds): def make_envelope(bounds):
return func.ST_MakeEnvelope(*bounds, srid) return func.ST_MakeEnvelope(*bounds, srid)
def get_bbox_centroid(bbox): def get_bbox_centroid(bbox):
bbox = make_envelope(bbox) bbox = make_envelope(bbox)
centroid = database.session.query(func.ST_AsText(func.ST_Centroid(bbox))).scalar() centroid = database.session.query(func.ST_AsText(func.ST_Centroid(bbox))).scalar()
return reversed(re_point.match(centroid).groups()) return reversed(re_point.match(centroid).groups())
def make_envelope_around_point(lat, lon, distance): def make_envelope_around_point(lat, lon, distance):
conn = database.session.connection() conn = database.session.connection()
p = func.ST_MakePoint(lon, lat) p = func.ST_MakePoint(lon, lat)
s = select([ s = select(
func.ST_AsText(func.ST_Project(p, distance, func.radians(0))), [
func.ST_AsText(func.ST_Project(p, distance, func.radians(90))), func.ST_AsText(func.ST_Project(p, distance, func.radians(0))),
func.ST_AsText(func.ST_Project(p, distance, func.radians(180))), func.ST_AsText(func.ST_Project(p, distance, func.radians(90))),
func.ST_AsText(func.ST_Project(p, distance, func.radians(270))), func.ST_AsText(func.ST_Project(p, distance, func.radians(180))),
]) func.ST_AsText(func.ST_Project(p, distance, func.radians(270))),
]
)
row = conn.execute(s).fetchone() row = conn.execute(s).fetchone()
coords = [[float(v) for v in re_point.match(i).groups()] for i in row] coords = [[float(v) for v in re_point.match(i).groups()] for i in row]
@ -110,49 +118,64 @@ def make_envelope_around_point(lat, lon, distance):
return func.ST_MakeEnvelope(west, south, east, north, srid) return func.ST_MakeEnvelope(west, south, east, north, srid)
def drop_way_area(tags):
""" Remove the way_area field from a tags dict. """ def drop_way_area(tags: dict[str, str]) -> dict[str, str]:
"""Remove the way_area field from a tags dict."""
if "way_area" in tags: if "way_area" in tags:
del tags["way_area"] del tags["way_area"]
return tags return tags
def get_part_of(table_name, src_id, bbox): def get_part_of(table_name, src_id, bbox):
table_map = {'point': point, 'line': line, 'polygon': polygon} table_map = {"point": point, "line": line, "polygon": polygon}
table_alias = table_map[table_name].alias() table_alias = table_map[table_name].alias()
s = (select([polygon.c.osm_id, s = (
polygon.c.tags, select(
func.ST_Area(func.ST_Collect(polygon.c.way))]). [
where(and_(func.ST_Intersects(bbox, polygon.c.way), polygon.c.osm_id,
func.ST_Covers(polygon.c.way, table_alias.c.way), polygon.c.tags,
table_alias.c.osm_id == src_id, func.ST_Area(func.ST_Collect(polygon.c.way)),
polygon.c.tags.has_key("name"), ]
or_( )
polygon.c.tags.has_key("landuse"), .where(
polygon.c.tags.has_key("amenity"), and_(
))). func.ST_Intersects(bbox, polygon.c.way),
group_by(polygon.c.osm_id, polygon.c.tags)) func.ST_Covers(polygon.c.way, table_alias.c.way),
table_alias.c.osm_id == src_id,
polygon.c.tags.has_key("name"),
or_(
polygon.c.tags.has_key("landuse"),
polygon.c.tags.has_key("amenity"),
),
)
)
.group_by(polygon.c.osm_id, polygon.c.tags)
)
conn = database.session.connection() conn = database.session.connection()
return [{ return [
"type": "way" if osm_id > 0 else "relation", {
"id": abs(osm_id), "type": "way" if osm_id > 0 else "relation",
"tags": tags, "id": abs(osm_id),
"area": area, "tags": tags,
} for osm_id, tags, area in conn.execute(s)] "area": area,
}
for osm_id, tags, area in conn.execute(s)
]
def get_and_save_item(qid):
""" Download an item from Wikidata and cache it in the database. """
def get_and_save_item(qid: str) -> model.Item | None:
"""Download an item from Wikidata and cache it in the database."""
entity = wikidata_api.get_entity(qid) entity = wikidata_api.get_entity(qid)
entity_qid = entity["id"] entity_qid = entity["id"]
if entity_qid != qid: if entity_qid != qid:
print(f'redirect {qid} -> {entity_qid}') print(f"redirect {qid} -> {entity_qid}")
item = model.Item.query.get(entity_qid[1:]) item = model.Item.query.get(entity_qid[1:])
return item return item
if "claims" not in entity: if "claims" not in entity:
return return None
coords = wikidata.get_entity_coords(entity["claims"]) coords = wikidata.get_entity_coords(entity["claims"])
item_id = int(qid[1:]) item_id = int(qid[1:])
@ -171,8 +194,9 @@ def get_and_save_item(qid):
return item return item
def get_isa_count(items): def get_isa_count(items: list[model.Item]) -> list[tuple[int, int]]:
isa_count = Counter() """List of IsA counts."""
isa_count: Counter[int] = Counter()
for item in items: for item in items:
if not item: if not item:
continue continue
@ -199,13 +223,11 @@ def get_items_in_bbox(bbox):
def get_osm_with_wikidata_tag(bbox, isa_filter=None): def get_osm_with_wikidata_tag(bbox, isa_filter=None):
bbox_str = ','.join(str(v) for v in bbox) bbox_str = ",".join(str(v) for v in bbox)
extra_sql = "" extra_sql = ""
if isa_filter: if isa_filter:
q = ( q = model.Item.query.join(model.ItemLocation).filter(
model.Item.query.join(model.ItemLocation) func.ST_Covers(make_envelope(bbox), model.ItemLocation.location)
.filter(func.ST_Covers(make_envelope(bbox),
model.ItemLocation.location))
) )
q = add_isa_filter(q, isa_filter) q = add_isa_filter(q, isa_filter)
qids = [isa.qid for isa in q] qids = [isa.qid for isa in q]
@ -216,7 +238,8 @@ def get_osm_with_wikidata_tag(bbox, isa_filter=None):
extra_sql += f" AND tags -> 'wikidata' in ({qid_list})" extra_sql += f" AND tags -> 'wikidata' in ({qid_list})"
# easier than building this query with SQLAlchemy # easier than building this query with SQLAlchemy
sql = f''' sql = (
f"""
SELECT tbl, osm_id, tags, ARRAY[ST_Y(centroid), ST_X(centroid)], geojson SELECT tbl, osm_id, tags, ARRAY[ST_Y(centroid), ST_X(centroid)], geojson
FROM ( FROM (
SELECT 'point' as tbl, osm_id, tags, ST_AsText(ST_Centroid(way)) as centroid, ST_AsGeoJSON(way) as geojson SELECT 'point' as tbl, osm_id, tags, ST_AsText(ST_Centroid(way)) as centroid, ST_AsGeoJSON(way) as geojson
@ -235,24 +258,29 @@ UNION
HAVING st_area(st_collect(way)) < 20 * st_area(ST_MakeEnvelope({bbox_str}, {srid})) HAVING st_area(st_collect(way)) < 20 * st_area(ST_MakeEnvelope({bbox_str}, {srid}))
) as anon ) as anon
WHERE tags ? 'wikidata' WHERE tags ? 'wikidata'
''' + extra_sql """
+ extra_sql
)
conn = database.session.connection() conn = database.session.connection()
result = conn.execute(text(sql)) result = conn.execute(text(sql))
print(sql) print(sql)
point_sql = f''' point_sql = (
f"""
SELECT 'point' as tbl, osm_id, tags, ST_AsText(ST_Centroid(way)) as centroid, ST_AsGeoJSON(way) as geojson SELECT 'point' as tbl, osm_id, tags, ST_AsText(ST_Centroid(way)) as centroid, ST_AsGeoJSON(way) as geojson
FROM planet_osm_point FROM planet_osm_point
WHERE ST_Intersects(ST_MakeEnvelope({bbox_str}, {srid}), way) and tags ? 'wikidata' WHERE ST_Intersects(ST_MakeEnvelope({bbox_str}, {srid}), way) and tags ? 'wikidata'
''' + extra_sql """
+ extra_sql
)
print("point") print("point")
print(point_sql) print(point_sql)
tagged = [] tagged = []
for tbl, osm_id, tags, centroid, geojson in result: for tbl, osm_id, tags, centroid, geojson in result:
if tbl == 'point': if tbl == "point":
osm_type = "node" osm_type = "node"
else: else:
osm_type = "way" if osm_id > 0 else "relation" osm_type = "way" if osm_id > 0 else "relation"
@ -260,15 +288,17 @@ WHERE tags ? 'wikidata'
name = tags.get("name") or tags.get("addr:housename") or "[no label]" name = tags.get("name") or tags.get("addr:housename") or "[no label]"
tagged.append({ tagged.append(
"identifier": f"{osm_type}/{osm_id}", {
"id": osm_id, "identifier": f"{osm_type}/{osm_id}",
"type": osm_type, "id": osm_id,
"geojson": json.loads(geojson), "type": osm_type,
"centroid": centroid, "geojson": json.loads(geojson),
"name": name, "centroid": centroid,
"wikidata": tags["wikidata"], "name": name,
}) "wikidata": tags["wikidata"],
}
)
return tagged return tagged
@ -310,11 +340,13 @@ def get_item_tags(item):
isa, isa_path = isa_items.pop() isa, isa_path = isa_items.pop()
if not isa: if not isa:
continue continue
isa_path = isa_path + [{'qid': isa.qid, 'label': isa.label()}] isa_path = isa_path + [{"qid": isa.qid, "label": isa.label()}]
osm = [v for v in isa.get_claim("P1282") if v not in skip_tags] osm = [v for v in isa.get_claim("P1282") if v not in skip_tags]
osm += [extra.tag_or_key osm += [
for extra in model.ItemExtraKeys.query.filter_by(item_id=isa.item_id)] extra.tag_or_key
for extra in model.ItemExtraKeys.query.filter_by(item_id=isa.item_id)
]
for i in osm: for i in osm:
osm_list[i].append(isa_path[:]) osm_list[i].append(isa_path[:])
@ -369,14 +401,16 @@ def get_tags_for_isa_item(item):
isa, isa_path = isa_items.pop() isa, isa_path = isa_items.pop()
if not isa: if not isa:
continue continue
isa_path = isa_path + [{'qid': isa.qid, 'label': isa.label()}] isa_path = isa_path + [{"qid": isa.qid, "label": isa.label()}]
if isa.item_id not in items_checked_done: if isa.item_id not in items_checked_done:
items_checked.append({'qid': isa.qid, 'label': isa.label()}) items_checked.append({"qid": isa.qid, "label": isa.label()})
items_checked_done.add(isa.item_id) items_checked_done.add(isa.item_id)
osm = [v for v in isa.get_claim("P1282") if v not in skip_tags] osm = [v for v in isa.get_claim("P1282") if v not in skip_tags]
osm += [extra.tag_or_key osm += [
for extra in model.ItemExtraKeys.query.filter_by(item_id=isa.item_id)] extra.tag_or_key
for extra in model.ItemExtraKeys.query.filter_by(item_id=isa.item_id)
]
for i in osm: for i in osm:
osm_list[i].append(isa_path[:]) osm_list[i].append(isa_path[:])
@ -403,34 +437,31 @@ def get_tags_for_isa_item(item):
seen.update(isa_list) seen.update(isa_list)
isa_items += [(isa, isa_path) for isa in get_items(isa_list)] isa_items += [(isa, isa_path) for isa in get_items(isa_list)]
return { return {
'tags': {key: list(values) for key, values in osm_list.items()}, "tags": {key: list(values) for key, values in osm_list.items()},
'checked': items_checked, "checked": items_checked,
} }
def add_isa_filter(q, isa_qids): def add_isa_filter(q, isa_qids):
q_subclass = database.session.query(model.Item.qid).filter( q_subclass = database.session.query(model.Item.qid).filter(
func.jsonb_path_query_array( func.jsonb_path_query_array(
model.Item.claims, model.Item.claims,
'$.P279[*].mainsnak.datavalue.value.id', "$.P279[*].mainsnak.datavalue.value.id",
).bool_op('?|')(list(isa_qids)) ).bool_op("?|")(list(isa_qids))
) )
subclass_qid = {qid for qid, in q_subclass.all()} subclass_qid = {qid for qid, in q_subclass.all()}
isa = func.jsonb_path_query_array( isa = func.jsonb_path_query_array(
model.Item.claims, model.Item.claims,
'$.P31[*].mainsnak.datavalue.value.id', "$.P31[*].mainsnak.datavalue.value.id",
).bool_op('?|') ).bool_op("?|")
return q.filter(isa(list(isa_qids | subclass_qid))) return q.filter(isa(list(isa_qids | subclass_qid)))
def wikidata_items_count(bounds, isa_filter=None): def wikidata_items_count(bounds, isa_filter=None):
q = model.Item.query.join(model.ItemLocation).filter(
q = ( func.ST_Covers(make_envelope(bounds), model.ItemLocation.location)
model.Item.query.join(model.ItemLocation)
.filter(func.ST_Covers(make_envelope(bounds), model.ItemLocation.location))
) )
if isa_filter: if isa_filter:
@ -440,12 +471,12 @@ def wikidata_items_count(bounds, isa_filter=None):
return q.count() return q.count()
def wikidata_isa_counts(bounds, isa_filter=None): def wikidata_isa_counts(bounds, isa_filter=None):
db_bbox = make_envelope(bounds) db_bbox = make_envelope(bounds)
q = ( q = model.Item.query.join(model.ItemLocation).filter(
model.Item.query.join(model.ItemLocation) func.ST_Covers(db_bbox, model.ItemLocation.location)
.filter(func.ST_Covers(db_bbox, model.ItemLocation.location))
) )
if isa_filter: if isa_filter:
@ -474,12 +505,13 @@ def wikidata_isa_counts(bounds, isa_filter=None):
return isa_count return isa_count
def get_tag_filter(tags, tag_list): def get_tag_filter(tags, tag_list):
tag_filter = [] tag_filter = []
for tag_or_key in tag_list: for tag_or_key in tag_list:
if tag_or_key.startswith("Key:"): if tag_or_key.startswith("Key:"):
key = tag_or_key[4:] key = tag_or_key[4:]
tag_filter.append(and_(tags.has_key(key), tags[key] != 'no')) tag_filter.append(and_(tags.has_key(key), tags[key] != "no"))
for prefix in tag_prefixes: for prefix in tag_prefixes:
tag_filter.append(tags.has_key(f"{prefix}:{key}")) tag_filter.append(tags.has_key(f"{prefix}:{key}"))
@ -495,11 +527,11 @@ def get_tag_filter(tags, tag_list):
def get_preset_translations(): def get_preset_translations():
app = current_app app = current_app
country_language = { country_language = {
'AU': 'en-AU', # Australia "AU": "en-AU", # Australia
'GB': 'en-GB', # United Kingdom "GB": "en-GB", # United Kingdom
'IE': 'en-GB', # Ireland "IE": "en-GB", # Ireland
'IN': 'en-IN', # India "IN": "en-IN", # India
'NZ': 'en-NZ', # New Zealand "NZ": "en-NZ", # New Zealand
} }
ts_dir = app.config["ID_TAGGING_SCHEMA_DIR"] ts_dir = app.config["ID_TAGGING_SCHEMA_DIR"]
translation_dir = os.path.join(ts_dir, "dist", "translations") translation_dir = os.path.join(ts_dir, "dist", "translations")
@ -520,13 +552,14 @@ def get_preset_translations():
return {} return {}
def get_presets_from_tags(ending, tags): def get_presets_from_tags(ending, tags):
translations = get_preset_translations() translations = get_preset_translations()
found = [] found = []
for k, v in tags.items(): for k, v in tags.items():
if k == 'amenity' and v == 'clock' and tags.get('display') == 'sundial': if k == "amenity" and v == "clock" and tags.get("display") == "sundial":
tag_or_key = f"Tag:{k}={v}" tag_or_key = f"Tag:{k}={v}"
found.append({"tag_or_key": tag_or_key, "name": "Sundial"}) found.append({"tag_or_key": tag_or_key, "name": "Sundial"})
continue continue
@ -604,8 +637,7 @@ def address_node_label(tags):
def get_address_nodes_within_building(osm_id, bbox_list): def get_address_nodes_within_building(osm_id, bbox_list):
q = model.Point.query.filter( q = model.Point.query.filter(
polygon.c.osm_id == osm_id, polygon.c.osm_id == osm_id,
or_(*[func.ST_Intersects(bbox, model.Point.way) or_(*[func.ST_Intersects(bbox, model.Point.way) for bbox in bbox_list]),
for bbox in bbox_list]),
func.ST_Covers(polygon.c.way, model.Point.way), func.ST_Covers(polygon.c.way, model.Point.way),
model.Point.tags.has_key("addr:street"), model.Point.tags.has_key("addr:street"),
model.Point.tags.has_key("addr:housenumber"), model.Point.tags.has_key("addr:housenumber"),
@ -615,8 +647,14 @@ def get_address_nodes_within_building(osm_id, bbox_list):
def osm_display_name(tags): def osm_display_name(tags):
keys = ("bridge:name", "tunnel:name", "lock_name", "name", "addr:housename", keys = (
"inscription") "bridge:name",
"tunnel:name",
"lock_name",
"name",
"addr:housename",
"inscription",
)
for key in keys: for key in keys:
if key in tags: if key in tags:
return tags[key] return tags[key]
@ -625,6 +663,7 @@ def osm_display_name(tags):
def street_address_in_tags(tags): def street_address_in_tags(tags):
return "addr:housenumber" in tags and "addr:street" in tags return "addr:housenumber" in tags and "addr:street" in tags
def find_osm_candidates(item, limit=80, max_distance=450, names=None): def find_osm_candidates(item, limit=80, max_distance=450, names=None):
item_id = item.item_id item_id = item.item_id
item_is_linear_feature = item.is_linear_feature() item_is_linear_feature = item.is_linear_feature()
@ -637,51 +676,94 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
check_is_street_number_first(item.locations[0].get_lat_lon()) check_is_street_number_first(item.locations[0].get_lat_lon())
bbox_list = [make_envelope_around_point(*loc.get_lat_lon(), max_distance) bbox_list = [
for loc in item.locations] make_envelope_around_point(*loc.get_lat_lon(), max_distance)
for loc in item.locations
]
null_area = cast(None, Float) null_area = cast(None, Float)
dist = column('dist') dist = column("dist")
tags = column('tags', postgresql.HSTORE) tags = column("tags", postgresql.HSTORE)
tag_list = get_item_tags(item) tag_list = get_item_tags(item)
# tag_filters = get_tag_filter(point.c.tags, tag_list) # tag_filters = get_tag_filter(point.c.tags, tag_list)
# print(tag_filters) # print(tag_filters)
s_point = (select([literal('point').label('t'), point.c.osm_id, point.c.tags.label('tags'), s_point = (
func.min(func.ST_DistanceSphere(model.ItemLocation.location, point.c.way)).label('dist'), select(
func.ST_AsText(point.c.way), [
func.ST_AsGeoJSON(point.c.way), literal("point").label("t"),
null_area]). point.c.osm_id,
where(and_( point.c.tags.label("tags"),
or_(*[func.ST_Intersects(bbox, point.c.way) func.min(
for bbox in bbox_list]), func.ST_DistanceSphere(model.ItemLocation.location, point.c.way)
model.ItemLocation.item_id == item_id, ).label("dist"),
or_(*get_tag_filter(point.c.tags, tag_list)))). func.ST_AsText(point.c.way),
group_by(point.c.osm_id, point.c.tags, point.c.way)) func.ST_AsGeoJSON(point.c.way),
null_area,
]
)
.where(
and_(
or_(*[func.ST_Intersects(bbox, point.c.way) for bbox in bbox_list]),
model.ItemLocation.item_id == item_id,
or_(*get_tag_filter(point.c.tags, tag_list)),
)
)
.group_by(point.c.osm_id, point.c.tags, point.c.way)
)
s_line = (select([literal('line').label('t'), line.c.osm_id, line.c.tags.label('tags'), s_line = (
func.min(func.ST_DistanceSphere(model.ItemLocation.location, line.c.way)).label('dist'), select(
func.ST_AsText(func.ST_Centroid(func.ST_Collect(line.c.way))), [
func.ST_AsGeoJSON(func.ST_Collect(line.c.way)), literal("line").label("t"),
null_area]). line.c.osm_id,
where(and_( line.c.tags.label("tags"),
or_(*[func.ST_Intersects(bbox, line.c.way) for bbox in bbox_list]), func.min(
model.ItemLocation.item_id == item_id, func.ST_DistanceSphere(model.ItemLocation.location, line.c.way)
or_(*get_tag_filter(line.c.tags, tag_list)))). ).label("dist"),
group_by(line.c.osm_id, line.c.tags)) func.ST_AsText(func.ST_Centroid(func.ST_Collect(line.c.way))),
func.ST_AsGeoJSON(func.ST_Collect(line.c.way)),
null_area,
]
)
.where(
and_(
or_(*[func.ST_Intersects(bbox, line.c.way) for bbox in bbox_list]),
model.ItemLocation.item_id == item_id,
or_(*get_tag_filter(line.c.tags, tag_list)),
)
)
.group_by(line.c.osm_id, line.c.tags)
)
s_polygon = (select([literal('polygon').label('t'), polygon.c.osm_id, polygon.c.tags.label('tags'), s_polygon = (
func.min(func.ST_DistanceSphere(model.ItemLocation.location, polygon.c.way)).label('dist'), select(
func.ST_AsText(func.ST_Centroid(func.ST_Collect(polygon.c.way))), [
func.ST_AsGeoJSON(func.ST_Collect(polygon.c.way)), literal("polygon").label("t"),
func.ST_Area(func.ST_Collect(polygon.c.way))]). polygon.c.osm_id,
where(and_( polygon.c.tags.label("tags"),
or_(*[func.ST_Intersects(bbox, polygon.c.way) for bbox in bbox_list]), func.min(
model.ItemLocation.item_id == item_id, func.ST_DistanceSphere(model.ItemLocation.location, polygon.c.way)
or_(*get_tag_filter(polygon.c.tags, tag_list)))). ).label("dist"),
group_by(polygon.c.osm_id, polygon.c.tags). func.ST_AsText(func.ST_Centroid(func.ST_Collect(polygon.c.way))),
having(func.ST_Area(func.ST_Collect(polygon.c.way)) < 20 * func.ST_Area(bbox_list[0]))) func.ST_AsGeoJSON(func.ST_Collect(polygon.c.way)),
func.ST_Area(func.ST_Collect(polygon.c.way)),
]
)
.where(
and_(
or_(*[func.ST_Intersects(bbox, polygon.c.way) for bbox in bbox_list]),
model.ItemLocation.item_id == item_id,
or_(*get_tag_filter(polygon.c.tags, tag_list)),
)
)
.group_by(polygon.c.osm_id, polygon.c.tags)
.having(
func.ST_Area(func.ST_Collect(polygon.c.way))
< 20 * func.ST_Area(bbox_list[0])
)
)
tables = ([] if item_is_linear_feature else [s_point]) + [s_line, s_polygon] tables = ([] if item_is_linear_feature else [s_point]) + [s_line, s_polygon]
s = select([union(*tables).alias()]).where(dist < max_distance).order_by(dist) s = select([union(*tables).alias()]).where(dist < max_distance).order_by(dist)
@ -695,10 +777,14 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
s = s.where(tags.has_key("name")) s = s.where(tags.has_key("name"))
if "Key:amenity" in tag_list: if "Key:amenity" in tag_list:
s = s.where(and_(tags["amenity"] != "bicycle_parking", s = s.where(
tags["amenity"] != "bicycle_repair_station", and_(
tags["amenity"] != "atm", tags["amenity"] != "bicycle_parking",
tags["amenity"] != "recycling")) tags["amenity"] != "bicycle_repair_station",
tags["amenity"] != "atm",
tags["amenity"] != "recycling",
)
)
if limit: if limit:
s = s.limit(limit) s = s.limit(limit)
@ -730,6 +816,8 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
shape = "area" if table == "polygon" else table shape = "area" if table == "polygon" else table
item_identifier_tags = item.get_identifiers_tags()
cur = { cur = {
"identifier": f"{osm_type}/{osm_id}", "identifier": f"{osm_type}/{osm_id}",
"type": osm_type, "type": osm_type,
@ -748,8 +836,9 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
part_of = [] part_of = []
for bbox in bbox_list: for bbox in bbox_list:
part_of += [i for i in get_part_of(table, src_id, bbox) part_of += [
if i["tags"]["name"] != name] i for i in get_part_of(table, src_id, bbox) if i["tags"]["name"] != name
]
if part_of: if part_of:
cur["part_of"] = part_of cur["part_of"] = part_of
@ -760,9 +849,9 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
return nearby return nearby
def get_item(item_id):
""" Retrieve a Wikidata item, either from the database or from Wikidata. """
def get_item(item_id):
"""Retrieve a Wikidata item, either from the database or from Wikidata."""
item = model.Item.query.get(item_id) item = model.Item.query.get(item_id)
return item or get_and_save_item(f"Q{item_id}") return item or get_and_save_item(f"Q{item_id}")
@ -774,7 +863,7 @@ def get_item_street_addresses(item):
for claim in item.claims["P669"]: for claim in item.claims["P669"]:
qualifiers = claim.get("qualifiers") qualifiers = claim.get("qualifiers")
if not qualifiers or 'P670' not in qualifiers: if not qualifiers or "P670" not in qualifiers:
continue continue
number = qualifiers["P670"][0]["datavalue"]["value"] number = qualifiers["P670"][0]["datavalue"]["value"]
@ -782,24 +871,26 @@ def get_item_street_addresses(item):
street = street_item.label() street = street_item.label()
for q in qualifiers["P670"]: for q in qualifiers["P670"]:
number = q["datavalue"]["value"] number = q["datavalue"]["value"]
address = (f"{number} {street}" address = (
if g.street_number_first f"{number} {street}" if g.street_number_first else f"{street} {number}"
else f"{street} {number}") )
street_address.append(address) street_address.append(address)
return street_address return street_address
def check_is_street_number_first(latlng): def check_is_street_number_first(latlng):
g.street_number_first = is_street_number_first(*latlng) g.street_number_first = is_street_number_first(*latlng)
def item_detail(item): def item_detail(item):
unsupported_relation_types = { unsupported_relation_types = {
'Q194356', # wind farm "Q194356", # wind farm
'Q2175765', # tram stop "Q2175765", # tram stop
} }
locations = [list(i.get_lat_lon()) for i in item.locations] locations = [list(i.get_lat_lon()) for i in item.locations]
if not hasattr(g, 'street_number_first'): if not hasattr(g, "street_number_first"):
g.street_number_first = is_street_number_first(*locations[0]) g.street_number_first = is_street_number_first(*locations[0])
image_filenames = item.get_claim("P18") image_filenames = item.get_claim("P18")
@ -809,20 +900,24 @@ def item_detail(item):
heritage_designation = [] heritage_designation = []
for v in item.get_claim("P1435"): for v in item.get_claim("P1435"):
if not v: if not v:
print('heritage designation missing:', item.qid) print("heritage designation missing:", item.qid)
continue continue
heritage_designation_item = get_item(v["numeric-id"]) heritage_designation_item = get_item(v["numeric-id"])
heritage_designation.append({ heritage_designation.append(
"qid": v["id"], {
"label": heritage_designation_item.label(), "qid": v["id"],
}) "label": heritage_designation_item.label(),
}
)
isa_items = [get_item(isa["numeric-id"]) for isa in item.get_isa()] isa_items = [get_item(isa["numeric-id"]) for isa in item.get_isa()]
isa_lookup = {isa.qid: isa for isa in isa_items} isa_lookup = {isa.qid: isa for isa in isa_items}
wikipedia_links = [{"lang": site[:-4], "title": link["title"]} wikipedia_links = [
for site, link in sorted(item.sitelinks.items()) {"lang": site[:-4], "title": link["title"]}
if site.endswith("wiki") and len(site) < 8] for site, link in sorted(item.sitelinks.items())
if site.endswith("wiki") and len(site) < 8
]
d = { d = {
"qid": item.qid, "qid": item.qid,
@ -831,7 +926,9 @@ def item_detail(item):
"markers": locations, "markers": locations,
"image_list": image_filenames, "image_list": image_filenames,
"street_address": street_address, "street_address": street_address,
"isa_list": [{"qid": isa.qid, "label": isa.label()} for isa in isa_items if isa], "isa_list": [
{"qid": isa.qid, "label": isa.label()} for isa in isa_items if isa
],
"closed": item.closed(), "closed": item.closed(),
"inception": item.time_claim("P571"), "inception": item.time_claim("P571"),
"p1619": item.time_claim("P1619"), "p1619": item.time_claim("P1619"),
@ -849,8 +946,9 @@ def item_detail(item):
unsupported = isa_lookup.keys() & unsupported_relation_types unsupported = isa_lookup.keys() & unsupported_relation_types
if unsupported: if unsupported:
d["unsupported_relation_types"] = [isa for isa in d["isa_list"] d["unsupported_relation_types"] = [
if isa["qid"] in isa_lookup] isa for isa in d["isa_list"] if isa["qid"] in isa_lookup
]
return d return d
@ -889,7 +987,7 @@ def wikidata_items(bounds, isa_filter=None):
} }
isa_count.append(isa) isa_count.append(isa)
return {'items': items, 'isa_count': isa_count} return {"items": items, "isa_count": isa_count}
def missing_wikidata_items(qids, lat, lon): def missing_wikidata_items(qids, lat, lon):
@ -924,12 +1022,13 @@ def missing_wikidata_items(qids, lat, lon):
return dict(items=items, isa_count=isa_count) return dict(items=items, isa_count=isa_count)
def isa_incremental_search(search_terms): def isa_incremental_search(search_terms):
en_label = func.jsonb_extract_path_text(model.Item.labels, "en", "value") en_label = func.jsonb_extract_path_text(model.Item.labels, "en", "value")
q = model.Item.query.filter( q = model.Item.query.filter(
model.Item.claims.has_key("P1282"), model.Item.claims.has_key("P1282"),
en_label.ilike(f"%{search_terms}%"), en_label.ilike(f"%{search_terms}%"),
func.length(en_label) < 20, func.length(en_label) < 20,
) )
print(q.statement.compile(compile_kwargs={"literal_binds": True})) print(q.statement.compile(compile_kwargs={"literal_binds": True}))
@ -943,13 +1042,18 @@ def isa_incremental_search(search_terms):
ret.append(cur) ret.append(cur)
return ret return ret
def get_place_items(osm_type, osm_id):
src_id = osm_id * {'way': 1, 'relation': -1}[osm_type]
q = (model.Item.query def get_place_items(osm_type, osm_id):
.join(model.ItemLocation) src_id = osm_id * {"way": 1, "relation": -1}[osm_type]
.join(model.Polygon, func.ST_Covers(model.Polygon.way, model.ItemLocation.location))
.filter(model.Polygon.src_id == src_id)) q = (
model.Item.query.join(model.ItemLocation)
.join(
model.Polygon,
func.ST_Covers(model.Polygon.way, model.ItemLocation.location),
)
.filter(model.Polygon.src_id == src_id)
)
# sql = q.statement.compile(compile_kwargs={"literal_binds": True}) # sql = q.statement.compile(compile_kwargs={"literal_binds": True})
item_count = q.count() item_count = q.count()

26
matcher/commons.py
View file

@ -1,18 +1,25 @@
import requests """Use mediawiki API to look up images on Wikimedia Commons."""
import urllib.parse import urllib.parse
from . import utils from typing import Any
import requests
from . import CallParams, utils
commons_start = "http://commons.wikimedia.org/wiki/Special:FilePath/" commons_start = "http://commons.wikimedia.org/wiki/Special:FilePath/"
commons_url = "https://www.wikidata.org/w/api.php" commons_url = "https://www.wikidata.org/w/api.php"
page_size = 50 page_size = 50
def commons_uri_to_filename(uri): def commons_uri_to_filename(uri: str) -> str:
"""Given the URI for a file on commons return the filename of the file."""
return urllib.parse.unquote(utils.drop_start(uri, commons_start)) return urllib.parse.unquote(utils.drop_start(uri, commons_start))
def api_call(params): def api_call(params: CallParams) -> requests.Response:
call_params = { """Call the Commons API."""
call_params: CallParams = {
"format": "json", "format": "json",
"formatversion": 2, "formatversion": 2,
**params, **params,
@ -21,8 +28,11 @@ def api_call(params):
return requests.get(commons_url, params=call_params, timeout=5) return requests.get(commons_url, params=call_params, timeout=5)
def image_detail(filenames, thumbheight=None, thumbwidth=None): def image_detail(
params = { filenames: list[str], thumbheight: int | None = None, thumbwidth: int | None = None
) -> dict[str, Any]:
"""Detail for multiple images."""
params: CallParams = {
"action": "query", "action": "query",
"prop": "imageinfo", "prop": "imageinfo",
"iiprop": "url", "iiprop": "url",
@ -32,7 +42,7 @@ def image_detail(filenames, thumbheight=None, thumbwidth=None):
if thumbwidth is not None: if thumbwidth is not None:
params["iiurlwidth"] = thumbwidth params["iiurlwidth"] = thumbwidth
images = {} images: dict[str, Any] = {}
for cur in utils.chunk(filenames, page_size): for cur in utils.chunk(filenames, page_size):
call_params = params.copy() call_params = params.copy()

39
matcher/database.py Normal file
View file

@ -0,0 +1,39 @@
"""Database functions."""
import flask
import sqlalchemy
from sqlalchemy import create_engine, func
from sqlalchemy.engine import reflection
from sqlalchemy.orm import scoped_session, sessionmaker
session: sqlalchemy.orm.scoping.scoped_session = scoped_session(sessionmaker())
def init_db(db_url: str, echo: bool = False) -> None:
"""Initialise database."""
session.configure(bind=get_engine(db_url, echo=echo))
def get_engine(db_url: str, echo: bool = False) -> sqlalchemy.engine.base.Engine:
"""Create an engine objcet."""
return create_engine(db_url, pool_recycle=3600, echo=echo)
def get_tables() -> list[str]:
"""Get a list of table names."""
tables: list[str] = reflection.Inspector.from_engine(session.bind).get_table_names()
return tables
def init_app(app: flask.app.Flask, echo: bool = False) -> None:
"""Initialise database connection within flask app."""
db_url = app.config["DB_URL"]
session.configure(bind=get_engine(db_url, echo=echo))
@app.teardown_appcontext
def shutdown_session(exception: Exception | None = None) -> None:
session.remove()
def now_utc():
return func.timezone("utc", func.now())

71
matcher/edit.py Normal file
View file

@ -0,0 +1,71 @@
from flask import g
from . import user_agent_headers, database, osm_oauth, mail
from .model import Changeset
import requests
import html
really_save = True
osm_api_base = "https://api.openstreetmap.org/api/0.6"
def new_changeset(comment):
return f"""
<osm>
<changeset>
<tag k="created_by" v="https://map.osm.wikidata.link/"/>
<tag k="comment" v="{html.escape(comment)}"/>
</changeset>
</osm>"""
def osm_request(path, **kwargs):
return osm_oauth.api_put_request(path, **kwargs)
def create_changeset(changeset):
try:
return osm_request("/changeset/create", data=changeset.encode("utf-8"))
except requests.exceptions.HTTPError as r:
print(changeset)
print(r.response.text)
raise
def close_changeset(changeset_id):
return osm_request(f"/changeset/{changeset_id}/close")
def save_element(osm_type, osm_id, element_data):
osm_path = f"/{osm_type}/{osm_id}"
r = osm_request(osm_path, data=element_data)
reply = r.text.strip()
if reply.isdigit():
return r
subject = f"matcher error saving element: {osm_path}"
username = g.user.username
body = f"""
https://www.openstreetmap.org{osm_path}
user: {username}
message user: https://www.openstreetmap.org/message/new/{username}
error:
{reply}
"""
mail.send_mail(subject, body)
def record_changeset(**kwargs):
change = Changeset(created=database.now_utc(), **kwargs)
database.session.add(change)
database.session.commit()
return change
def get_existing(osm_type, osm_id):
url = f"{osm_api_base}/{osm_type}/{osm_id}"
return requests.get(url, headers=user_agent_headers())

22
matcher/error_mail.py
View file

@ -1,13 +1,19 @@
"""Send mail to admins when there is an error."""
import logging import logging
from logging.handlers import SMTPHandler
from logging import Formatter from logging import Formatter
from flask import request from logging.handlers import SMTPHandler
import flask
PROJECT = "osm-wikidata" PROJECT = "osm-wikidata"
class MatcherSMTPHandler(SMTPHandler): class MatcherSMTPHandler(SMTPHandler):
def getSubject(self, record): # noqa: N802 """Custom SMTP handler to change subject line."""
def getSubject(self, record: logging.LogRecord) -> str: # noqa: N802
"""Return subject line for error mail."""
return ( return (
f"{PROJECT} error: {record.exc_info[0].__name__}" f"{PROJECT} error: {record.exc_info[0].__name__}"
if (record.exc_info and record.exc_info[0]) if (record.exc_info and record.exc_info[0])
@ -16,12 +22,16 @@ class MatcherSMTPHandler(SMTPHandler):
class RequestFormatter(Formatter): class RequestFormatter(Formatter):
def format(self, record): """Custom request formatter."""
record.request = request
def format(self, record: logging.LogRecord) -> str:
"""Add request to log record."""
record.request = flask.request
return super().format(record) return super().format(record)
def setup_error_mail(app): def setup_error_mail(app: flask.Flask) -> None:
"""Configure logging to catch errors and email them."""
if not app.config.get("ERROR_MAIL"): if not app.config.get("ERROR_MAIL"):
return return
formatter = RequestFormatter( formatter = RequestFormatter(

18
matcher/planet.py
View file

@ -1,24 +1,32 @@
from sqlalchemy import Table, Column, Integer, String, Float, MetaData """Planet tables."""
from sqlalchemy.dialects import postgresql
from geoalchemy2 import Geometry from geoalchemy2 import Geometry
from sqlalchemy import Column, Float, Integer, MetaData, String, Table
from sqlalchemy.dialects import postgresql
metadata = MetaData() metadata = MetaData()
point = Table("planet_osm_point", metadata, point = Table(
"planet_osm_point",
metadata,
Column("osm_id", Integer), Column("osm_id", Integer),
Column("name", String), Column("name", String),
Column("tags", postgresql.HSTORE), Column("tags", postgresql.HSTORE),
Column("way", Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False), Column("way", Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False),
) )
line = Table("planet_osm_line", metadata, line = Table(
"planet_osm_line",
metadata,
Column("osm_id", Integer), Column("osm_id", Integer),
Column("name", String), Column("name", String),
Column("tags", postgresql.HSTORE), Column("tags", postgresql.HSTORE),
Column("way", Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False), Column("way", Geometry("GEOMETRY", srid=4326, spatial_index=True), nullable=False),
) )
polygon = Table("planet_osm_polygon", metadata, polygon = Table(
"planet_osm_polygon",
metadata,
Column("osm_id", Integer), Column("osm_id", Integer),
Column("name", String), Column("name", String),
Column("tags", postgresql.HSTORE), Column("tags", postgresql.HSTORE),

168
matcher/utils.py
View file

@ -1,113 +1,139 @@
from flask import current_app, request """Utility functions."""
from itertools import islice
import os.path
import json import json
import math import math
import user_agents import os.path
import re import re
import typing
from datetime import date from datetime import date
from itertools import islice
from typing import Any, cast
import flask
import user_agents
from num2words import num2words from num2words import num2words
metres_per_mile = 1609.344 metres_per_mile = 1609.344
feet_per_metre = 3.28084 feet_per_metre = 3.28084
feet_per_mile = 5280 feet_per_mile = 5280
T = typing.TypeVar("T")
def chunk(it, size):
def chunk(it: typing.Iterable[T], size: int) -> typing.Iterator[tuple[T, ...]]:
"""Split an iterable into chunks of the given size."""
it = iter(it) it = iter(it)
return iter(lambda: tuple(islice(it, size)), ()) return iter(lambda: tuple(islice(it, size)), ())
def flatten(l): def flatten(top_list: list[list[T]]) -> list[T]:
return [item for sublist in l for item in sublist] """Flatten a list."""
return [item for sub_list in top_list for item in sub_list]
def drop_start(s, start): def drop_start(s: str, start: str) -> str:
"""Remove string prefix, otherwise throw an error."""
assert s.startswith(start) assert s.startswith(start)
return s[len(start) :] return s[len(start) :]
def remove_start(s, start): def remove_start(s: str, start: str) -> str:
"""Remove a string prefix, if present."""
return s[len(start) :] if s.startswith(start) else s return s[len(start) :] if s.startswith(start) else s
def normalize_url(url): def normalize_url(url: str) -> str:
"""Standardize URLs to help in comparison."""
for start in "http://", "https://", "www.": for start in "http://", "https://", "www.":
url = remove_start(url, start) url = remove_start(url, start)
return url.rstrip("/") return url.rstrip("/")
def contains_digit(s): def contains_digit(s: str) -> bool:
"""Check if string contains a digit."""
return any(c.isdigit() for c in s) return any(c.isdigit() for c in s)
def cache_dir(): def cache_dir() -> str:
return current_app.config["CACHE_DIR"] """Get cache dir location."""
d: str = flask.current_app.config["CACHE_DIR"]
return d
def cache_filename(filename): def cache_filename(filename: str) -> str:
"""Get absolute path for cache file."""
return os.path.join(cache_dir(), filename) return os.path.join(cache_dir(), filename)
def load_from_cache(filename): def load_from_cache(filename: str) -> Any:
"""Load JSON data from cache."""
return json.load(open(cache_filename(filename))) return json.load(open(cache_filename(filename)))
def get_radius(default=1000): def get_radius(default: int = 1000) -> int | None:
arg_radius = request.args.get("radius") """Get radius request argument with default."""
arg_radius = flask.request.args.get("radius")
return int(arg_radius) if arg_radius and arg_radius.isdigit() else default return int(arg_radius) if arg_radius and arg_radius.isdigit() else default
def get_int_arg(name): def get_int_arg(name: str) -> int | None:
if name in request.args and request.args[name].isdigit(): """Get an request arg and convert to integer."""
return int(request.args[name]) v = flask.request.args.get(name)
return int(v) if v and v.isdigit() else None
def calc_chunk_size(area_in_sq_km, size=22): def calc_chunk_size(area_in_sq_km: float, size: int = 22) -> int:
"""Work out the size of a chunk."""
side = math.sqrt(area_in_sq_km) side = math.sqrt(area_in_sq_km)
return max(1, math.ceil(side / size)) return max(1, math.ceil(side / size))
def file_missing_or_empty(filename): def file_missing_or_empty(filename: str) -> bool:
"""Check if a file is missing or empty."""
return os.path.exists(filename) or os.stat(filename).st_size == 0 return os.path.exists(filename) or os.stat(filename).st_size == 0
def is_bot(): def is_bot() -> bool:
""" Is the current request from a web robot? """ """Is the current request from a web robot."""
ua = request.headers.get("User-Agent") ua = flask.request.headers.get("User-Agent")
return ua and user_agents.parse(ua).is_bot return bool(ua and user_agents.parse(ua).is_bot)
def log_location(): def log_location() -> str:
return current_app.config["LOG_DIR"] """Get log location from Flask config."""
return cast(str, flask.current_app.config["LOG_DIR"])
def good_location(): def capfirst(value: str) -> str:
return os.path.join(log_location(), "complete") """Uppercase first letter of string, leave rest as is."""
def capfirst(value):
""" Uppercase first letter of string, leave rest as is. """
return value[0].upper() + value[1:] if value else value return value[0].upper() + value[1:] if value else value
def any_upper(value): def any_upper(value: str) -> bool:
"""Check if string contains any uppercase characters."""
return any(c.isupper() for c in value) return any(c.isupper() for c in value)
def find_log_file(place): def get_free_space(config: flask.config.Config) -> int:
start = f"{place.place_id}_" """Return the amount of available free space."""
for f in os.scandir(good_location()):
if f.name.startswith(start):
return f.path
def get_free_space(config):
s = os.statvfs(config["FREE_SPACE_PATH"]) s = os.statvfs(config["FREE_SPACE_PATH"])
return s.f_bsize * s.f_bavail return s.f_bsize * s.f_bavail
def display_distance(units, dist): def metric_display_distance(units: str, dist: float) -> str | None:
"""Convert distance from metres to the specified metric units."""
if units == "km_and_metres":
units = "km" if dist > 500 else "metres"
if units == "metres":
return f"{dist:,.0f} m"
if units == "km":
return f"{dist / 1000:,.2f} km"
return None
def display_distance(units: str, dist: float) -> str | None:
"""Convert distance from metres to the specified units."""
if units in ("miles_and_feet", "miles_and_yards"): if units in ("miles_and_feet", "miles_and_yards"):
total_feet = dist * feet_per_metre total_feet = dist * feet_per_metre
miles = total_feet / feet_per_mile miles = total_feet / feet_per_mile
@ -124,20 +150,15 @@ def display_distance(units, dist):
miles = dist / metres_per_mile miles = dist / metres_per_mile
return f"{miles:,.2f} miles" if miles > 0.5 else f"{dist:,.0f} metres" return f"{miles:,.2f} miles" if miles > 0.5 else f"{dist:,.0f} metres"
if units == "km_and_metres": return metric_display_distance(units, dist)
units = "km" if dist > 500 else "metres"
if units == "metres":
return f"{dist:,.0f} m"
if units == "km":
return f"{dist / 1000:,.2f} km"
re_range = re.compile(r"\b(\d+) ?(?:to|-) ?(\d+)\b", re.I) def is_in_range(address_range: str, address: str) -> bool:
re_number_list = re.compile(r"\b([\d, ]+) (?:and|&) (\d+)\b", re.I) """Check if an address is within a range."""
re_number = re.compile(r"^(?:No\.?|Number)? ?(\d+)\b") re_range = re.compile(r"\b(\d+) ?(?:to|-) ?(\d+)\b", re.I)
re_number_list = re.compile(r"\b([\d, ]+) (?:and|&) (\d+)\b", re.I)
re_number = re.compile(r"^(?:No\.?|Number)? ?(\d+)\b")
def is_in_range(address_range, address):
m_number = re_number.match(address) m_number = re_number.match(address)
if not m_number: if not m_number:
return False return False
@ -159,20 +180,27 @@ def is_in_range(address_range, address):
return False return False
def format_wikibase_time(v): class WikibaseTime(typing.TypedDict):
p = v["precision"] """Wikibase Time dict."""
precision: int
time: str
def format_wikibase_time(v: WikibaseTime) -> str | None:
"""Format wikibase time value into human readable string."""
t = v["time"] t = v["time"]
# TODO: handle dates with century precision (7) match v["precision"]:
# example: https://www.wikidata.org/wiki/Q108266998 case 11: # year, month and day
return date.fromisoformat(t[1:11]).strftime("%-d %B %Y")
if p == 11: case 10: # year and month
return date.fromisoformat(t[1:11]).strftime("%-d %B %Y") return date.fromisoformat(t[1:8] + "-01").strftime("%B %Y")
if p == 10: case 9: # year
return date.fromisoformat(t[1:8] + "-01").strftime("%B %Y") return t[1:5]
if p == 9: case 7: # century
return t[1:5] century = ((int(t[:5]) - 1) // 100) + 1
if p == 7: ordinal_num: str = num2words(abs(century), to="ordinal_num")
century = ((int(t[:5]) - 1) // 100) + 1 return f"{ordinal_num} {century}{' BC' if century < 0 else ''}"
end = " BC" if century < 0 else "" case _: # not handled
return num2words(abs(century), to="ordinal_num") + " century" + end return None

6
package.json
View file

@ -6,12 +6,12 @@
"test": "echo \"This template does not include a test runner by default.\" && exit 1" "test": "echo \"This template does not include a test runner by default.\" && exit 1"
}, },
"dependencies": { "dependencies": {
"bootstrap": "^5.1.3", "@popperjs/core": "^2.11.0",
"fork-awesome": "^1.2.0", "fork-awesome": "^1.2.0",
"leaflet": "^1.7.1", "leaflet": "^1.8.0",
"leaflet-extra-markers": "^1.2.1", "leaflet-extra-markers": "^1.2.1",
"redaxios": "^0.4.1", "redaxios": "^0.4.1",
"vue": "^3.1.15" "vue": "^3.2.26"
}, },
"devDependencies": { "devDependencies": {
"@snowpack/plugin-dotenv": "^2.1.0", "@snowpack/plugin-dotenv": "^2.1.0",

4
snowpack.config.mjs
View file

@ -1,9 +1,7 @@
import pkg from './package.json';
/** @type {import("snowpack").SnowpackUserConfig } */ /** @type {import("snowpack").SnowpackUserConfig } */
export default { export default {
mount: { mount: {
// public: {url: '/', static: true}, public: {url: '/', static: true},
frontend: {url: '/dist'}, frontend: {url: '/dist'},
}, },
plugins: [ plugins: [

2
templates/isa.html
View file

@ -14,7 +14,7 @@
</a></div> </a></div>
<div class="my-2"> <div class="my-2">
<form method="POST"> <form method="GET" action="{{ url_for("refresh_item", item_id=item.item_id) }}">
<input type="hidden" name="action" value="refresh"> <input type="hidden" name="action" value="refresh">
<input type="submit" value="refresh item" class="btn btn-sm btn-primary"> <input type="submit" value="refresh item" class="btn btn-sm btn-primary">
</form> </form>

7
templates/map.html
View file

@ -4,10 +4,13 @@
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Wikidata items linked to OSM</title> <title>Wikidata items linked to OSM</title>
<!--
<link rel="stylesheet" href="https://unpkg.com/bootstrap@5.1.3/dist/css/bootstrap.min.css"> <link rel="stylesheet" href="https://unpkg.com/bootstrap@5.1.3/dist/css/bootstrap.min.css">
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css">
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.8.0/dist/leaflet.css">
<link rel="stylesheet" href="https://unpkg.com/fork-awesome@1.2.0/css/fork-awesome.min.css"> <link rel="stylesheet" href="https://unpkg.com/fork-awesome@1.2.0/css/fork-awesome.min.css">
<link rel="stylesheet" href="https://unpkg.com/leaflet-extra-markers@1.2.1/dist/css/leaflet.extra-markers.min.css"> <link rel="stylesheet" href="https://unpkg.com/leaflet-extra-markers@1.2.1/dist/css/leaflet.extra-markers.min.css">
-->
<link rel="stylesheet" href="{{ url_for("static", filename="frontend/style.css") }}"> <link rel="stylesheet" href="{{ url_for("static", filename="frontend/style.css") }}">
</head> </head>
@ -16,7 +19,7 @@
{% block nav %}{{ navbar() }}{% endblock %} {% block nav %}{{ navbar() }}{% endblock %}
<div id="app"></div> <div id="app"></div>
<script src="https://unpkg.com/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js" crossorigin="anonymous"></script> <!-- <script src="https://unpkg.com/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js" crossorigin="anonymous"></script> -->
<script type="module"> <script type="module">
import main from {{ url_for('static', filename='frontend/owl.es.js') | tojson }}; import main from {{ url_for('static', filename='frontend/owl.es.js') | tojson }};

432
web_view.py
View file

@ -1,36 +1,61 @@
#!/usr/bin/python3.9 #!/usr/bin/python3
from flask import (Flask, render_template, request, jsonify, redirect, url_for, g, import json
flash, session, Response, stream_with_context, abort) import re
from time import sleep, time
import flask_login
import GeoIP
import maxminddb
import requests
import sqlalchemy
from flask import (
Flask,
Response,
abort,
flash,
g,
jsonify,
redirect,
render_template,
request,
session,
stream_with_context,
url_for,
)
from lxml import etree
from requests_oauthlib import OAuth1Session
from sqlalchemy import func from sqlalchemy import func
from sqlalchemy.sql.expression import update from sqlalchemy.sql.expression import update
from matcher import (nominatim, model, database, commons, wikidata, wikidata_api,
osm_oauth, edit, mail, api, error_mail) from matcher import (
from werkzeug.debug.tbtools import get_current_traceback api,
commons,
database,
edit,
error_mail,
mail,
model,
nominatim,
osm_oauth,
wikidata,
wikidata_api,
)
from matcher.data import property_map from matcher.data import property_map
from time import time, sleep
from requests_oauthlib import OAuth1Session # from werkzeug.debug.tbtools import get_current_traceback
from lxml import etree
import werkzeug.exceptions
import inspect
import flask_login
import requests
import json
import GeoIP
import re
import maxminddb
srid = 4326 srid = 4326
re_point = re.compile(r'^POINT\((.+) (.+)\)$') re_point = re.compile(r"^POINT\((.+) (.+)\)$")
app = Flask(__name__) app = Flask(__name__)
app.debug = True app.debug = True
app.config.from_object('config.default') app.config.from_object("config.default")
error_mail.setup_error_mail(app) error_mail.setup_error_mail(app)
login_manager = flask_login.LoginManager(app) login_manager = flask_login.LoginManager(app)
login_manager.login_view = 'login_route' login_manager.login_view = "login_route"
osm_api_base = 'https://api.openstreetmap.org/api/0.6' osm_api_base = "https://api.openstreetmap.org/api/0.6"
maxminddb_reader = maxminddb.open_database(app.config["GEOLITE2"]) maxminddb_reader = maxminddb.open_database(app.config["GEOLITE2"])
@ -38,7 +63,7 @@ DB_URL = "postgresql:///matcher"
database.init_db(DB_URL) database.init_db(DB_URL)
entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"} entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"}
re_qid = re.compile(r'^Q\d+$') re_qid = re.compile(r"^Q\d+$")
@app.teardown_appcontext @app.teardown_appcontext
@ -50,37 +75,40 @@ def shutdown_session(exception=None):
def global_user(): def global_user():
g.user = flask_login.current_user._get_current_object() g.user = flask_login.current_user._get_current_object()
def dict_repr_values(d): def dict_repr_values(d):
return {key: repr(value) for key, value in d.items()} return {key: repr(value) for key, value in d.items()}
@app.errorhandler(werkzeug.exceptions.InternalServerError) # @app.errorhandler(werkzeug.exceptions.InternalServerError)
def exception_handler(e): # def exception_handler(e):
tb = get_current_traceback() # tb = get_current_traceback()
last_frame = next(frame for frame in reversed(tb.frames) if not frame.is_library) # last_frame = next(frame for frame in reversed(tb.frames) if not frame.is_library)
last_frame_args = inspect.getargs(last_frame.code) # last_frame_args = inspect.getargs(last_frame.code)
if request.path.startswith("/api/"): # if request.path.startswith("/api/"):
return cors_jsonify({ # return cors_jsonify({
"success": False, # "success": False,
"error": tb.exception, # "error": tb.exception,
"traceback": tb.plaintext, # "traceback": tb.plaintext,
"locals": dict_repr_values(last_frame.locals), # "locals": dict_repr_values(last_frame.locals),
"last_function": { # "last_function": {
"name": tb.frames[-1].function_name, # "name": tb.frames[-1].function_name,
"args": repr(last_frame_args), # "args": repr(last_frame_args),
}, # },
}), 500 # }), 500
#
# return render_template('show_error.html',
# tb=tb,
# last_frame=last_frame,
# last_frame_args=last_frame_args), 500
return render_template('show_error.html',
tb=tb,
last_frame=last_frame,
last_frame_args=last_frame_args), 500
def cors_jsonify(*args, **kwargs): def cors_jsonify(*args, **kwargs):
response = jsonify(*args, **kwargs) response = jsonify(*args, **kwargs)
response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Origin"] = "*"
return response return response
def check_for_tagged_qids(qids): def check_for_tagged_qids(qids):
tagged = set() tagged = set()
for qid in qids: for qid in qids:
@ -107,12 +135,12 @@ def check_for_tagged_qid(qid):
def geoip_user_record(): def geoip_user_record():
gi = GeoIP.open(app.config["GEOIP_DATA"], GeoIP.GEOIP_STANDARD) gi = GeoIP.open(app.config["GEOIP_DATA"], GeoIP.GEOIP_STANDARD)
remote_ip = request.get('ip', request.remote_addr) remote_ip = request.get("ip", request.remote_addr)
return gi.record_by_addr(remote_ip) return gi.record_by_addr(remote_ip)
def get_user_location(): def get_user_location():
remote_ip = request.args.get('ip', request.remote_addr) remote_ip = request.args.get("ip", request.remote_addr)
maxmind = maxminddb_reader.get(remote_ip) maxmind = maxminddb_reader.get(remote_ip)
return maxmind.get("location") if maxmind else None return maxmind.get("location") if maxmind else None
@ -153,13 +181,15 @@ def isa_page(item_id):
subclass_list = [] subclass_list = []
for s in item.get_claim(subclass_property): for s in item.get_claim(subclass_property):
subclass = api.get_item(s["numeric-id"]) subclass = api.get_item(s["numeric-id"])
subclass_list.append({ subclass_list.append(
"qid": s["id"], {
"item_id": s["numeric-id"], "qid": s["id"],
"label": subclass.label(), "item_id": s["numeric-id"],
"description": subclass.description(), "label": subclass.label(),
"isa_page_url": url_for("isa_page", item_id=s["numeric-id"]), "description": subclass.description(),
}) "isa_page_url": url_for("isa_page", item_id=s["numeric-id"]),
}
)
tags = api.get_tags_for_isa_item(item) tags = api.get_tags_for_isa_item(item)
@ -253,14 +283,16 @@ def map_start_page():
lat, lon = 42.2917, -85.5872 lat, lon = 42.2917, -85.5872
radius = 5 radius = 5
return redirect(url_for( return redirect(
'map_location', url_for(
lat=f'{lat:.5f}', "map_location",
lon=f'{lon:.5f}', lat=f"{lat:.5f}",
zoom=16, lon=f"{lon:.5f}",
radius=radius, zoom=16,
ip=request.args.get('ip'), radius=radius,
)) ip=request.args.get("ip"),
)
)
@app.route("/documentation") @app.route("/documentation")
@ -269,16 +301,14 @@ def documentation_page():
username = user.username if user.is_authenticated else None username = user.username if user.is_authenticated else None
return render_template( return render_template(
"documentation.html", "documentation.html", active_tab="documentation", username=username
active_tab="documentation",
username=username
) )
@app.route("/search") @app.route("/search")
def search_page(): def search_page():
loc = get_user_location() loc = get_user_location()
q = request.args.get('q') q = request.args.get("q")
user = flask_login.current_user user = flask_login.current_user
username = user.username if user.is_authenticated else None username = user.username if user.is_authenticated else None
@ -295,6 +325,7 @@ def search_page():
q=q, q=q,
) )
@app.route("/map/<int:zoom>/<float(signed=True):lat>/<float(signed=True):lon>") @app.route("/map/<int:zoom>/<float(signed=True):lat>/<float(signed=True):lon>")
def map_location(zoom, lat, lon): def map_location(zoom, lat, lon):
qid = request.args.get("item") qid = request.args.get("item")
@ -327,6 +358,37 @@ def map_location(zoom, lat, lon):
item_type_filter=isa_list, item_type_filter=isa_list,
) )
@app.route("/item/Q<int:item_id>")
def lookup_item(item_id):
item = api.get_item(item_id)
if not item:
# TODO: show nicer page for Wikidata item not found
return abort(404)
try:
lat, lon = item.locations[0].get_lat_lon()
except IndexError:
# TODO: show nicer page for Wikidata item without coordinates
return abort(404)
return render_template(
"map.html",
active_tab="map",
zoom=16,
lat=lat,
lon=lon,
username=get_username(),
mode="map",
q=None,
qid=item.qid,
item_type_filter=[],
)
url = url_for("map_location", zoom=16, lat=lat, lon=lon, item=item.qid)
return redirect(url)
@app.route("/item/Q<int:item_id>") @app.route("/item/Q<int:item_id>")
def lookup_item(item_id): def lookup_item(item_id):
item = api.get_item(item_id) item = api.get_item(item_id)
@ -396,10 +458,12 @@ def old_search_page():
def read_bounds_param(): def read_bounds_param():
return [float(i) for i in request.args["bounds"].split(",")] return [float(i) for i in request.args["bounds"].split(",")]
def read_isa_filter_param(): def read_isa_filter_param():
isa_param = request.args.get('isa') isa_param = request.args.get("isa")
if isa_param: if isa_param:
return set(qid.strip() for qid in isa_param.upper().split(',')) return set(qid.strip() for qid in isa_param.upper().split(","))
@app.route("/api/1/location") @app.route("/api/1/location")
def show_user_location(): def show_user_location():
@ -415,6 +479,7 @@ def api_wikidata_items_count():
t1 = time() - t0 t1 = time() - t0
return cors_jsonify(success=True, count=count, duration=t1) return cors_jsonify(success=True, count=count, duration=t1)
@app.route("/api/1/isa_search") @app.route("/api/1/isa_search")
def api_isa_search(): def api_isa_search():
t0 = time() t0 = time()
@ -450,6 +515,7 @@ def api_wikidata_items():
t1 = time() - t0 t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **ret) return cors_jsonify(success=True, duration=t1, **ret)
@app.route("/api/1/place/<osm_type>/<int:osm_id>") @app.route("/api/1/place/<osm_type>/<int:osm_id>")
def api_place_items(osm_type, osm_id): def api_place_items(osm_type, osm_id):
t0 = time() t0 = time()
@ -476,9 +542,7 @@ def api_get_item(item_id):
detail = api.item_detail(item) detail = api.item_detail(item)
t1 = time() - t0 t1 = time() - t0
return cors_jsonify(success=True, return cors_jsonify(success=True, duration=t1, **detail)
duration=t1,
**detail)
@app.route("/api/1/item/Q<int:item_id>/tags") @app.route("/api/1/item/Q<int:item_id>/tags")
@ -489,25 +553,23 @@ def api_get_item_tags(item_id):
osm_list = sorted(tags.keys()) osm_list = sorted(tags.keys())
t1 = time() - t0 t1 = time() - t0
return cors_jsonify(success=True, return cors_jsonify(
qid=item.qid, success=True, qid=item.qid, tag_or_key_list=osm_list, tag_src=tags, duration=t1
tag_or_key_list=osm_list, )
tag_src=tags,
duration=t1)
def expand_street_name(from_names): def expand_street_name(from_names):
ret = set(from_names) ret = set(from_names)
for name in from_names: for name in from_names:
if any(name.startswith(st) for st in ('St ', 'St. ')): if any(name.startswith(st) for st in ("St ", "St. ")):
first_space = name.find(' ') first_space = name.find(" ")
ret.add("Saint" + name[first_space:]) ret.add("Saint" + name[first_space:])
if ', ' in name: if ", " in name:
for n in set(ret): for n in set(ret):
comma = n.find(", ") comma = n.find(", ")
ret.add(name[:comma]) ret.add(name[:comma])
elif '/' in name: elif "/" in name:
for n in set(ret): for n in set(ret):
ret.extend(part.strip() for part in n.split("/")) ret.extend(part.strip() for part in n.split("/"))
@ -520,14 +582,12 @@ def api_find_osm_candidates(item_id):
t0 = time() t0 = time()
item = model.Item.query.get(item_id) item = model.Item.query.get(item_id)
if not item: if not item:
return cors_jsonify(success=True, return cors_jsonify(success=True, qid=f"Q{item_id}", error="item doesn't exist")
qid=f'Q{item_id}',
error="item doesn't exist")
if not item.locations: if not item.locations:
return cors_jsonify(success=True, return cors_jsonify(
qid=f'Q{item_id}', success=True, qid=f"Q{item_id}", error="item has no coordinates"
error="item has no coordinates") )
label = item.label() label = item.label()
item_is_street = item.is_street() item_is_street = item.is_street()
@ -545,17 +605,15 @@ def api_find_osm_candidates(item_id):
max_distance = 1_000 max_distance = 1_000
limit = 40 limit = 40
names = None names = None
nearby = api.find_osm_candidates(item, nearby = api.find_osm_candidates(
limit=limit, item, limit=limit, max_distance=max_distance, names=names
max_distance=max_distance, )
names=names)
if (item_is_street or item_is_watercourse) and not nearby: if (item_is_street or item_is_watercourse) and not nearby:
# nearby = [osm for osm in nearby if street_name_match(label, osm)] # nearby = [osm for osm in nearby if street_name_match(label, osm)]
# try again without name filter # try again without name filter
nearby = api.find_osm_candidates(item, limit=100, nearby = api.find_osm_candidates(item, limit=100, max_distance=1_000)
max_distance=1_000)
t1 = time() - t0 t1 = time() - t0
return cors_jsonify( return cors_jsonify(
@ -563,7 +621,7 @@ def api_find_osm_candidates(item_id):
qid=item.qid, qid=item.qid,
nearby=nearby, nearby=nearby,
duration=t1, duration=t1,
max_distance=max_distance max_distance=max_distance,
) )
@ -572,10 +630,12 @@ def api_missing_wikidata_items():
t0 = time() t0 = time()
qids_arg = request.args.get("qids") qids_arg = request.args.get("qids")
if not qids_arg: if not qids_arg:
return cors_jsonify(success=False, return cors_jsonify(
error="required parameter 'qids' is missing", success=False,
items=[], error="required parameter 'qids' is missing",
isa_count=[]) items=[],
isa_count=[],
)
qids = [] qids = []
for qid in qids_arg.upper().split(","): for qid in qids_arg.upper().split(","):
@ -591,10 +651,7 @@ def api_missing_wikidata_items():
ret = api.missing_wikidata_items(qids, lat, lon) ret = api.missing_wikidata_items(qids, lat, lon)
t1 = time() - t0 t1 = time() - t0
return cors_jsonify( return cors_jsonify(success=True, duration=t1, **ret)
success=True,
duration=t1,
**ret)
@app.route("/api/1/search") @app.route("/api/1/search")
@ -605,24 +662,28 @@ def api_search():
hit["name"] = nominatim.get_hit_name(hit) hit["name"] = nominatim.get_hit_name(hit)
hit["label"] = nominatim.get_hit_label(hit) hit["label"] = nominatim.get_hit_label(hit)
hit["address"] = list(hit["address"].items()) hit["address"] = list(hit["address"].items())
hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}" if "osm_type" in hit and "osm_id" in hit:
hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}"
else:
print(hit)
print(q)
return cors_jsonify(success=True, hits=hits) return cors_jsonify(success=True, hits=hits)
@app.route("/api/1/polygon/<osm_type>/<int:osm_id>") @app.route("/api/1/polygon/<osm_type>/<int:osm_id>")
def api_polygon(osm_type, osm_id): def api_polygon(osm_type, osm_id):
obj = model.Polygon.get_osm(osm_type, osm_id) obj = model.Polygon.get_osm(osm_type, osm_id)
return cors_jsonify(successful=True, return cors_jsonify(
osm_type=osm_type, successful=True, osm_type=osm_type, osm_id=osm_id, geojson=obj.geojson()
osm_id=osm_id, )
geojson=obj.geojson())
@app.route("/refresh/Q<int:item_id>") @app.route("/refresh/Q<int:item_id>")
def refresh_item(item_id): def refresh_item(item_id):
assert not model.Item.query.get(item_id) assert not model.Item.query.get(item_id)
qid = f'Q{item_id}' qid = f"Q{item_id}"
entity = wikidata_api.get_entity(qid) entity = wikidata_api.get_entity(qid)
entity_qid = entity.pop("id") entity_qid = entity.pop("id")
assert qid == entity_qid assert qid == entity_qid
@ -637,100 +698,110 @@ def refresh_item(item_id):
database.session.add(item) database.session.add(item)
database.session.commit() database.session.commit()
return 'done' return "done"
@app.route('/login')
@app.route("/login")
def login_openstreetmap(): def login_openstreetmap():
return redirect(url_for('start_oauth', return redirect(url_for("start_oauth", next=request.args.get("next")))
next=request.args.get('next')))
@app.route('/logout')
@app.route("/logout")
def logout(): def logout():
next_url = request.args.get('next') or url_for('map_start_page') next_url = request.args.get("next") or url_for("map_start_page")
flask_login.logout_user() flask_login.logout_user()
flash('you are logged out') flash("you are logged out")
return redirect(next_url) return redirect(next_url)
@app.route('/done/')
@app.route("/done/")
def done(): def done():
flash('login successful') flash("login successful")
return redirect(url_for('map_start_page')) return redirect(url_for("map_start_page"))
@app.route('/oauth/start')
@app.route("/oauth/start")
def start_oauth(): def start_oauth():
next_page = request.args.get('next') next_page = request.args.get("next")
if next_page: if next_page:
session['next'] = next_page session["next"] = next_page
client_key = app.config['CLIENT_KEY'] client_key = app.config["CLIENT_KEY"]
client_secret = app.config['CLIENT_SECRET'] client_secret = app.config["CLIENT_SECRET"]
request_token_url = 'https://www.openstreetmap.org/oauth/request_token' request_token_url = "https://www.openstreetmap.org/oauth/request_token"
callback = url_for('oauth_callback', _external=True) callback = url_for("oauth_callback", _external=True)
oauth = OAuth1Session(client_key, oauth = OAuth1Session(
client_secret=client_secret, client_key, client_secret=client_secret, callback_uri=callback
callback_uri=callback) )
fetch_response = oauth.fetch_request_token(request_token_url) fetch_response = oauth.fetch_request_token(request_token_url)
session['owner_key'] = fetch_response.get('oauth_token') session["owner_key"] = fetch_response.get("oauth_token")
session['owner_secret'] = fetch_response.get('oauth_token_secret') session["owner_secret"] = fetch_response.get("oauth_token_secret")
base_authorization_url = 'https://www.openstreetmap.org/oauth/authorize' base_authorization_url = "https://www.openstreetmap.org/oauth/authorize"
authorization_url = oauth.authorization_url(base_authorization_url, authorization_url = oauth.authorization_url(
oauth_consumer_key=client_key) base_authorization_url, oauth_consumer_key=client_key
)
return redirect(authorization_url) return redirect(authorization_url)
@login_manager.user_loader @login_manager.user_loader
def load_user(user_id): def load_user(user_id):
return model.User.query.get(user_id) return model.User.query.get(user_id)
@app.route("/oauth/callback", methods=["GET"]) @app.route("/oauth/callback", methods=["GET"])
def oauth_callback(): def oauth_callback():
client_key = app.config['CLIENT_KEY'] client_key = app.config["CLIENT_KEY"]
client_secret = app.config['CLIENT_SECRET'] client_secret = app.config["CLIENT_SECRET"]
oauth = OAuth1Session(client_key, oauth = OAuth1Session(
client_secret=client_secret, client_key,
resource_owner_key=session['owner_key'], client_secret=client_secret,
resource_owner_secret=session['owner_secret']) resource_owner_key=session["owner_key"],
resource_owner_secret=session["owner_secret"],
)
oauth_response = oauth.parse_authorization_response(request.url) oauth_response = oauth.parse_authorization_response(request.url)
verifier = oauth_response.get('oauth_verifier') verifier = oauth_response.get("oauth_verifier")
access_token_url = 'https://www.openstreetmap.org/oauth/access_token' access_token_url = "https://www.openstreetmap.org/oauth/access_token"
oauth = OAuth1Session(client_key, oauth = OAuth1Session(
client_secret=client_secret, client_key,
resource_owner_key=session['owner_key'], client_secret=client_secret,
resource_owner_secret=session['owner_secret'], resource_owner_key=session["owner_key"],
verifier=verifier) resource_owner_secret=session["owner_secret"],
verifier=verifier,
)
oauth_tokens = oauth.fetch_access_token(access_token_url) oauth_tokens = oauth.fetch_access_token(access_token_url)
session['owner_key'] = oauth_tokens.get('oauth_token') session["owner_key"] = oauth_tokens.get("oauth_token")
session['owner_secret'] = oauth_tokens.get('oauth_token_secret') session["owner_secret"] = oauth_tokens.get("oauth_token_secret")
r = oauth.get(osm_api_base + '/user/details') r = oauth.get(osm_api_base + "/user/details")
info = osm_oauth.parse_userinfo_call(r.content) info = osm_oauth.parse_userinfo_call(r.content)
user = model.User.query.filter_by(osm_id=info['id']).one_or_none() user = model.User.query.filter_by(osm_id=info["id"]).one_or_none()
if user: if user:
user.osm_oauth_token = oauth_tokens.get('oauth_token') user.osm_oauth_token = oauth_tokens.get("oauth_token")
user.osm_oauth_token_secret = oauth_tokens.get('oauth_token_secret') user.osm_oauth_token_secret = oauth_tokens.get("oauth_token_secret")
else: else:
user = model.User( user = model.User(
username=info['username'], username=info["username"],
description=info['description'], description=info["description"],
img=info['img'], img=info["img"],
osm_id=info['id'], osm_id=info["id"],
osm_account_created=info['account_created'], osm_account_created=info["account_created"],
mock_upload=False, mock_upload=False,
) )
database.session.add(user) database.session.add(user)
database.session.commit() database.session.commit()
flask_login.login_user(user) flask_login.login_user(user)
next_page = session.get('next') or url_for('map_start_page') next_page = session.get("next") or url_for("map_start_page")
return redirect(next_page) return redirect(next_page)
@ -738,14 +809,13 @@ def validate_edit_list(edits):
for e in edits: for e in edits:
assert model.Item.get_by_qid(e["qid"]) assert model.Item.get_by_qid(e["qid"])
assert e["op"] in {"add", "remove", "change"} assert e["op"] in {"add", "remove", "change"}
osm_type, _, osm_id = e['osm'].partition('/') osm_type, _, osm_id = e["osm"].partition("/")
osm_id = int(osm_id) osm_id = int(osm_id)
if osm_type == 'node': if osm_type == "node":
assert model.Point.query.get(osm_id) assert model.Point.query.get(osm_id)
else: else:
src_id = osm_id if osm_type == "way" else -osm_id src_id = osm_id if osm_type == "way" else -osm_id
assert (model.Line.query.get(src_id) assert model.Line.query.get(src_id) or model.Polygon.query.get(src_id)
or model.Polygon.query.get(src_id))
@app.route("/api/1/edit", methods=["POST"]) @app.route("/api/1/edit", methods=["POST"])
@ -754,9 +824,9 @@ def api_new_edit_session():
incoming = request.json incoming = request.json
validate_edit_list(incoming["edit_list"]) validate_edit_list(incoming["edit_list"])
es = model.EditSession(user=user, es = model.EditSession(
edit_list=incoming['edit_list'], user=user, edit_list=incoming["edit_list"], comment=incoming["comment"]
comment=incoming['comment']) )
database.session.add(es) database.session.add(es)
database.session.commit() database.session.commit()
@ -764,13 +834,14 @@ def api_new_edit_session():
return cors_jsonify(success=True, session_id=session_id) return cors_jsonify(success=True, session_id=session_id)
@app.route("/api/1/edit/<int:session_id>", methods=["POST"]) @app.route("/api/1/edit/<int:session_id>", methods=["POST"])
def api_edit_session(session_id): def api_edit_session(session_id):
es = model.EditSession.query.get(session_id) es = model.EditSession.query.get(session_id)
assert flask_login.current_user.id == es.user_id assert flask_login.current_user.id == es.user_id
incoming = request.json incoming = request.json
for f in 'edit_list', 'comment': for f in "edit_list", "comment":
if f not in incoming: if f not in incoming:
continue continue
setattr(es, f, incoming[f]) setattr(es, f, incoming[f])
@ -778,21 +849,24 @@ def api_edit_session(session_id):
return cors_jsonify(success=True, session_id=session_id) return cors_jsonify(success=True, session_id=session_id)
class VersionMismatch(Exception): class VersionMismatch(Exception):
pass pass
def osm_object(osm_type, osm_id): def osm_object(osm_type, osm_id):
if osm_type == "node": if osm_type == "node":
return model.Point.query.get(osm_id) return model.Point.query.get(osm_id)
src_id = int(osm_id) * {'way': 1, 'relation': -1}[osm_type] src_id = int(osm_id) * {"way": 1, "relation": -1}[osm_type]
for cls in model.Line, model.Polygon: for cls in model.Line, model.Polygon:
obj = cls.query.get(src_id) obj = cls.query.get(src_id)
if obj: if obj:
return obj return obj
def process_edit(changeset_id, e): def process_edit(changeset_id, e):
osm_type, _, osm_id = e['osm'].partition('/') osm_type, _, osm_id = e["osm"].partition("/")
qid = e["qid"] qid = e["qid"]
item_id = qid[1:] item_id = qid[1:]
@ -845,9 +919,7 @@ def process_edit(changeset_id, e):
cls = type(osm) cls = type(osm)
database.session.execute( database.session.execute(
update(cls). update(cls).where(cls.src_id == osm.src_id).values(tags=new_tags)
where(cls.src_id == osm.src_id).
values(tags=new_tags)
) )
db_edit = model.ChangesetEdit( db_edit = model.ChangesetEdit(
@ -861,6 +933,7 @@ def process_edit(changeset_id, e):
return "saved" return "saved"
@app.route("/api/1/save/<int:session_id>") @app.route("/api/1/save/<int:session_id>")
def api_save_changeset(session_id): def api_save_changeset(session_id):
assert g.user.is_authenticated assert g.user.is_authenticated
@ -870,6 +943,18 @@ def api_save_changeset(session_id):
return api_call(session_id) return api_call(session_id)
@app.route("/sql", methods=["GET", "POST"])
def run_sql():
if request.method != "POST":
return render_template("run_sql.html")
sql = request.form["sql"]
conn = database.session.connection()
result = conn.execute(sqlalchemy.text(sql))
return render_template("run_sql.html", result=result)
def api_real_save_changeset(session_id): def api_real_save_changeset(session_id):
es = model.EditSession.query.get(session_id) es = model.EditSession.query.get(session_id)
@ -920,7 +1005,8 @@ def api_real_save_changeset(session_id):
edit.close_changeset(changeset_id) edit.close_changeset(changeset_id)
yield send("done") yield send("done")
return Response(stream_with_context(stream(g.user)), mimetype='text/event-stream') return Response(stream_with_context(stream(g.user)), mimetype="text/event-stream")
def api_mock_save_changeset(session_id): def api_mock_save_changeset(session_id):
es = model.EditSession.query.get(session_id) es = model.EditSession.query.get(session_id)
@ -930,7 +1016,7 @@ def api_mock_save_changeset(session_id):
return f"data: {json.dumps(data)}\n\n" return f"data: {json.dumps(data)}\n\n"
def stream(user): def stream(user):
print('stream') print("stream")
changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar() changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar()
sleep(1) sleep(1)
yield send("open", id=changeset_id) yield send("open", id=changeset_id)
@ -938,12 +1024,12 @@ def api_mock_save_changeset(session_id):
update_count = 0 update_count = 0
print('record_changeset', changeset_id) print("record_changeset", changeset_id)
edit.record_changeset( edit.record_changeset(
id=changeset_id, user=user, comment=es.comment, update_count=update_count id=changeset_id, user=user, comment=es.comment, update_count=update_count
) )
print('edits') print("edits")
for num, e in enumerate(es.edit_list): for num, e in enumerate(es.edit_list):
print(num, e) print(num, e)
@ -952,12 +1038,12 @@ def api_mock_save_changeset(session_id):
yield send("saved", edit=e, num=num) yield send("saved", edit=e, num=num)
sleep(1) sleep(1)
print('closing') print("closing")
yield send("closing") yield send("closing")
sleep(1) sleep(1)
yield send("done") yield send("done")
return Response(stream(g.user), mimetype='text/event-stream') return Response(stream(g.user), mimetype="text/event-stream")
if __name__ == "__main__": if __name__ == "__main__":