owl-map/web_view.py

1065 lines
29 KiB
Python
Raw Normal View History

2023-05-14 10:17:30 +01:00
#!/usr/bin/python3
2021-05-07 16:46:47 +01:00
2023-05-20 07:35:48 +01:00
"""Views for the web app."""
2024-05-04 09:44:44 +01:00
import inspect
import json
2021-05-15 13:05:44 +01:00
import re
2024-05-04 09:44:44 +01:00
import sys
import traceback
2024-05-04 07:48:22 +01:00
import typing
from time import sleep, time
2023-05-14 18:25:35 +01:00
import flask
2024-05-04 07:48:22 +01:00
import flask_login # type: ignore
import GeoIP # type: ignore
2023-05-14 17:04:53 +01:00
import lxml
2021-07-06 16:24:14 +01:00
import maxminddb
import requests
2023-05-13 14:01:28 +01:00
import sqlalchemy
2024-05-04 07:48:22 +01:00
import werkzeug
from requests_oauthlib import OAuth1Session
from sqlalchemy import func
from sqlalchemy.sql.expression import update
from matcher import (
api,
commons,
database,
edit,
error_mail,
mail,
model,
nominatim,
osm_oauth,
wikidata,
wikidata_api,
)
from matcher.data import property_map
2024-05-04 07:48:22 +01:00
StrDict = dict[str, typing.Any]
2021-05-07 16:46:47 +01:00
srid = 4326
re_point = re.compile(r"^POINT\((.+) (.+)\)$")
2021-05-07 16:46:47 +01:00
2023-05-14 18:25:35 +01:00
app = flask.Flask(__name__)
2021-05-07 16:46:47 +01:00
app.debug = True
app.config.from_object("config.default")
error_mail.setup_error_mail(app)
2021-05-07 16:46:47 +01:00
2021-06-16 14:42:04 +01:00
login_manager = flask_login.LoginManager(app)
login_manager.login_view = "login_route"
osm_api_base = "https://api.openstreetmap.org/api/0.6"
2021-06-16 14:42:04 +01:00
2021-07-06 16:24:14 +01:00
maxminddb_reader = maxminddb.open_database(app.config["GEOLITE2"])
2021-06-16 14:42:04 +01:00
2021-05-07 16:46:47 +01:00
DB_URL = "postgresql:///matcher"
database.init_db(DB_URL)
entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"}
2021-05-07 16:46:47 +01:00
re_qid = re.compile(r"^Q\d+$")
2021-06-16 13:31:58 +01:00
2021-05-07 16:46:47 +01:00
@app.teardown_appcontext
2024-05-04 07:48:22 +01:00
def shutdown_session(exception=None) -> None:
"""Shutdown session."""
2021-05-07 16:46:47 +01:00
database.session.remove()
2021-06-17 18:17:28 +01:00
@app.before_request
2023-05-14 18:25:35 +01:00
def global_user() -> None:
"""Make user object available globally."""
flask.g.user = flask_login.current_user._get_current_object()
2021-06-17 18:17:28 +01:00
def dict_repr_values(d):
return {key: repr(value) for key, value in d.items()}
2024-05-04 09:44:44 +01:00
@app.errorhandler(werkzeug.exceptions.InternalServerError)
def exception_handler(
e: werkzeug.exceptions.InternalServerError,
) -> tuple[str | werkzeug.wrappers.Response, int]:
"""Handle exception."""
exec_type, exc_value, current_traceback = sys.exc_info()
assert exc_value
tb = werkzeug.debug.tbtools.DebugTraceback(exc_value)
# summary = tb.render_traceback_html(include_title=False)
last_frame = list(traceback.walk_tb(current_traceback))[-1][0]
last_frame_args = inspect.getargs(last_frame.f_code)
if flask.request.path.startswith("/api/"):
return (
cors_jsonify(
{
"success": False,
"error": tb._te.exc_type.__name__,
"traceback": tb.render_traceback_text(),
"locals": dict_repr_values(last_frame.f_locals),
"last_function": {
"name": last_frame.f_code.co_name,
"args": repr(last_frame_args),
},
}
),
500,
)
return (
flask.render_template(
"show_error.html",
tb=tb,
last_frame=last_frame,
last_frame_args=last_frame_args,
),
500,
)
2024-05-04 07:48:22 +01:00
def cors_jsonify(*args, **kwargs) -> flask.Response:
2023-05-14 18:25:35 +01:00
"""Add CORS header to JSON."""
response = flask.jsonify(*args, **kwargs)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
2024-05-04 07:48:22 +01:00
def check_for_tagged_qids(qids: list[str]) -> set[str]:
"""Check OSM for existing wikidata tags for given QIDs."""
2021-05-07 16:46:47 +01:00
tagged = set()
for qid in qids:
for cls in model.Point, model.Polygon, model.Line:
q = cls.query.filter(cls.tags["wikidata"] == qid)
if q.count():
tagged.add(qid)
break
return tagged
def check_for_tagged_qid(qid):
return any(
database.session.query(
cls.query.filter(
cls.tags.has_key("wikidata"), cls.tags["wikidata"] == qid
).exists()
).scalar()
for cls in (model.Point, model.Polygon, model.Line)
)
2021-06-25 10:12:50 +01:00
def geoip_user_record():
gi = GeoIP.open(app.config["GEOIP_DATA"], GeoIP.GEOIP_STANDARD)
2021-05-07 16:46:47 +01:00
2024-05-04 07:48:22 +01:00
remote_ip = flask.request.args.get("ip", flask.request.remote_addr)
2021-06-25 10:12:50 +01:00
return gi.record_by_addr(remote_ip)
2024-05-04 07:48:22 +01:00
def get_user_location() -> StrDict | None:
"""Get user location."""
2023-05-14 18:25:35 +01:00
remote_ip = flask.request.args.get("ip", flask.request.remote_addr)
2024-05-04 07:48:22 +01:00
assert remote_ip
2022-04-08 10:37:43 +01:00
maxmind = maxminddb_reader.get(remote_ip)
2024-05-04 07:48:22 +01:00
return typing.cast(StrDict, maxmind.get("location")) if maxmind else None
2021-05-07 16:46:47 +01:00
@app.route("/")
2024-05-04 07:48:22 +01:00
def redirect_from_root() -> werkzeug.wrappers.Response:
"""Redirect from root to map start page."""
2023-05-14 18:25:35 +01:00
return flask.redirect(flask.url_for("map_start_page"))
2021-05-07 16:46:47 +01:00
@app.route("/index")
2024-05-04 07:48:22 +01:00
def index_page() -> str:
"""Index page."""
2023-05-14 18:25:35 +01:00
return flask.render_template("index.html")
2021-05-07 16:46:47 +01:00
2021-10-22 10:02:52 +01:00
2023-05-14 17:05:19 +01:00
def get_username() -> str | None:
"""Username for current user."""
2021-10-22 10:02:52 +01:00
user = flask_login.current_user
2023-05-14 17:05:19 +01:00
return user.username if user.is_authenticated else None
2021-10-22 10:02:52 +01:00
@app.route("/isa/Q<int:item_id>", methods=["GET", "POST"])
2024-05-04 07:48:22 +01:00
def isa_page(item_id: int) -> werkzeug.wrappers.Response | str:
"""Return IsA page."""
2021-10-22 10:02:52 +01:00
item = api.get_item(item_id)
2023-05-14 18:25:35 +01:00
if flask.request.method == "POST":
tag_or_key = flask.request.form["tag_or_key"]
2022-04-08 10:37:43 +01:00
extra = model.ItemExtraKeys(item=item, tag_or_key=tag_or_key)
database.session.add(extra)
database.session.commit()
2023-05-14 18:25:35 +01:00
flask.flash("extra OSM tag/key added")
2022-04-08 10:37:43 +01:00
2024-05-04 07:48:22 +01:00
endpoint = flask.request.endpoint
assert endpoint
return flask.redirect(flask.url_for(endpoint, item_id=item_id))
2021-10-22 10:02:52 +01:00
q = model.ItemExtraKeys.query.filter_by(item=item)
extra = [e.tag_or_key for e in q]
subclass_property = "P279"
subclass_list = []
2024-05-04 07:48:22 +01:00
assert item
2021-10-22 10:02:52 +01:00
for s in item.get_claim(subclass_property):
2024-05-04 07:48:22 +01:00
assert isinstance(s, dict)
subclass_item_id = s["numeric-id"]
assert subclass_item_id and isinstance(subclass_item_id, int)
subclass = api.get_item(subclass_item_id)
assert subclass
subclass_list.append(
{
"qid": s["id"],
2024-05-04 07:48:22 +01:00
"item_id": subclass_item_id,
"label": subclass.label(),
"description": subclass.description(),
2023-05-14 18:25:35 +01:00
"isa_page_url": flask.url_for("isa_page", item_id=s["numeric-id"]),
}
)
2021-10-22 10:02:52 +01:00
2021-11-13 16:42:46 +00:00
tags = api.get_tags_for_isa_item(item)
2023-05-14 18:25:35 +01:00
return flask.render_template(
2021-10-22 10:02:52 +01:00
"isa.html",
item=item,
extra=extra,
subclass_list=subclass_list,
2021-11-13 16:42:46 +00:00
username=get_username(),
tags=tags,
2021-10-22 10:02:52 +01:00
)
2021-06-25 13:52:06 +01:00
@app.route("/admin/skip_isa")
def admin_skip_isa_list():
q = model.Item.query.join(model.SkipIsA).order_by(model.Item.item_id)
2023-05-14 18:25:35 +01:00
return flask.render_template("admin/skip_isa.html", q=q)
2021-06-25 13:52:06 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/identifier")
def identifier_index():
2023-05-14 18:25:35 +01:00
return flask.render_template("identifier_index.html", property_map=property_map)
2021-05-07 16:46:47 +01:00
2021-05-08 09:39:06 +01:00
@app.route("/commons/<filename>")
def get_commons_image(filename):
if filename == "null":
flask.abort(404)
detail = commons.image_detail([filename], thumbheight=1200, thumbwidth=1200)
if filename not in detail:
flask.abort(404)
return flask.redirect(detail[filename]["thumburl"])
2021-05-08 09:39:06 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/identifier/<pid>")
def identifier_page(pid):
per_page = 10
2023-05-14 18:25:35 +01:00
page = int(flask.request.args.get("page", 1))
2021-05-07 16:46:47 +01:00
property_dict = {pid: (osm_keys, label) for pid, osm_keys, label in property_map}
osm_keys, label = property_dict[pid]
wd = model.Item.query.filter(model.Item.claims.has_key(pid))
total = wd.count()
start = per_page * (page - 1)
items = wd.all()[start : per_page * page]
qids = [item.qid for item in items]
print(qids)
# pred = None
# values = set()
# for item in items:
# values |= set(item.get_claim(pid))
#
# for key in osm_keys:
# if key == 'ref':
# continue
# if pred is None:
# pred = model.Point.tags[key].in_(values)
# else:
# pred |= model.Point.tags[key].in_(values)
#
osm_points = {}
for qid in qids:
osm_points[qid] = model.Point.query.filter(
model.Point.tags["wikidata"] == qid
).all()
osm_total = len(osm_points)
2023-05-14 18:25:35 +01:00
return flask.render_template(
2021-05-07 16:46:47 +01:00
"identifier_page.html",
pid=pid,
osm_keys=osm_keys,
label=label,
items=items,
total=total,
osm_total=osm_total,
osm_points=osm_points,
)
@app.route("/map")
2024-05-04 07:48:22 +01:00
def map_start_page() -> werkzeug.wrappers.Response:
"""Map start page."""
2021-07-06 16:24:14 +01:00
loc = get_user_location()
2021-06-17 18:12:07 +01:00
2022-04-08 10:37:43 +01:00
if loc:
lat, lon = loc["latitude"], loc["longitude"]
radius = loc["accuracy_radius"]
else:
lat, lon = 42.2917, -85.5872
radius = 5
2023-05-14 18:25:35 +01:00
return flask.redirect(
flask.url_for(
"map_location",
lat=f"{lat:.5f}",
lon=f"{lon:.5f}",
zoom=16,
radius=radius,
2023-05-14 18:25:35 +01:00
ip=flask.request.args.get("ip"),
)
)
2021-06-14 23:08:07 +01:00
2021-07-11 16:18:45 +01:00
2021-07-14 14:57:21 +01:00
@app.route("/documentation")
2023-05-14 17:05:19 +01:00
def documentation_page() -> str:
"""Documentation."""
2021-07-14 14:57:21 +01:00
user = flask_login.current_user
username = user.username if user.is_authenticated else None
2023-05-14 18:25:35 +01:00
return flask.render_template(
"documentation.html", active_tab="documentation", username=username
2021-07-14 14:57:21 +01:00
)
2021-07-11 16:18:45 +01:00
@app.route("/search")
2023-05-14 17:05:19 +01:00
def search_page() -> str:
"""Search."""
2021-07-11 16:18:45 +01:00
loc = get_user_location()
2024-05-04 07:48:22 +01:00
assert loc
2023-05-14 18:25:35 +01:00
q = flask.request.args.get("q")
2021-07-11 16:18:45 +01:00
user = flask_login.current_user
username = user.username if user.is_authenticated else None
2023-05-14 18:25:35 +01:00
return flask.render_template(
2021-07-11 16:18:45 +01:00
"map.html",
2021-07-14 14:57:21 +01:00
active_tab="map",
2021-07-11 16:18:45 +01:00
lat=f'{loc["latitude"]:.5f}',
lon=f'{loc["longitude"]:.5f}',
zoom=16,
radius=loc["accuracy_radius"],
username=username,
mode="search",
q=q,
)
2021-06-14 23:08:07 +01:00
@app.route("/map/<int:zoom>/<float(signed=True):lat>/<float(signed=True):lon>")
2023-05-20 07:35:48 +01:00
def map_location(zoom: int, lat: float, lon: float) -> str:
2024-05-04 07:48:22 +01:00
"""Map location."""
2023-05-14 18:25:35 +01:00
qid = flask.request.args.get("item")
isa_param = flask.request.args.get("isa")
if qid:
2024-05-04 07:48:22 +01:00
api.get_item(int(qid[1:]))
2021-06-17 18:12:07 +01:00
2022-04-08 10:37:43 +01:00
isa_list = []
if isa_param:
for isa_qid in isa_param.split(";"):
2024-05-04 07:48:22 +01:00
isa = api.get_item(int(isa_qid[1:]))
2022-04-08 10:37:43 +01:00
if not isa:
continue
cur = {
"qid": isa.qid,
"label": isa.label(),
}
isa_list.append(cur)
2023-05-14 18:25:35 +01:00
return flask.render_template(
2021-07-06 16:24:14 +01:00
"map.html",
2021-07-14 14:57:21 +01:00
active_tab="map",
2021-07-06 16:24:14 +01:00
zoom=zoom,
lat=lat,
lon=lon,
2023-05-14 18:25:35 +01:00
radius=flask.request.args.get("radius"),
2021-10-22 10:02:52 +01:00
username=get_username(),
2021-07-11 16:18:45 +01:00
mode="map",
q=None,
2022-04-08 10:37:43 +01:00
item_type_filter=isa_list,
2021-07-06 16:24:14 +01:00
)
2021-06-14 23:08:07 +01:00
2022-04-08 10:37:43 +01:00
@app.route("/item/Q<int:item_id>")
2024-05-04 07:48:22 +01:00
def lookup_item(item_id: int):
"""Lookup item."""
2022-04-08 10:37:43 +01:00
item = api.get_item(item_id)
if not item:
# TODO: show nicer page for Wikidata item not found
2023-05-14 18:25:35 +01:00
return flask.abort(404)
2022-04-08 10:37:43 +01:00
try:
lat, lon = item.locations[0].get_lat_lon()
except IndexError:
# TODO: show nicer page for Wikidata item without coordinates
2023-05-14 18:25:35 +01:00
return flask.abort(404)
2022-04-08 10:37:43 +01:00
2023-05-14 18:25:35 +01:00
return flask.render_template(
2022-04-08 10:37:43 +01:00
"map.html",
active_tab="map",
zoom=16,
lat=lat,
lon=lon,
username=get_username(),
mode="map",
q=None,
qid=item.qid,
item_type_filter=[],
)
2023-05-14 18:25:35 +01:00
url = flask.url_for("map_location", zoom=16, lat=lat, lon=lon, item=item.qid)
return flask.redirect(url)
2022-04-08 10:37:43 +01:00
2021-06-14 23:08:07 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/search/map")
2024-05-04 07:48:22 +01:00
def search_map_page() -> str:
"""Search map page."""
2021-05-07 16:46:47 +01:00
user_lat, user_lon = get_user_location() or (None, None)
2023-05-14 18:25:35 +01:00
q = flask.request.args.get("q")
2021-05-07 16:46:47 +01:00
if not q:
2023-05-14 18:25:35 +01:00
return flask.render_template("map.html", user_lat=user_lat, user_lon=user_lon)
2021-05-07 16:46:47 +01:00
hits = nominatim.lookup(q)
for hit in hits:
if "geotext" in hit:
del hit["geotext"]
bbox = [hit["boundingbox"] for hit in hits]
2023-05-14 18:25:35 +01:00
return flask.render_template(
2021-05-07 16:46:47 +01:00
"search_map.html",
hits=hits,
bbox_list=bbox,
user_lat=user_lat,
user_lon=user_lon,
)
def read_bounds_param() -> list[float]:
"""Read bounds parameter."""
2023-05-14 18:25:35 +01:00
return [float(i) for i in flask.request.args["bounds"].split(",")]
2021-05-07 16:46:47 +01:00
2021-07-30 15:02:41 +01:00
def read_isa_filter_param():
2023-05-14 18:25:35 +01:00
isa_param = flask.request.args.get("isa")
2021-07-30 15:02:41 +01:00
if isa_param:
return set(qid.strip() for qid in isa_param.upper().split(","))
2021-05-07 16:46:47 +01:00
2021-06-25 10:12:50 +01:00
@app.route("/api/1/location")
2024-05-04 07:48:22 +01:00
def show_user_location() -> werkzeug.wrappers.Response:
"""User location."""
2021-06-25 10:12:50 +01:00
return cors_jsonify(get_user_location())
2021-06-17 18:14:42 +01:00
@app.route("/api/1/count")
2024-05-04 07:48:22 +01:00
def api_wikidata_items_count() -> werkzeug.wrappers.Response:
2021-06-17 18:14:42 +01:00
t0 = time()
2021-07-30 15:02:41 +01:00
isa_filter = read_isa_filter_param()
count = api.wikidata_items_count(read_bounds_param(), isa_filter=isa_filter)
2021-06-17 18:14:42 +01:00
t1 = time() - t0
2021-06-25 13:52:42 +01:00
return cors_jsonify(success=True, count=count, duration=t1)
2021-06-17 18:14:42 +01:00
2021-07-30 15:02:41 +01:00
@app.route("/api/1/isa_search")
def api_isa_search():
t0 = time()
2023-05-14 18:25:35 +01:00
search_terms = flask.request.args.get("q")
2021-07-30 15:02:41 +01:00
items = api.isa_incremental_search(search_terms)
t1 = time() - t0
return cors_jsonify(success=True, items=items, duration=t1)
2021-06-17 18:14:42 +01:00
2021-06-24 17:42:33 +01:00
@app.route("/api/1/isa")
def api_wikidata_isa_counts():
t0 = time()
2021-06-25 13:52:42 +01:00
bounds = read_bounds_param()
2021-07-30 15:02:41 +01:00
isa_filter = read_isa_filter_param()
isa_count = api.wikidata_isa_counts(bounds, isa_filter=isa_filter)
2021-06-24 17:42:33 +01:00
t1 = time() - t0
return cors_jsonify(success=True, isa_count=isa_count, bounds=bounds, duration=t1)
2021-06-24 17:42:33 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/api/1/items")
def api_wikidata_items():
t0 = time()
2021-06-25 13:52:42 +01:00
bounds = read_bounds_param()
2021-07-30 15:02:41 +01:00
isa_filter = read_isa_filter_param()
ret = api.wikidata_items(bounds, isa_filter=isa_filter)
2021-05-07 16:46:47 +01:00
2021-06-25 13:52:42 +01:00
t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **ret)
2021-05-07 16:46:47 +01:00
2022-04-18 12:24:16 +01:00
@app.route("/api/1/place/<osm_type>/<int:osm_id>")
def api_place_items(osm_type, osm_id):
t0 = time()
ret = api.get_place_items(osm_type, osm_id)
t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **ret)
2021-05-07 16:46:47 +01:00
@app.route("/api/1/osm")
def api_osm_objects():
t0 = time()
2021-07-30 15:02:41 +01:00
isa_filter = read_isa_filter_param()
objects = api.get_osm_with_wikidata_tag(read_bounds_param(), isa_filter=isa_filter)
2021-05-07 16:46:47 +01:00
t1 = time() - t0
return cors_jsonify(success=True, objects=objects, duration=t1)
2021-05-07 16:46:47 +01:00
@app.route("/api/1/item/Q<int:item_id>")
def api_get_item(item_id):
t0 = time()
item = model.Item.query.get(item_id)
detail = api.item_detail(item)
t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **detail)
2021-05-12 08:27:34 +01:00
@app.route("/api/1/item/Q<int:item_id>/tags")
def api_get_item_tags(item_id):
t0 = time()
item = model.Item.query.get(item_id)
tags = api.get_item_tags(item)
osm_list = sorted(tags.keys())
2021-05-12 08:27:34 +01:00
t1 = time() - t0
return cors_jsonify(
success=True, qid=item.qid, tag_or_key_list=osm_list, tag_src=tags, duration=t1
)
2021-05-12 08:27:34 +01:00
2021-05-14 16:09:56 +01:00
def expand_street_name(from_names):
ret = set(from_names)
for name in from_names:
if any(name.startswith(st) for st in ("St ", "St. ")):
first_space = name.find(" ")
ret.add("Saint" + name[first_space:])
if ", " in name:
for n in set(ret):
comma = n.find(", ")
ret.add(name[:comma])
elif "/" in name:
for n in set(ret):
ret.extend(part.strip() for part in n.split("/"))
ret.update({"The " + name for name in ret if not name.startswith("The ")})
return ret
2021-05-12 15:55:40 +01:00
@app.route("/api/1/item/Q<int:item_id>/candidates")
def api_find_osm_candidates(item_id):
t0 = time()
item = model.Item.query.get(item_id)
if not item:
return cors_jsonify(success=True, qid=f"Q{item_id}", error="item doesn't exist")
2021-10-22 11:31:34 +01:00
if not item.locations:
return cors_jsonify(
success=True, qid=f"Q{item_id}", error="item has no coordinates"
)
2021-10-22 11:31:34 +01:00
label = item.label()
item_is_street = item.is_street()
2021-10-22 11:31:34 +01:00
item_is_watercourse = item.is_watercourse()
if item_is_street:
2021-10-22 11:31:34 +01:00
max_distance = 5_000
limit = None
names = expand_street_name([label] + item.get_aliases())
elif item_is_watercourse:
max_distance = 20_000
limit = None
2021-10-22 11:31:34 +01:00
names = {label}
else:
2021-10-22 11:31:34 +01:00
max_distance = 1_000
limit = 40
names = None
nearby = api.find_osm_candidates(
item, limit=limit, max_distance=max_distance, names=names
)
2021-10-22 11:31:34 +01:00
if (item_is_street or item_is_watercourse) and not nearby:
# nearby = [osm for osm in nearby if street_name_match(label, osm)]
# try again without name filter
nearby = api.find_osm_candidates(item, limit=100, max_distance=1_000)
2021-05-12 15:55:40 +01:00
t1 = time() - t0
return cors_jsonify(
success=True,
qid=item.qid,
nearby=nearby,
duration=t1,
max_distance=max_distance,
)
2021-05-12 15:55:40 +01:00
2021-05-10 13:59:08 +01:00
@app.route("/api/1/missing")
def api_missing_wikidata_items():
t0 = time()
2023-05-14 18:25:35 +01:00
qids_arg = flask.request.args.get("qids")
2021-06-16 13:31:58 +01:00
if not qids_arg:
return cors_jsonify(
success=False,
error="required parameter 'qids' is missing",
items=[],
isa_count=[],
)
2021-06-16 13:31:58 +01:00
qids = []
for qid in qids_arg.upper().split(","):
qid = qid.strip()
m = re_qid.match(qid)
if not m:
continue
qids.append(qid)
if not qids:
2023-05-14 18:25:35 +01:00
return flask.jsonify(success=True, items=[], isa_count=[])
2021-05-10 13:59:08 +01:00
2023-05-14 18:25:35 +01:00
lat, lon = flask.request.args.get("lat"), flask.request.args.get("lon")
2021-05-15 13:05:44 +01:00
2021-06-25 13:52:42 +01:00
ret = api.missing_wikidata_items(qids, lat, lon)
t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **ret)
2021-05-10 13:59:08 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/api/1/search")
def api_search():
2023-05-14 18:25:35 +01:00
q = flask.request.args["q"]
2021-05-07 16:46:47 +01:00
hits = nominatim.lookup(q)
for hit in hits:
hit["name"] = nominatim.get_hit_name(hit)
2021-07-08 13:35:32 +01:00
hit["label"] = nominatim.get_hit_label(hit)
hit["address"] = list(hit["address"].items())
2023-05-13 14:01:28 +01:00
if "osm_type" in hit and "osm_id" in hit:
hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}"
else:
print(hit)
print(q)
2021-06-14 23:09:05 +01:00
return cors_jsonify(success=True, hits=hits)
2021-06-14 23:09:05 +01:00
@app.route("/api/1/polygon/<osm_type>/<int:osm_id>")
def api_polygon(osm_type, osm_id):
obj = model.Polygon.get_osm(osm_type, osm_id)
return cors_jsonify(
successful=True, osm_type=osm_type, osm_id=osm_id, geojson=obj.geojson()
)
2021-06-14 23:09:05 +01:00
@app.route("/refresh/Q<int:item_id>")
2023-05-14 18:25:35 +01:00
def refresh_item(item_id: int) -> str:
"""Refresh the local mirror of a Wikidata item."""
2024-06-19 14:23:20 +01:00
item = model.Item.query.get(item_id)
2021-06-14 23:09:05 +01:00
qid = f"Q{item_id}"
2021-06-14 23:09:05 +01:00
entity = wikidata_api.get_entity(qid)
entity_qid = entity.pop("id")
assert qid == entity_qid
coords = wikidata.get_entity_coords(entity["claims"])
obj = {k: v for k, v in entity.items() if k in entity_keys}
2024-06-19 14:23:20 +01:00
if item:
2024-05-04 07:48:22 +01:00
for k, v in obj.items():
2024-06-19 14:23:20 +01:00
setattr(item, k, v)
2024-05-04 07:48:22 +01:00
else:
item = model.Item(item_id=item_id, **obj)
database.session.add(item)
if coords:
item.locations = model.location_objects(coords)
2021-06-14 23:09:05 +01:00
database.session.commit()
2021-05-07 16:46:47 +01:00
return "done"
2021-05-07 16:46:47 +01:00
@app.route("/login")
2023-05-14 18:25:35 +01:00
def login_openstreetmap() -> flask.Response:
"""Redirect to login."""
return flask.redirect(
flask.url_for("start_oauth", next=flask.request.args.get("next"))
)
2021-06-16 14:42:04 +01:00
@app.route("/logout")
2023-05-14 18:25:35 +01:00
def logout() -> flask.Response:
"""Logout."""
next_url = flask.request.args.get("next") or flask.url_for("map_start_page")
2021-06-16 14:42:04 +01:00
flask_login.logout_user()
2023-05-14 18:25:35 +01:00
flask.flash("you are logged out")
return flask.redirect(next_url)
2021-06-16 14:42:04 +01:00
@app.route("/done/")
2021-06-16 14:42:04 +01:00
def done():
2023-05-14 18:25:35 +01:00
flask.flash("login successful")
return flask.redirect(flask.url_for("map_start_page"))
2021-06-16 14:42:04 +01:00
@app.route("/oauth/start")
2021-06-16 14:42:04 +01:00
def start_oauth():
2023-05-14 18:25:35 +01:00
"""Start OAuth."""
next_page = flask.request.args.get("next")
2021-06-16 14:42:04 +01:00
if next_page:
2023-05-14 18:25:35 +01:00
flask.session["next"] = next_page
2021-06-16 14:42:04 +01:00
client_key = app.config["CLIENT_KEY"]
client_secret = app.config["CLIENT_SECRET"]
2021-06-16 14:42:04 +01:00
request_token_url = "https://www.openstreetmap.org/oauth/request_token"
2021-06-16 14:42:04 +01:00
2023-05-14 18:25:35 +01:00
callback = flask.url_for("oauth_callback", _external=True)
2021-06-16 14:42:04 +01:00
oauth = OAuth1Session(
client_key, client_secret=client_secret, callback_uri=callback
)
2021-06-16 14:42:04 +01:00
fetch_response = oauth.fetch_request_token(request_token_url)
2023-05-14 18:25:35 +01:00
flask.session["owner_key"] = fetch_response.get("oauth_token")
flask.session["owner_secret"] = fetch_response.get("oauth_token_secret")
2021-06-16 14:42:04 +01:00
base_authorization_url = "https://www.openstreetmap.org/oauth/authorize"
authorization_url = oauth.authorization_url(
base_authorization_url, oauth_consumer_key=client_key
)
2023-05-14 18:25:35 +01:00
return flask.redirect(authorization_url)
2021-06-16 14:42:04 +01:00
2021-06-16 14:42:04 +01:00
@login_manager.user_loader
2023-05-14 17:05:19 +01:00
def load_user(user_id) -> model.User:
"""User with the given user_id."""
2021-06-16 14:42:04 +01:00
return model.User.query.get(user_id)
2021-06-16 14:42:04 +01:00
@app.route("/oauth/callback", methods=["GET"])
def oauth_callback():
client_key = app.config["CLIENT_KEY"]
client_secret = app.config["CLIENT_SECRET"]
oauth = OAuth1Session(
client_key,
client_secret=client_secret,
2023-05-14 18:25:35 +01:00
resource_owner_key=flask.session["owner_key"],
resource_owner_secret=flask.session["owner_secret"],
)
2021-06-16 14:42:04 +01:00
2023-05-14 18:25:35 +01:00
oauth_response = oauth.parse_authorization_response(flask.request.url)
verifier = oauth_response.get("oauth_verifier")
access_token_url = "https://www.openstreetmap.org/oauth/access_token"
oauth = OAuth1Session(
client_key,
client_secret=client_secret,
2023-05-14 18:25:35 +01:00
resource_owner_key=flask.session["owner_key"],
resource_owner_secret=flask.session["owner_secret"],
verifier=verifier,
)
2021-06-16 14:42:04 +01:00
oauth_tokens = oauth.fetch_access_token(access_token_url)
2023-05-14 18:25:35 +01:00
flask.session["owner_key"] = oauth_tokens.get("oauth_token")
flask.session["owner_secret"] = oauth_tokens.get("oauth_token_secret")
2021-06-16 14:42:04 +01:00
r = oauth.get(osm_api_base + "/user/details")
2021-06-16 14:42:04 +01:00
info = osm_oauth.parse_userinfo_call(r.content)
user = model.User.query.filter_by(osm_id=info["id"]).one_or_none()
2021-06-16 14:42:04 +01:00
if user:
user.osm_oauth_token = oauth_tokens.get("oauth_token")
user.osm_oauth_token_secret = oauth_tokens.get("oauth_token_secret")
2021-06-16 14:42:04 +01:00
else:
user = model.User(
username=info["username"],
description=info["description"],
img=info["img"],
osm_id=info["id"],
osm_account_created=info["account_created"],
2021-08-05 07:43:15 +01:00
mock_upload=False,
2021-06-16 14:42:04 +01:00
)
database.session.add(user)
database.session.commit()
flask_login.login_user(user)
2023-05-14 18:25:35 +01:00
next_page = flask.session.get("next") or flask.url_for("map_start_page")
return flask.redirect(next_page)
2021-06-16 14:42:04 +01:00
2021-05-07 16:46:47 +01:00
def validate_edit_list(edits):
for e in edits:
assert model.Item.get_by_qid(e["qid"])
assert e["op"] in {"add", "remove", "change"}
osm_type, _, osm_id = e["osm"].partition("/")
osm_id = int(osm_id)
if osm_type == "node":
2021-07-16 11:00:20 +01:00
assert model.Point.query.get(osm_id)
else:
src_id = osm_id if osm_type == "way" else -osm_id
assert model.Line.query.get(src_id) or model.Polygon.query.get(src_id)
@app.route("/api/1/edit", methods=["POST"])
def api_new_edit_session():
user = flask_login.current_user
2023-05-14 18:25:35 +01:00
incoming = flask.request.json
validate_edit_list(incoming["edit_list"])
es = model.EditSession(
user=user, edit_list=incoming["edit_list"], comment=incoming["comment"]
)
database.session.add(es)
database.session.commit()
session_id = es.id
return cors_jsonify(success=True, session_id=session_id)
@app.route("/api/1/edit/<int:session_id>", methods=["POST"])
def api_edit_session(session_id):
es = model.EditSession.query.get(session_id)
assert flask_login.current_user.id == es.user_id
2023-05-14 18:25:35 +01:00
incoming = flask.request.json
for f in "edit_list", "comment":
if f not in incoming:
continue
setattr(es, f, incoming[f])
database.session.commit()
return cors_jsonify(success=True, session_id=session_id)
2021-07-16 11:00:20 +01:00
class VersionMismatch(Exception):
2023-05-14 17:05:19 +01:00
"""Version doesn't match."""
2021-07-16 11:00:20 +01:00
2023-05-14 17:05:19 +01:00
def osm_object(
osm_type: str, osm_id: int
) -> model.Point | model.Line | model.Polygon | None:
"""Get an OSM object from the database."""
2021-07-16 11:00:20 +01:00
if osm_type == "node":
return model.Point.query.get(osm_id)
src_id = int(osm_id) * {"way": 1, "relation": -1}[osm_type]
2021-07-16 11:00:20 +01:00
for cls in model.Line, model.Polygon:
obj = cls.query.get(src_id)
if obj:
return obj
2021-07-19 21:06:36 +01:00
def process_edit(changeset_id, e):
osm_type, _, osm_id = e["osm"].partition("/")
2021-07-16 11:00:20 +01:00
qid = e["qid"]
item_id = qid[1:]
osm = osm_object(osm_type, osm_id)
assert osm
r = edit.get_existing(osm_type, osm_id)
if r.status_code == 410 or r.content == b"":
return "deleted"
2023-05-14 17:04:53 +01:00
root = lxml.etree.fromstring(r.content)
2021-07-16 11:00:20 +01:00
existing = root.find('.//tag[@k="wikidata"]')
if e["op"] == "add" and existing is not None:
return "already_added"
if e["op"] == "remove":
if existing is None:
return "already_removed"
if existing.get("v") != qid:
return "different_qid"
2021-07-16 11:00:20 +01:00
root[0].set("changeset", str(changeset_id))
if e["op"] == "add":
2023-05-14 17:04:53 +01:00
tag = lxml.etree.Element("tag", k="wikidata", v=qid)
2021-07-16 11:00:20 +01:00
root[0].append(tag)
if e["op"] == "remove":
root[0].remove(existing)
if e["op"] == "change":
existing.set("v", qid)
2021-07-16 11:00:20 +01:00
2023-05-14 17:04:53 +01:00
element_data = lxml.etree.tostring(root)
2021-07-16 11:00:20 +01:00
try:
success = edit.save_element(osm_type, osm_id, element_data)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 409 and "Version mismatch" in r.text:
raise VersionMismatch
mail.error_mail(
"error saving element", element_data.decode("utf-8"), e.response
)
database.session.commit()
return "element-error"
if not success:
return "element-error"
new_tags = dict(osm.tags)
if e["op"] in ("add", "change"):
new_tags["wikidata"] = qid
if e["op"] == "remove":
del new_tags["wikidata"]
cls = type(osm)
database.session.execute(
update(cls).where(cls.src_id == osm.src_id).values(tags=new_tags)
)
2021-07-16 11:00:20 +01:00
db_edit = model.ChangesetEdit(
changeset_id=changeset_id,
item_id=item_id,
osm_id=osm_id,
osm_type=osm_type,
)
database.session.add(db_edit)
database.session.commit()
return "saved"
2021-07-16 11:00:20 +01:00
@app.route("/api/1/save/<int:session_id>")
2023-05-14 17:05:19 +01:00
def api_save_changeset(session_id: int):
2023-05-14 18:25:35 +01:00
assert flask.g.user.is_authenticated
2021-07-16 11:00:20 +01:00
2023-05-14 18:25:35 +01:00
mock = flask.g.user.mock_upload
2021-07-16 11:00:20 +01:00
api_call = api_mock_save_changeset if mock else api_real_save_changeset
return api_call(session_id)
2023-05-13 14:01:28 +01:00
@app.route("/sql", methods=["GET", "POST"])
2023-05-14 16:46:38 +01:00
def run_sql() -> str:
"""Web form where the user can run an SQL query."""
2023-05-14 18:25:35 +01:00
if flask.request.method != "POST":
return flask.render_template("run_sql.html")
2023-05-13 14:01:28 +01:00
2023-05-14 18:25:35 +01:00
sql = flask.request.form["sql"]
2023-05-13 14:01:28 +01:00
conn = database.session.connection()
result = conn.execute(sqlalchemy.text(sql))
2023-05-14 18:25:35 +01:00
return flask.render_template("run_sql.html", result=result)
2023-05-13 14:01:28 +01:00
2021-07-16 11:00:20 +01:00
def api_real_save_changeset(session_id):
es = model.EditSession.query.get(session_id)
2021-07-16 11:00:20 +01:00
def send(event, **data):
data["type"] = event
return f"data: {json.dumps(data)}\n\n"
2021-07-16 11:00:20 +01:00
def stream(user):
changeset = edit.new_changeset(es.comment)
r = edit.create_changeset(changeset)
reply = r.text.strip()
if reply == "Couldn't authenticate you":
mail.open_changeset_error(session_id, changeset, r)
2021-07-16 11:00:20 +01:00
yield send("auth-fail", error=reply)
return
if not reply.isdigit():
mail.open_changeset_error(session_id, changeset, r)
2021-07-16 11:00:20 +01:00
yield send("changeset-error", error=reply)
return
changeset_id = int(reply)
2021-07-16 11:00:20 +01:00
yield send("open", id=changeset_id)
update_count = 0
2021-07-16 11:00:20 +01:00
change = edit.record_changeset(
id=changeset_id, user=user, comment=es.comment, update_count=update_count
)
2021-07-16 11:00:20 +01:00
# each edit contains these keys:
# qid: Wikidata item QID
# osm: OpenStreetMap identifier
# op: either 'add' or 'remove'
2021-07-16 11:00:20 +01:00
for num, e in enumerate(es.edit_list):
print(num, e)
yield send("progress", edit=e, num=num)
2021-07-19 21:06:36 +01:00
result = process_edit(changeset_id, e)
2021-07-16 11:00:20 +01:00
yield send(result, edit=e, num=num)
if result == "saved":
update_count += 1
change.update_count = update_count
database.session.commit()
2021-07-16 11:00:20 +01:00
yield send("closing")
edit.close_changeset(changeset_id)
yield send("done")
2023-05-14 18:25:35 +01:00
return flask.Response(
flask.stream_with_context(stream(flask.g.user)), mimetype="text/event-stream"
)
2021-07-16 11:00:20 +01:00
def api_mock_save_changeset(session_id):
es = model.EditSession.query.get(session_id)
def send(event, **data):
data["type"] = event
return f"data: {json.dumps(data)}\n\n"
def stream(user):
print("stream")
changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar()
sleep(1)
yield send("open", id=changeset_id)
sleep(1)
update_count = 0
print("record_changeset", changeset_id)
edit.record_changeset(
id=changeset_id, user=user, comment=es.comment, update_count=update_count
)
print("edits")
for num, e in enumerate(es.edit_list):
print(num, e)
yield send("progress", edit=e, num=num)
sleep(1)
yield send("saved", edit=e, num=num)
sleep(1)
print("closing")
yield send("closing")
sleep(1)
yield send("done")
2023-05-14 18:25:35 +01:00
return flask.Response(stream(flask.g.user), mimetype="text/event-stream")
2021-05-07 16:46:47 +01:00
if __name__ == "__main__":
app.run(host="0.0.0.0")