owl-map/web_view.py

593 lines
17 KiB
Python
Raw Normal View History

2021-05-07 16:46:47 +01:00
#!/usr/bin/python3
2021-06-16 14:42:04 +01:00
from flask import (Flask, render_template, request, jsonify, redirect, url_for, g,
flash, session, Response)
2021-06-25 13:52:42 +01:00
from sqlalchemy import func
from matcher import (nominatim, model, database, commons, wikidata, wikidata_api,
2021-06-25 13:52:42 +01:00
osm_oauth, edit, mail, api)
from matcher.data import property_map
from time import time, sleep
2021-06-16 14:42:04 +01:00
from requests_oauthlib import OAuth1Session
import flask_login
import json
2021-05-07 16:46:47 +01:00
import GeoIP
2021-05-15 13:05:44 +01:00
import re
2021-07-06 16:24:14 +01:00
import maxminddb
2021-05-07 16:46:47 +01:00
srid = 4326
2021-05-15 13:05:44 +01:00
re_point = re.compile(r'^POINT\((.+) (.+)\)$')
2021-05-07 16:46:47 +01:00
app = Flask(__name__)
app.debug = True
2021-05-14 10:05:58 +01:00
app.config.from_object('config.default')
2021-05-07 16:46:47 +01:00
2021-06-16 14:42:04 +01:00
login_manager = flask_login.LoginManager(app)
login_manager.login_view = 'login_route'
osm_api_base = 'https://api.openstreetmap.org/api/0.6'
2021-07-06 16:24:14 +01:00
maxminddb_reader = maxminddb.open_database(app.config["GEOLITE2"])
2021-06-16 14:42:04 +01:00
2021-05-07 16:46:47 +01:00
DB_URL = "postgresql:///matcher"
database.init_db(DB_URL)
entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"}
2021-05-07 16:46:47 +01:00
2021-06-16 13:31:58 +01:00
re_qid = re.compile(r'^Q\d+$')
2021-05-07 16:46:47 +01:00
@app.teardown_appcontext
def shutdown_session(exception=None):
database.session.remove()
2021-06-17 18:17:28 +01:00
@app.before_request
def global_user():
g.user = flask_login.current_user._get_current_object()
def cors_jsonify(*args, **kwargs):
response = jsonify(*args, **kwargs)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
2021-05-07 16:46:47 +01:00
def check_for_tagged_qids(qids):
tagged = set()
for qid in qids:
for cls in model.Point, model.Polygon, model.Line:
q = cls.query.filter(cls.tags["wikidata"] == qid)
if q.count():
tagged.add(qid)
break
return tagged
def check_for_tagged_qid(qid):
return any(
database.session.query(
cls.query.filter(
cls.tags.has_key("wikidata"), cls.tags["wikidata"] == qid
).exists()
).scalar()
for cls in (model.Point, model.Polygon, model.Line)
)
2021-06-25 10:12:50 +01:00
def geoip_user_record():
gi = GeoIP.open(app.config["GEOIP_DATA"], GeoIP.GEOIP_STANDARD)
2021-05-07 16:46:47 +01:00
2021-07-06 16:24:14 +01:00
remote_ip = request.get('ip', request.remote_addr)
2021-06-25 10:12:50 +01:00
return gi.record_by_addr(remote_ip)
def get_user_location():
2021-07-06 16:24:14 +01:00
remote_ip = request.args.get('ip', request.remote_addr)
return maxminddb_reader.get(remote_ip)["location"]
2021-06-25 10:12:50 +01:00
gir = geoip_user_record()
return (gir["latitude"], gir["longitude"]) if gir else None
2021-05-07 16:46:47 +01:00
@app.route("/")
def redirect_from_root():
return redirect(url_for("map_start_page"))
@app.route("/index")
def index_page():
return render_template("index.html")
2021-06-25 13:52:06 +01:00
@app.route("/admin/skip_isa")
def admin_skip_isa_list():
q = model.Item.query.join(model.SkipIsA).order_by(model.Item.item_id)
return render_template("admin/skip_isa.html", q=q)
2021-05-07 16:46:47 +01:00
@app.route("/identifier")
def identifier_index():
return render_template("identifier_index.html", property_map=property_map)
2021-05-08 09:39:06 +01:00
@app.route("/commons/<filename>")
def get_commons_image(filename):
detail = commons.image_detail([filename], thumbheight=1200, thumbwidth=1200)
2021-05-08 09:39:06 +01:00
image = detail[filename]
return redirect(image["thumburl"])
2021-05-07 16:46:47 +01:00
@app.route("/identifier/<pid>")
def identifier_page(pid):
per_page = 10
page = int(request.args.get("page", 1))
property_dict = {pid: (osm_keys, label) for pid, osm_keys, label in property_map}
osm_keys, label = property_dict[pid]
wd = model.Item.query.filter(model.Item.claims.has_key(pid))
total = wd.count()
start = per_page * (page - 1)
items = wd.all()[start : per_page * page]
qids = [item.qid for item in items]
print(qids)
# pred = None
# values = set()
# for item in items:
# values |= set(item.get_claim(pid))
#
# for key in osm_keys:
# if key == 'ref':
# continue
# if pred is None:
# pred = model.Point.tags[key].in_(values)
# else:
# pred |= model.Point.tags[key].in_(values)
#
osm_points = {}
for qid in qids:
osm_points[qid] = model.Point.query.filter(
model.Point.tags["wikidata"] == qid
).all()
osm_total = len(osm_points)
return render_template(
"identifier_page.html",
pid=pid,
osm_keys=osm_keys,
label=label,
items=items,
total=total,
osm_total=osm_total,
osm_points=osm_points,
)
@app.route("/map")
def map_start_page():
2021-07-06 16:24:14 +01:00
loc = get_user_location()
2021-06-17 18:12:07 +01:00
2021-06-14 23:08:07 +01:00
return redirect(url_for(
'map_location',
2021-07-06 16:24:14 +01:00
lat=f'{loc["latitude"]:.5f}',
lon=f'{loc["longitude"]:.5f}',
2021-06-14 23:08:07 +01:00
zoom=16,
2021-07-06 16:24:14 +01:00
radius=loc["accuracy_radius"],
ip=request.args.get('ip'),
2021-06-14 23:08:07 +01:00
))
@app.route("/map/<int:zoom>/<float(signed=True):lat>/<float(signed=True):lon>")
def map_location(zoom, lat, lon):
2021-06-17 18:12:07 +01:00
user = flask_login.current_user
username = user.username if user.is_authenticated else None
2021-07-06 16:24:14 +01:00
return render_template(
"map.html",
zoom=zoom,
lat=lat,
lon=lon,
radius=request.args.get('radius'),
username=username
)
2021-06-14 23:08:07 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/search/map")
def search_map_page():
user_lat, user_lon = get_user_location() or (None, None)
q = request.args.get("q")
if not q:
return render_template("map.html", user_lat=user_lat, user_lon=user_lon)
hits = nominatim.lookup(q)
for hit in hits:
if "geotext" in hit:
del hit["geotext"]
bbox = [hit["boundingbox"] for hit in hits]
return render_template(
"search_map.html",
hits=hits,
bbox_list=bbox,
user_lat=user_lat,
user_lon=user_lon,
)
@app.route("/search")
def search_page():
q = request.args.get("q")
if not q:
return render_template("search.html", hits=None, bbox_list=None)
hits = nominatim.lookup(q)
for hit in hits:
if "geotext" in hit:
del hit["geotext"]
bbox = [hit["boundingbox"] for hit in hits]
return render_template("search.html", hits=hits, bbox_list=bbox)
2021-06-25 13:52:42 +01:00
def read_bounds_param():
return [float(i) for i in request.args["bounds"].split(",")]
2021-05-07 16:46:47 +01:00
2021-06-25 10:12:50 +01:00
@app.route("/api/1/location")
def show_user_location():
return cors_jsonify(get_user_location())
2021-06-17 18:14:42 +01:00
@app.route("/api/1/count")
def api_wikidata_items_count():
t0 = time()
2021-06-25 13:52:42 +01:00
count = api.wikidata_items_count(read_bounds_param())
2021-06-17 18:14:42 +01:00
t1 = time() - t0
2021-06-25 13:52:42 +01:00
return cors_jsonify(success=True, count=count, duration=t1)
2021-06-17 18:14:42 +01:00
2021-06-24 17:42:33 +01:00
@app.route("/api/1/isa")
def api_wikidata_isa_counts():
t0 = time()
2021-06-25 13:52:42 +01:00
bounds = read_bounds_param()
isa_count = api.wikidata_isa_counts(bounds)
2021-06-24 17:42:33 +01:00
t1 = time() - t0
return cors_jsonify(success=True, isa_count=isa_count, bounds=bounds, duration=t1)
2021-06-24 17:42:33 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/api/1/items")
def api_wikidata_items():
t0 = time()
2021-06-25 13:52:42 +01:00
bounds = read_bounds_param()
ret = api.wikidata_items(bounds)
2021-05-07 16:46:47 +01:00
2021-06-25 13:52:42 +01:00
t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **ret)
2021-05-07 16:46:47 +01:00
@app.route("/api/1/osm")
def api_osm_objects():
t0 = time()
2021-06-25 13:52:42 +01:00
objects = api.get_osm_with_wikidata_tag(read_bounds_param())
2021-05-07 16:46:47 +01:00
t1 = time() - t0
return cors_jsonify(success=True, objects=objects, duration=t1)
2021-05-07 16:46:47 +01:00
@app.route("/api/1/item/Q<int:item_id>")
def api_get_item(item_id):
t0 = time()
item = model.Item.query.get(item_id)
detail = api.item_detail(item)
t1 = time() - t0
return cors_jsonify(success=True,
duration=t1,
**detail)
2021-05-12 08:27:34 +01:00
@app.route("/api/1/item/Q<int:item_id>/tags")
def api_get_item_tags(item_id):
t0 = time()
item = model.Item.query.get(item_id)
tags = api.get_item_tags(item)
osm_list = sorted(tags.keys())
2021-05-12 08:27:34 +01:00
t1 = time() - t0
return cors_jsonify(success=True,
qid=item.qid,
tag_or_key_list=osm_list,
tag_src=tags,
duration=t1)
2021-05-12 08:27:34 +01:00
2021-05-14 16:09:56 +01:00
2021-05-12 15:55:40 +01:00
@app.route("/api/1/item/Q<int:item_id>/candidates")
def api_find_osm_candidates(item_id):
t0 = time()
2021-06-25 13:52:42 +01:00
bounds = read_bounds_param()
2021-05-12 15:55:40 +01:00
item = model.Item.query.get(item_id)
2021-06-25 13:52:42 +01:00
nearby = api.find_osm_candidates(item, bounds)
2021-05-12 15:55:40 +01:00
t1 = time() - t0
return cors_jsonify(success=True, qid=item.qid, nearby=nearby, duration=t1)
2021-05-12 15:55:40 +01:00
2021-05-10 13:59:08 +01:00
@app.route("/api/1/missing")
def api_missing_wikidata_items():
qids_arg = request.args.get("qids")
2021-06-16 13:31:58 +01:00
if not qids_arg:
return cors_jsonify(success=False,
error="required parameter 'qids' is missing",
items=[],
isa_count=[])
2021-06-16 13:31:58 +01:00
qids = []
for qid in qids_arg.upper().split(","):
qid = qid.strip()
m = re_qid.match(qid)
if not m:
continue
qids.append(qid)
if not qids:
2021-05-10 13:59:08 +01:00
return jsonify(success=True, items=[], isa_count=[])
2021-05-15 13:05:44 +01:00
lat, lon = request.args.get("lat"), request.args.get("lon")
2021-06-25 13:52:42 +01:00
ret = api.missing_wikidata_items(qids, lat, lon)
return cors_jsonify(success=True, **ret)
2021-05-10 13:59:08 +01:00
2021-05-07 16:46:47 +01:00
@app.route("/api/1/search")
def api_search():
q = request.args["q"]
hits = nominatim.lookup(q)
for hit in hits:
if "geotext" in hit:
del hit["geotext"]
hit["name"] = nominatim.get_hit_name(hit)
2021-07-08 13:35:32 +01:00
hit["label"] = nominatim.get_hit_label(hit)
hit["address"] = list(hit["address"].items())
2021-06-14 23:09:05 +01:00
hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}"
return cors_jsonify(success=True, hits=hits)
2021-06-14 23:09:05 +01:00
@app.route("/refresh/Q<int:item_id>")
def refresh_item(item_id):
assert not model.Item.query.get(item_id)
qid = f'Q{item_id}'
entity = wikidata_api.get_entity(qid)
entity_qid = entity.pop("id")
assert qid == entity_qid
coords = wikidata.get_entity_coords(entity["claims"])
assert coords
obj = {k: v for k, v in entity.items() if k in entity_keys}
item = model.Item(item_id=item_id, **obj)
print(item)
item.locations = model.location_objects(coords)
database.session.add(item)
database.session.commit()
2021-05-07 16:46:47 +01:00
2021-06-14 23:09:05 +01:00
return 'done'
2021-05-07 16:46:47 +01:00
2021-06-16 14:42:04 +01:00
@app.route('/login')
def login_openstreetmap():
return redirect(url_for('start_oauth',
next=request.args.get('next')))
@app.route('/logout')
def logout():
2021-07-08 13:34:55 +01:00
next_url = request.args.get('next') or url_for('map_start_page')
2021-06-16 14:42:04 +01:00
flask_login.logout_user()
flash('you are logged out')
return redirect(next_url)
@app.route('/done/')
def done():
flash('login successful')
return redirect(url_for('index'))
@app.route('/oauth/start')
def start_oauth():
next_page = request.args.get('next')
if next_page:
session['next'] = next_page
client_key = app.config['CLIENT_KEY']
client_secret = app.config['CLIENT_SECRET']
request_token_url = 'https://www.openstreetmap.org/oauth/request_token'
callback = url_for('oauth_callback', _external=True)
oauth = OAuth1Session(client_key,
client_secret=client_secret,
callback_uri=callback)
fetch_response = oauth.fetch_request_token(request_token_url)
session['owner_key'] = fetch_response.get('oauth_token')
session['owner_secret'] = fetch_response.get('oauth_token_secret')
base_authorization_url = 'https://www.openstreetmap.org/oauth/authorize'
authorization_url = oauth.authorization_url(base_authorization_url,
oauth_consumer_key=client_key)
return redirect(authorization_url)
@login_manager.user_loader
def load_user(user_id):
return model.User.query.get(user_id)
@app.route("/oauth/callback", methods=["GET"])
def oauth_callback():
client_key = app.config['CLIENT_KEY']
client_secret = app.config['CLIENT_SECRET']
oauth = OAuth1Session(client_key,
client_secret=client_secret,
resource_owner_key=session['owner_key'],
resource_owner_secret=session['owner_secret'])
oauth_response = oauth.parse_authorization_response(request.url)
verifier = oauth_response.get('oauth_verifier')
access_token_url = 'https://www.openstreetmap.org/oauth/access_token'
oauth = OAuth1Session(client_key,
client_secret=client_secret,
resource_owner_key=session['owner_key'],
resource_owner_secret=session['owner_secret'],
verifier=verifier)
oauth_tokens = oauth.fetch_access_token(access_token_url)
session['owner_key'] = oauth_tokens.get('oauth_token')
session['owner_secret'] = oauth_tokens.get('oauth_token_secret')
r = oauth.get(osm_api_base + '/user/details')
info = osm_oauth.parse_userinfo_call(r.content)
user = model.User.query.filter_by(osm_id=info['id']).one_or_none()
if user:
user.osm_oauth_token = oauth_tokens.get('oauth_token')
user.osm_oauth_token_secret = oauth_tokens.get('oauth_token_secret')
else:
user = model.User(
username=info['username'],
description=info['description'],
img=info['img'],
osm_id=info['id'],
osm_account_created=info['account_created'],
)
database.session.add(user)
database.session.commit()
flask_login.login_user(user)
next_page = session.get('next') or url_for('index_page')
return redirect(next_page)
2021-05-07 16:46:47 +01:00
def validate_edit_list(edits):
for e in edits:
assert model.Item.get_by_qid(e["qid"])
assert e["op"] in {"add", "remove"}
osm_type, _, osm_id = e['osm'].partition('/')
osm_id = int(osm_id)
if osm_type == 'node':
assert model.Point.get(osm_id)
else:
src_id = osm_id if osm_type == "way" else -osm_id
assert (model.Line.query.get(src_id)
or model.Polygon.query.get(src_id))
@app.route("/api/1/edit", methods=["POST"])
def api_new_edit_session():
user = flask_login.current_user
incoming = request.json
validate_edit_list(incoming["edit_list"])
es = model.EditSession(user=user,
edit_list=incoming['edit_list'],
comment=incoming['comment'])
database.session.add(es)
database.session.commit()
session_id = es.id
return cors_jsonify(success=True, session_id=session_id)
@app.route("/api/1/edit/<int:session_id>", methods=["POST"])
def api_edit_session(session_id):
es = model.EditSession.query.get(session_id)
assert flask_login.current_user.id == es.user_id
incoming = request.json
for f in 'edit_list', 'comment':
if f not in incoming:
continue
setattr(es, f, incoming[f])
database.session.commit()
return cors_jsonify(success=True, session_id=session_id)
@app.route("/api/1/real_save/<int:session_id>")
def api_save_changeset(session_id):
es = model.EditSession.query.get(session_id)
def send_message(event, **data):
data["type"] = event
return f"data: {json.dumps(data)}\n\n"
def stream():
changeset = edit.new_changeset(es.comment)
r = edit.create_changeset(changeset)
reply = r.text.strip()
if reply == "Couldn't authenticate you":
mail.open_changeset_error(session_id, changeset, r)
yield send_message("auth-fail", error=reply)
return
if not reply.isdigit():
mail.open_changeset_error(session_id, changeset, r)
yield send_message("changeset-error", error=reply)
return
changeset_id = int(reply)
yield send_message("open", id=changeset_id)
update_count = 0
edit.record_changeset(
id=changeset_id, comment=es.comment, update_count=update_count
)
for e in es.edit_list:
pass
return Response(stream(), mimetype='text/event-stream')
@app.route("/api/1/save/<int:session_id>")
def mock_api_save_changeset(session_id):
es = model.EditSession.query.get(session_id)
def send(event, **data):
data["type"] = event
return f"data: {json.dumps(data)}\n\n"
def stream(user):
print('stream')
changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar()
sleep(1)
yield send("open", id=changeset_id)
sleep(1)
update_count = 0
print('record_changeset', changeset_id)
edit.record_changeset(
id=changeset_id, user=user, comment=es.comment, update_count=update_count
)
print('edits')
for num, e in enumerate(es.edit_list):
print(num, e)
yield send("progress", edit=e, num=num)
sleep(1)
yield send("saved", edit=e, num=num)
sleep(1)
print('closing')
yield send("closing")
sleep(1)
yield send("done")
return Response(stream(g.user), mimetype='text/event-stream')
2021-05-07 16:46:47 +01:00
if __name__ == "__main__":
app.run(host="0.0.0.0")