This commit is contained in:
Edward Betts 2023-05-13 15:01:28 +02:00
parent 47e1280269
commit c607351699
7 changed files with 274 additions and 69 deletions

View File

@ -303,7 +303,7 @@
v-bind:key="isa.qid"
v-for="isa in item_type_hits"
href="#"
@click.prevent="item_type_filters.includes(isa) || item_type_filters.push(isa)"
@click.prevent="add_item_type_filter(isa)"
>
{{ isa.label }} ({{ isa.qid }})
</a>
@ -458,12 +458,33 @@
<br>{{ wd_item.aliases.join("; ") }}
</span>
<br><strong>item coordinates</strong>
<span v-for="marker in wd_item.markers">
<br><a target="_blank" :href="marker_osm_url(marker)" @click.stop>
{{ marker[0].toFixed(5) }},
{{ marker[1].toFixed(5) }}
<i class="fa fa-map-o"></i></a>
</span>
<br><strong>item type</strong>
<span v-bind:key="`isa-${wd_item.qid}-${isa.qid}`" v-for="isa in wd_item.isa_list">
<br><a :href="qid_url(isa.qid)" target="_blank">{{isa.label}}</a> ({{isa.qid}})
<a :href="'/isa/' + isa.qid" target="_blank"><i class="fa fa-pencil-square-o"></i></a>
</span>
<span v-if="wd_item.wikipedia.length > 0">
<br>
<strong>
Wikipedia
<i class="fa fa-external-link"></i>
</strong>
<br>
<span v-for="wp in wd_item.wikipedia">
<a :href="wikipedia_link(wp.lang, wp.title)" target="_blank">{{wp.lang}}</a>
&nbsp;
</span>
</span>
<span v-if="wd_item.street_address.length">
<br><strong>street address</strong>
<br>{{wd_item.street_address[0]}}
@ -524,6 +545,15 @@
</a>
</span>
<span v-if="wd_item.commons !== undefined">
<br><strong>Images on Commons</strong>
<br><a
:href="`https://commons.wikimedia.org/wiki/${wd_item.commons.replaceAll(' ', '_')}`"
target="_blank">
{{wd_item.commons}} <i class="fa fa-external-link"></i>
</a>
</span>
</div></div>
</div>
@ -548,7 +578,7 @@
<div v-if="current_item.nearby && !current_item.nearby.length">
<strong>No OSM matches found nearby</strong>
<div class="mt-2" v-if="current_item.tag_or_key_list.length">
<div class="mt-2" v-if="current_item.tag_or_key_list && current_item.tag_or_key_list.length">
<p>The OSM tags/keys used as the search criteria to find matching
OSM objects are listed below, along with the Wikidata item that was
the source.</p>
@ -766,6 +796,8 @@ export default {
startLon: Number,
startZoom: Number,
startRadius: Number,
startItem: String,
startItemTypeFilter: Array,
username: String,
startMode: String,
q: String,
@ -994,10 +1026,26 @@ export default {
}
},
methods: {
api_call(endpoint, options) {
wikipedia_link(lang, title) {
var norm_title = title.replaceAll(" ", "_");
return `https://${lang}.wikipedia.org/wiki/${norm_title}`;
},
marker_osm_url(marker) {
var lat = marker[0].toFixed(5);
var lon = marker[1].toFixed(5);
return `https://www.openstreetmap.org/?mlat=${lat}&mlon=${lon}#map=18/${lat}/${lon}`
},
add_item_type_filter(isa) {
if (this.item_type_filters.includes(isa)) {
return;
}
this.item_type_filters.push(isa);
this.update_map_path();
},
api_call(endpoint, options) {
var url = `${this.api_base_url}/api/1/${endpoint}`;
return axios.get(url, options).catch(this.show_api_error_modal);
},
return axios.get(url, options).catch(this.show_api_error_modal);
},
update_unload_warning(edit_list) {
if (edit_list.length) {
addEventListener("beforeunload", beforeUnloadListener, {capture: true});
@ -1216,14 +1264,19 @@ export default {
this.isa_ticked = Object.keys(this.isa_labels);
},
build_map_path() {
if (this.current_item) {
return `/item/${this.current_qid}`;
}
var zoom = this.map.getZoom();
var c = this.map.getCenter();
var lat = c.lat.toFixed(5);
var lng = c.lng.toFixed(5);
var path = `/map/${zoom}/${lat}/${lng}`;
if (this.current_item) {
path += `?item=${this.current_qid}`;
if (this.item_type_filters.length) {
path += "?isa=" + this.item_type_filters.map((t) => t.qid).join(";");
}
return path;
},
@ -1755,6 +1808,11 @@ export default {
this.zoom = this.startZoom;
this.mode = this.startMode;
this.changeset_comment = this.defaultComment || '+wikidata';
console.log(this.startItemTypeFilter);
if (this.startItemTypeFilter.length) {
this.show_item_type_filter = true;
}
this.item_type_filters = this.startItemTypeFilter;
},
mounted() {
@ -1764,7 +1822,6 @@ export default {
zoom: this.zoom || 16,
};
var map = L.map("map", options);
var osm_url = "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png";
var tile_url = "https://tile-c.openstreetmap.fr/hot/{z}/{x}/{y}.png";
@ -1788,13 +1845,13 @@ export default {
this.search_text = this.q.trim();
this.run_search();
} else {
this.detail_qid = this.qid_from_url();
this.detail_qid = this.startItem;
if (this.detail_qid) {
this.load_wikidata_items(bounds);
} else {
this.auto_load(bounds);
this.update_map_path();
}
this.update_map_path();
}
window.onpopstate = this.onpopstate;

View File

@ -38,6 +38,14 @@ skip_tags = {
}
def get_country_iso3166_1(lat, lon):
"""
For a given lat/lon return a set of ISO country codes.
Also cache the country code in the global object.
Normally there should be only one country.
"""
point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), srid)
alpha2_codes = set()
q = model.Polygon.query.filter(func.ST_Covers(model.Polygon.way, point),
@ -57,7 +65,18 @@ def is_street_number_first(lat, lon):
return True
alpha2 = get_country_iso3166_1(lat, lon)
alpha2_number_first = {'GB', 'IE', 'US', 'MX', 'CA', 'FR', 'AU', 'NZ', 'ZA'}
# Incomplete list of countries that put street number first.
alpha2_number_first = {
'GB', # United Kingdom
'IE', # Ireland
'US', # United States
'MX', # Mexico
'CA', # Canada
'FR', # France
'AU', # Australia
'NZ', # New Zealand
'ZA', # South Africa
}
return bool(alpha2_number_first & alpha2)
@ -92,6 +111,7 @@ def make_envelope_around_point(lat, lon, distance):
return func.ST_MakeEnvelope(west, south, east, north, srid)
def drop_way_area(tags):
""" Remove the way_area field from a tags dict. """
if "way_area" in tags:
del tags["way_area"]
return tags
@ -122,6 +142,8 @@ def get_part_of(table_name, src_id, bbox):
} for osm_id, tags, area in conn.execute(s)]
def get_and_save_item(qid):
""" Download an item from Wikidata and cache it in the database. """
entity = wikidata_api.get_entity(qid)
entity_qid = entity["id"]
if entity_qid != qid:
@ -396,7 +418,6 @@ def add_isa_filter(q, isa_qids):
)
subclass_qid = {qid for qid, in q_subclass.all()}
# print(subclass_qid)
isa = func.jsonb_path_query_array(
model.Item.claims,
@ -419,7 +440,7 @@ def wikidata_items_count(bounds, isa_filter=None):
return q.count()
def wikidata_isa_counts(bounds):
def wikidata_isa_counts(bounds, isa_filter=None):
db_bbox = make_envelope(bounds)
q = (
@ -427,6 +448,9 @@ def wikidata_isa_counts(bounds):
.filter(func.ST_Covers(db_bbox, model.ItemLocation.location))
)
if isa_filter:
q = add_isa_filter(q, isa_filter)
db_items = q.all()
counts = get_isa_count(db_items)
@ -605,7 +629,11 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
item_id = item.item_id
item_is_linear_feature = item.is_linear_feature()
item_is_street = item.is_street()
item_names = {n.lower() for n in item.names().keys()}
item_names_dict = item.names()
if item_names_dict:
item_names = {n.lower() for n in item_names_dict.keys()}
else:
item_names = set()
check_is_street_number_first(item.locations[0].get_lat_lon())
@ -702,6 +730,8 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
shape = "area" if table == "polygon" else table
item_identifier_tags = item.get_identifiers_tags()
cur = {
"identifier": f"{osm_type}/{osm_id}",
"type": osm_type,
@ -733,6 +763,8 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
return nearby
def get_item(item_id):
""" Retrieve a Wikidata item, either from the database or from Wikidata. """
item = model.Item.query.get(item_id)
return item or get_and_save_item(f"Q{item_id}")
@ -763,6 +795,11 @@ def check_is_street_number_first(latlng):
g.street_number_first = is_street_number_first(*latlng)
def item_detail(item):
unsupported_relation_types = {
'Q194356', # wind farm
'Q2175765', # tram stop
}
locations = [list(i.get_lat_lon()) for i in item.locations]
if not hasattr(g, 'street_number_first'):
g.street_number_first = is_street_number_first(*locations[0])
@ -783,6 +820,11 @@ def item_detail(item):
})
isa_items = [get_item(isa["numeric-id"]) for isa in item.get_isa()]
isa_lookup = {isa.qid: isa for isa in isa_items}
wikipedia_links = [{"lang": site[:-4], "title": link["title"]}
for site, link in sorted(item.sitelinks.items())
if site.endswith("wiki") and len(site) < 8]
d = {
"qid": item.qid,
@ -797,11 +839,21 @@ def item_detail(item):
"p1619": item.time_claim("P1619"),
"p576": item.time_claim("P576"),
"heritage_designation": heritage_designation,
"wikipedia": wikipedia_links,
"identifiers": item.get_identifiers(),
}
if aliases := item.get_aliases():
d["aliases"] = aliases
if "commonswiki" in item.sitelinks:
d["commons"] = item.sitelinks["commonswiki"]["title"]
unsupported = isa_lookup.keys() & unsupported_relation_types
if unsupported:
d["unsupported_relation_types"] = [isa for isa in d["isa_list"]
if isa["qid"] in isa_lookup]
return d
@ -892,3 +944,21 @@ def isa_incremental_search(search_terms):
}
ret.append(cur)
return ret
def get_place_items(osm_type, osm_id):
src_id = osm_id * {'way': 1, 'relation': -1}[osm_type]
q = (model.Item.query
.join(model.ItemLocation)
.join(model.Polygon, func.ST_Covers(model.Polygon.way, model.ItemLocation.location))
.filter(model.Polygon.src_id == src_id))
# sql = q.statement.compile(compile_kwargs={"literal_binds": True})
item_count = q.count()
items = []
for item in q:
keys = ["item_id", "labels", "descriptions", "aliases", "sitelinks", "claims"]
item_dict = {key: getattr(item, key) for key in keys}
items.append(item_dict)
return {"count": item_count, "items": items}

View File

@ -83,8 +83,8 @@ def get_hit_name(hit):
if len(address) == 1:
return n1
country = address.pop("country")
country_code = address.pop("country_code")
country = address.pop("country", None)
country_code = address.pop("country_code", None)
if country_code:
country_code == country_code.lower()

View File

@ -5,7 +5,6 @@ import json
import math
import user_agents
import re
import pattern.en
from datetime import date
from num2words import num2words
@ -160,18 +159,6 @@ def is_in_range(address_range, address):
return False
def pluralize_label(label):
text = label["value"]
if label["language"] != "en":
return text
# pattern.en.pluralize has the plural of 'mine' as 'ours'
if text == "mine":
return "mines"
return pattern.en.pluralize(text)
def format_wikibase_time(v):
p = v["precision"]
t = v["time"]
@ -180,11 +167,12 @@ def format_wikibase_time(v):
# example: https://www.wikidata.org/wiki/Q108266998
if p == 11:
return date.fromisoformat(t[1:11]).strftime("%d %B %Y")
return date.fromisoformat(t[1:11]).strftime("%-d %B %Y")
if p == 10:
return date.fromisoformat(t[1:8] + "-01").strftime("%B %Y")
if p == 9:
return t[1:5]
if p == 7:
century = ((int(t[:5]) - 1) // 100) + 1
return num2words(century, to="ordinal_num") + " century"
end = " BC" if century < 0 else ""
return num2words(abs(century), to="ordinal_num") + " century" + end

View File

@ -4,6 +4,7 @@
{% block content %}
<div class="container my-2">
{% include "flash_msg.html" %}
<h1>{{ self.title() }}</h1>
@ -13,7 +14,7 @@
</a></div>
<div class="my-2">
<form method="POST">
<form method="GET" action="{{ url_for("refresh_item", item_id=item.item_id) }}">
<input type="hidden" name="action" value="refresh">
<input type="submit" value="refresh item" class="btn btn-sm btn-primary">
</form>

View File

@ -4,26 +4,32 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Wikidata items linked to OSM</title>
<link rel="stylesheet" href="https://unpkg.com/bootstrap@5.0.1/dist/css/bootstrap.min.css">
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css">
<link rel="stylesheet" href="https://unpkg.com/fork-awesome@1.1.7/css/fork-awesome.min.css">
<!--
<link rel="stylesheet" href="https://unpkg.com/bootstrap@5.1.3/dist/css/bootstrap.min.css">
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.8.0/dist/leaflet.css">
<link rel="stylesheet" href="https://unpkg.com/fork-awesome@1.2.0/css/fork-awesome.min.css">
<link rel="stylesheet" href="https://unpkg.com/leaflet-extra-markers@1.2.1/dist/css/leaflet.extra-markers.min.css">
-->
<link rel="stylesheet" href="{{ url_for("static", filename="frontend/style.css") }}">
</head>
{% from "navbar.html" import navbar with context %}
<body>
{% block nav %}{{ navbar() }}{% endblock %}
<div id="app"></div>
<script src="https://unpkg.com/bootstrap@5.0.1/dist/js/bootstrap.bundle.min.js" integrity="sha384-gtEjrD/SeCtmISkJkNUaaKMoLD0//ElJ19smozuHV6z3Iehds+3Ulb9Bn9Plx0x4" crossorigin="anonymous"></script>
<!-- <script src="https://unpkg.com/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js" crossorigin="anonymous"></script> -->
<script type="module">
import { createApp } from "https://cdn.skypack.dev/vue@^3.0.11";
import App from {{ url_for('static', filename='snowpack/App.vue.js') | tojson }};
import main from {{ url_for('static', filename='frontend/owl.es.js') | tojson }};
const props = {
startLat: {{ lat }},
startLon: {{ lon }},
startZoom: {{ zoom }},
startRadius: {{ radius | tojson }},
startRadius: {{ (radius or None) | tojson }},
startItem: {{ (qid or None) | tojson }},
startItemTypeFilter: {{ (item_type_filter or []) | tojson }},
defaultComment: {{ config.DEFAULT_COMMENT | tojson }},
username: {{ username | tojson }},
startMode: {{ mode | tojson }},
@ -32,7 +38,7 @@
mockUpload: false,
};
const app = createApp(App, props).mount('#app');
main(props);
</script>
</body>

View File

@ -1,12 +1,12 @@
#!/usr/bin/python3.9
from flask import (Flask, render_template, request, jsonify, redirect, url_for, g,
flash, session, Response, stream_with_context)
flash, session, Response, stream_with_context, abort, send_file)
from sqlalchemy import func
from sqlalchemy.sql.expression import update
from matcher import (nominatim, model, database, commons, wikidata, wikidata_api,
osm_oauth, edit, mail, api, error_mail)
from werkzeug.debug.tbtools import get_current_traceback
# from werkzeug.debug.tbtools import get_current_traceback
from matcher.data import property_map
from time import time, sleep
from requests_oauthlib import OAuth1Session
@ -19,6 +19,7 @@ import json
import GeoIP
import re
import maxminddb
import sqlalchemy
srid = 4326
re_point = re.compile(r'^POINT\((.+) (.+)\)$')
@ -54,27 +55,27 @@ def dict_repr_values(d):
return {key: repr(value) for key, value in d.items()}
@app.errorhandler(werkzeug.exceptions.InternalServerError)
def exception_handler(e):
tb = get_current_traceback()
last_frame = next(frame for frame in reversed(tb.frames) if not frame.is_library)
last_frame_args = inspect.getargs(last_frame.code)
if request.path.startswith("/api/"):
return cors_jsonify({
"success": False,
"error": tb.exception,
"traceback": tb.plaintext,
"locals": dict_repr_values(last_frame.locals),
"last_function": {
"name": tb.frames[-1].function_name,
"args": repr(last_frame_args),
},
}), 500
return render_template('show_error.html',
tb=tb,
last_frame=last_frame,
last_frame_args=last_frame_args), 500
# @app.errorhandler(werkzeug.exceptions.InternalServerError)
# def exception_handler(e):
# tb = get_current_traceback()
# last_frame = next(frame for frame in reversed(tb.frames) if not frame.is_library)
# last_frame_args = inspect.getargs(last_frame.code)
# if request.path.startswith("/api/"):
# return cors_jsonify({
# "success": False,
# "error": tb.exception,
# "traceback": tb.plaintext,
# "locals": dict_repr_values(last_frame.locals),
# "last_function": {
# "name": tb.frames[-1].function_name,
# "args": repr(last_frame_args),
# },
# }), 500
#
# return render_template('show_error.html',
# tb=tb,
# last_frame=last_frame,
# last_frame_args=last_frame_args), 500
def cors_jsonify(*args, **kwargs):
response = jsonify(*args, **kwargs)
@ -113,8 +114,8 @@ def geoip_user_record():
def get_user_location():
remote_ip = request.args.get('ip', request.remote_addr)
maxmind = maxminddb_reader.get(remote_ip)["location"]
return maxmind["location"] if maxmind else None
maxmind = maxminddb_reader.get(remote_ip)
return maxmind.get("location") if maxmind else None
@app.route("/")
@ -138,6 +139,12 @@ def isa_page(item_id):
item = api.get_item(item_id)
if request.method == "POST":
tag_or_key = request.form["tag_or_key"]
extra = model.ItemExtraKeys(item=item, tag_or_key=tag_or_key)
database.session.add(extra)
database.session.commit()
flash("extra OSM tag/key added")
return redirect(url_for(request.endpoint, item_id=item_id))
q = model.ItemExtraKeys.query.filter_by(item=item)
@ -240,12 +247,19 @@ def identifier_page(pid):
def map_start_page():
loc = get_user_location()
if loc:
lat, lon = loc["latitude"], loc["longitude"]
radius = loc["accuracy_radius"]
else:
lat, lon = 42.2917, -85.5872
radius = 5
return redirect(url_for(
'map_location',
lat=f'{loc["latitude"]:.5f}',
lon=f'{loc["longitude"]:.5f}',
lat=f'{lat:.5f}',
lon=f'{lon:.5f}',
zoom=16,
radius=loc["accuracy_radius"],
radius=radius,
ip=request.args.get('ip'),
))
@ -285,9 +299,22 @@ def search_page():
@app.route("/map/<int:zoom>/<float(signed=True):lat>/<float(signed=True):lon>")
def map_location(zoom, lat, lon):
qid = request.args.get("item")
isa_param = request.args.get("isa")
if qid:
api.get_item(qid[1:])
isa_list = []
if isa_param:
for isa_qid in isa_param.split(";"):
isa = api.get_item(isa_qid[1:])
if not isa:
continue
cur = {
"qid": isa.qid,
"label": isa.label(),
}
isa_list.append(cur)
return render_template(
"map.html",
active_tab="map",
@ -298,9 +325,40 @@ def map_location(zoom, lat, lon):
username=get_username(),
mode="map",
q=None,
item_type_filter=isa_list,
)
@app.route("/item/Q<int:item_id>")
def lookup_item(item_id):
item = api.get_item(item_id)
if not item:
# TODO: show nicer page for Wikidata item not found
return abort(404)
try:
lat, lon = item.locations[0].get_lat_lon()
except IndexError:
# TODO: show nicer page for Wikidata item without coordinates
return abort(404)
return render_template(
"map.html",
active_tab="map",
zoom=16,
lat=lat,
lon=lon,
username=get_username(),
mode="map",
q=None,
qid=item.qid,
item_type_filter=[],
)
url = url_for("map_location", zoom=16, lat=lat, lon=lon, item=item.qid)
return redirect(url)
@app.route("/search/map")
def search_map_page():
user_lat, user_lon = get_user_location() or (None, None)
@ -394,6 +452,15 @@ def api_wikidata_items():
t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **ret)
@app.route("/api/1/place/<osm_type>/<int:osm_id>")
def api_place_items(osm_type, osm_id):
t0 = time()
ret = api.get_place_items(osm_type, osm_id)
t1 = time() - t0
return cors_jsonify(success=True, duration=t1, **ret)
@app.route("/api/1/osm")
def api_osm_objects():
@ -540,7 +607,11 @@ def api_search():
hit["name"] = nominatim.get_hit_name(hit)
hit["label"] = nominatim.get_hit_label(hit)
hit["address"] = list(hit["address"].items())
hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}"
if "osm_type" in hit and "osm_id" in hit:
hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}"
else:
print(hit)
print(q)
return cors_jsonify(success=True, hits=hits)
@ -805,6 +876,18 @@ def api_save_changeset(session_id):
return api_call(session_id)
@app.route("/sql", methods=["GET", "POST"])
def run_sql():
if request.method != "POST":
return render_template("run_sql.html")
sql = request.form["sql"]
conn = database.session.connection()
result = conn.execute(sqlalchemy.text(sql))
return render_template("run_sql.html", result=result)
def api_real_save_changeset(session_id):
es = model.EditSession.query.get(session_id)