Update

frontend/App.vue

@@ -303,7 +303,7 @@
                     v-bind:key="isa.qid"
                     v-for="isa in item_type_hits"
                     href="#"
-                    @click.prevent="item_type_filters.includes(isa) || item_type_filters.push(isa)"
+                    @click.prevent="add_item_type_filter(isa)"
                     >
                   {{ isa.label }} ({{ isa.qid }})
                 </a>
@@ -458,12 +458,33 @@
             <br>{{ wd_item.aliases.join("; ") }}
           </span>
 
+          <br><strong>item coordinates</strong>
+          <span v-for="marker in wd_item.markers">
+            <br><a target="_blank" :href="marker_osm_url(marker)" @click.stop>
+              {{ marker[0].toFixed(5) }},
+              {{ marker[1].toFixed(5) }}
+              <i class="fa fa-map-o"></i></a>
+          </span>
+
           <br><strong>item type</strong>
           <span v-bind:key="`isa-${wd_item.qid}-${isa.qid}`" v-for="isa in wd_item.isa_list">
             <br><a :href="qid_url(isa.qid)" target="_blank">{{isa.label}}</a> ({{isa.qid}})
             <a :href="'/isa/' + isa.qid" target="_blank"><i class="fa fa-pencil-square-o"></i></a>
           </span>
 
+          <span v-if="wd_item.wikipedia.length > 0">
+            <br>
+            <strong>
+              Wikipedia
+              <i class="fa fa-external-link"></i>
+            </strong>
+            <br>
+            <span v-for="wp in wd_item.wikipedia">
+              <a :href="wikipedia_link(wp.lang, wp.title)" target="_blank">{{wp.lang}}</a>
+              &nbsp;
+            </span>
+          </span>
+
           <span v-if="wd_item.street_address.length">
             <br><strong>street address</strong>
             <br>{{wd_item.street_address[0]}}
@@ -524,6 +545,15 @@
             </a>
           </span>
 
+          <span v-if="wd_item.commons !== undefined">
+            <br><strong>Images on Commons</strong>
+            <br><a
+                :href="`https://commons.wikimedia.org/wiki/${wd_item.commons.replaceAll(' ', '_')}`"
+                target="_blank">
+              {{wd_item.commons}} <i class="fa fa-external-link"></i>
+            </a>
+          </span>
+
         </div></div>
         </div>
 
@@ -548,7 +578,7 @@
         <div v-if="current_item.nearby && !current_item.nearby.length">
           <strong>No OSM matches found nearby</strong>
 
-          <div class="mt-2" v-if="current_item.tag_or_key_list.length">
+          <div class="mt-2" v-if="current_item.tag_or_key_list && current_item.tag_or_key_list.length">
             <p>The OSM tags/keys used as the search criteria to find matching
             OSM objects are listed below, along with the Wikidata item that was
             the source.</p>
@@ -766,6 +796,8 @@ export default {
     startLon: Number,
     startZoom: Number,
     startRadius: Number,
+    startItem: String,
+    startItemTypeFilter: Array,
     username: String,
     startMode: String,
     q: String,
@@ -994,6 +1026,22 @@ export default {
     }
   },
   methods: {
+    wikipedia_link(lang, title) {
+      var norm_title = title.replaceAll(" ", "_");
+      return `https://${lang}.wikipedia.org/wiki/${norm_title}`;
+    },
+    marker_osm_url(marker) {
+      var lat = marker[0].toFixed(5);
+      var lon = marker[1].toFixed(5);
+      return `https://www.openstreetmap.org/?mlat=${lat}&mlon=${lon}#map=18/${lat}/${lon}`
+    },
+    add_item_type_filter(isa) {
+      if (this.item_type_filters.includes(isa)) {
+        return;
+      }
+      this.item_type_filters.push(isa);
+      this.update_map_path();
+    },
     api_call(endpoint, options) {
       var url = `${this.api_base_url}/api/1/${endpoint}`;
       return axios.get(url, options).catch(this.show_api_error_modal);
@@ -1216,14 +1264,19 @@ export default {
       this.isa_ticked = Object.keys(this.isa_labels);
     },
     build_map_path() {
+      if (this.current_item) {
+        return `/item/${this.current_qid}`;
+      }
       var zoom = this.map.getZoom();
       var c = this.map.getCenter();
       var lat = c.lat.toFixed(5);
       var lng = c.lng.toFixed(5);
       var path = `/map/${zoom}/${lat}/${lng}`;
-      if (this.current_item) {
+
-        path += `?item=${this.current_qid}`;
+      if (this.item_type_filters.length) {
+        path += "?isa=" + this.item_type_filters.map((t) => t.qid).join(";");
       }
+
       return path;
     },
 
@@ -1755,6 +1808,11 @@ export default {
     this.zoom = this.startZoom;
     this.mode = this.startMode;
     this.changeset_comment = this.defaultComment || '+wikidata';
+    console.log(this.startItemTypeFilter);
+    if (this.startItemTypeFilter.length) {
+      this.show_item_type_filter = true;
+    }
+    this.item_type_filters = this.startItemTypeFilter;
   },
   mounted() {
 
@@ -1764,7 +1822,6 @@ export default {
         zoom: this.zoom || 16,
       };
 
-
       var map = L.map("map", options);
       var osm_url = "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png";
       var tile_url = "https://tile-c.openstreetmap.fr/hot/{z}/{x}/{y}.png";
@@ -1788,14 +1845,14 @@ export default {
         this.search_text = this.q.trim();
         this.run_search();
       } else {
-        this.detail_qid = this.qid_from_url();
+        this.detail_qid = this.startItem;
         if (this.detail_qid) {
           this.load_wikidata_items(bounds);
         } else {
           this.auto_load(bounds);
-        }
           this.update_map_path();
         }
+      }
 
       window.onpopstate = this.onpopstate;
     });

matcher/api.py

@@ -38,6 +38,14 @@ skip_tags = {
 }
 
 def get_country_iso3166_1(lat, lon):
+    """
+    For a given lat/lon return a set of ISO country codes.
+
+    Also cache the country code in the global object.
+
+    Normally there should be only one country.
+    """
+
     point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), srid)
     alpha2_codes = set()
     q = model.Polygon.query.filter(func.ST_Covers(model.Polygon.way, point),
@@ -57,7 +65,18 @@ def is_street_number_first(lat, lon):
         return True
 
     alpha2 = get_country_iso3166_1(lat, lon)
-    alpha2_number_first = {'GB', 'IE', 'US', 'MX', 'CA', 'FR', 'AU', 'NZ', 'ZA'}
+    # Incomplete list of countries that put street number first.
+    alpha2_number_first = {
+        'GB',  # United Kingdom
+        'IE',  # Ireland
+        'US',  # United States
+        'MX',  # Mexico
+        'CA',  # Canada
+        'FR',  # France
+        'AU',  # Australia
+        'NZ',  # New Zealand
+        'ZA',  # South Africa
+    }
 
     return bool(alpha2_number_first & alpha2)
 
@@ -92,6 +111,7 @@ def make_envelope_around_point(lat, lon, distance):
     return func.ST_MakeEnvelope(west, south, east, north, srid)
 
 def drop_way_area(tags):
+    """ Remove the way_area field from a tags dict. """
     if "way_area" in tags:
         del tags["way_area"]
     return tags
@@ -122,6 +142,8 @@ def get_part_of(table_name, src_id, bbox):
     } for osm_id, tags, area in conn.execute(s)]
 
 def get_and_save_item(qid):
+    """ Download an item from Wikidata and cache it in the database. """
+
     entity = wikidata_api.get_entity(qid)
     entity_qid = entity["id"]
     if entity_qid != qid:
@@ -396,7 +418,6 @@ def add_isa_filter(q, isa_qids):
     )
 
     subclass_qid = {qid for qid, in q_subclass.all()}
-    # print(subclass_qid)
 
     isa = func.jsonb_path_query_array(
         model.Item.claims,
@@ -419,7 +440,7 @@ def wikidata_items_count(bounds, isa_filter=None):
 
     return q.count()
 
-def wikidata_isa_counts(bounds):
+def wikidata_isa_counts(bounds, isa_filter=None):
     db_bbox = make_envelope(bounds)
 
     q = (
@@ -427,6 +448,9 @@ def wikidata_isa_counts(bounds):
         .filter(func.ST_Covers(db_bbox, model.ItemLocation.location))
     )
 
+    if isa_filter:
+        q = add_isa_filter(q, isa_filter)
+
     db_items = q.all()
 
     counts = get_isa_count(db_items)
@@ -605,7 +629,11 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
     item_id = item.item_id
     item_is_linear_feature = item.is_linear_feature()
     item_is_street = item.is_street()
-    item_names = {n.lower() for n in item.names().keys()}
+    item_names_dict = item.names()
+    if item_names_dict:
+        item_names = {n.lower() for n in item_names_dict.keys()}
+    else:
+        item_names = set()
 
     check_is_street_number_first(item.locations[0].get_lat_lon())
 
@@ -702,6 +730,8 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
 
         shape = "area" if table == "polygon" else table
 
+        item_identifier_tags = item.get_identifiers_tags()
+
         cur = {
             "identifier": f"{osm_type}/{osm_id}",
             "type": osm_type,
@@ -733,6 +763,8 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
     return nearby
 
 def get_item(item_id):
+    """ Retrieve a Wikidata item, either from the database or from Wikidata. """
+
     item = model.Item.query.get(item_id)
     return item or get_and_save_item(f"Q{item_id}")
 
@@ -763,6 +795,11 @@ def check_is_street_number_first(latlng):
     g.street_number_first = is_street_number_first(*latlng)
 
 def item_detail(item):
+    unsupported_relation_types = {
+        'Q194356',   # wind farm
+        'Q2175765',  # tram stop
+    }
+
     locations = [list(i.get_lat_lon()) for i in item.locations]
     if not hasattr(g, 'street_number_first'):
         g.street_number_first = is_street_number_first(*locations[0])
@@ -783,6 +820,11 @@ def item_detail(item):
         })
 
     isa_items = [get_item(isa["numeric-id"]) for isa in item.get_isa()]
+    isa_lookup = {isa.qid: isa for isa in isa_items}
+
+    wikipedia_links = [{"lang": site[:-4], "title": link["title"]}
+                       for site, link in sorted(item.sitelinks.items())
+                       if site.endswith("wiki") and len(site) < 8]
 
     d = {
         "qid": item.qid,
@@ -797,11 +839,21 @@ def item_detail(item):
         "p1619": item.time_claim("P1619"),
         "p576": item.time_claim("P576"),
         "heritage_designation": heritage_designation,
+        "wikipedia": wikipedia_links,
+        "identifiers": item.get_identifiers(),
     }
 
     if aliases := item.get_aliases():
         d["aliases"] = aliases
 
+    if "commonswiki" in item.sitelinks:
+        d["commons"] = item.sitelinks["commonswiki"]["title"]
+
+    unsupported = isa_lookup.keys() & unsupported_relation_types
+    if unsupported:
+        d["unsupported_relation_types"] = [isa for isa in d["isa_list"]
+                                           if isa["qid"] in isa_lookup]
+
     return d
 
 
@@ -892,3 +944,21 @@ def isa_incremental_search(search_terms):
         }
         ret.append(cur)
     return ret
+
+def get_place_items(osm_type, osm_id):
+    src_id = osm_id * {'way': 1, 'relation': -1}[osm_type]
+
+    q = (model.Item.query
+                   .join(model.ItemLocation)
+                   .join(model.Polygon, func.ST_Covers(model.Polygon.way, model.ItemLocation.location))
+                   .filter(model.Polygon.src_id == src_id))
+    # sql = q.statement.compile(compile_kwargs={"literal_binds": True})
+
+    item_count = q.count()
+    items = []
+    for item in q:
+        keys = ["item_id", "labels", "descriptions", "aliases", "sitelinks", "claims"]
+        item_dict = {key: getattr(item, key) for key in keys}
+        items.append(item_dict)
+
+    return {"count": item_count, "items": items}

matcher/nominatim.py

@@ -83,8 +83,8 @@ def get_hit_name(hit):
     if len(address) == 1:
         return n1
 
-    country = address.pop("country")
+    country = address.pop("country", None)
-    country_code = address.pop("country_code")
+    country_code = address.pop("country_code", None)
     if country_code:
         country_code == country_code.lower()
 

matcher/utils.py

@@ -5,7 +5,6 @@ import json
 import math
 import user_agents
 import re
-import pattern.en
 from datetime import date
 from num2words import num2words
 
@@ -160,18 +159,6 @@ def is_in_range(address_range, address):
     return False
 
 
-def pluralize_label(label):
-    text = label["value"]
-    if label["language"] != "en":
-        return text
-
-    # pattern.en.pluralize has the plural of 'mine' as 'ours'
-    if text == "mine":
-        return "mines"
-
-    return pattern.en.pluralize(text)
-
-
 def format_wikibase_time(v):
     p = v["precision"]
     t = v["time"]
@@ -180,11 +167,12 @@ def format_wikibase_time(v):
     # example: https://www.wikidata.org/wiki/Q108266998
 
     if p == 11:
-        return date.fromisoformat(t[1:11]).strftime("%d %B %Y")
+        return date.fromisoformat(t[1:11]).strftime("%-d %B %Y")
     if p == 10:
         return date.fromisoformat(t[1:8] + "-01").strftime("%B %Y")
     if p == 9:
         return t[1:5]
     if p == 7:
         century = ((int(t[:5]) - 1) // 100) + 1
-        return num2words(century, to="ordinal_num") + " century"
+        end = " BC" if century < 0 else ""
+        return num2words(abs(century), to="ordinal_num") + " century" + end

templates/isa.html

@@ -4,6 +4,7 @@
 
 {% block content %}
 <div class="container my-2">
+  {% include "flash_msg.html" %}
 
   <h1>{{ self.title() }}</h1>
 
@@ -13,7 +14,7 @@
   </a></div>
 
   <div class="my-2">
-  <form method="POST">
+  <form method="GET" action="{{ url_for("refresh_item", item_id=item.item_id) }}">
     <input type="hidden" name="action" value="refresh">
     <input type="submit" value="refresh item" class="btn btn-sm btn-primary">
   </form>

templates/map.html

@@ -4,26 +4,32 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>Wikidata items linked to OSM</title>
-    <link rel="stylesheet" href="https://unpkg.com/bootstrap@5.0.1/dist/css/bootstrap.min.css">
+    <!--
-    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css">
+    <link rel="stylesheet" href="https://unpkg.com/bootstrap@5.1.3/dist/css/bootstrap.min.css">
-    <link rel="stylesheet" href="https://unpkg.com/fork-awesome@1.1.7/css/fork-awesome.min.css">
+
+    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.8.0/dist/leaflet.css">
+    <link rel="stylesheet" href="https://unpkg.com/fork-awesome@1.2.0/css/fork-awesome.min.css">
     <link rel="stylesheet" href="https://unpkg.com/leaflet-extra-markers@1.2.1/dist/css/leaflet.extra-markers.min.css">
+    -->
+
+    <link rel="stylesheet" href="{{ url_for("static", filename="frontend/style.css") }}">
   </head>
   {% from "navbar.html" import navbar with context %}
   <body>
     {% block nav %}{{ navbar() }}{% endblock %}
     <div id="app"></div>
 
-    <script src="https://unpkg.com/bootstrap@5.0.1/dist/js/bootstrap.bundle.min.js" integrity="sha384-gtEjrD/SeCtmISkJkNUaaKMoLD0//ElJ19smozuHV6z3Iehds+3Ulb9Bn9Plx0x4" crossorigin="anonymous"></script>
+    <!-- <script src="https://unpkg.com/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js" crossorigin="anonymous"></script> -->
     <script type="module">
-      import { createApp } from "https://cdn.skypack.dev/vue@^3.0.11";
+      import main from {{ url_for('static', filename='frontend/owl.es.js') | tojson }};
-      import App from {{ url_for('static', filename='snowpack/App.vue.js') | tojson }};
 
       const props = {
         startLat: {{ lat }},
         startLon: {{ lon }},
         startZoom: {{ zoom }},
-        startRadius: {{ radius | tojson }},
+        startRadius: {{ (radius or None) | tojson }},
+        startItem: {{ (qid or None) | tojson }},
+        startItemTypeFilter: {{ (item_type_filter or []) | tojson }},
         defaultComment: {{ config.DEFAULT_COMMENT | tojson }},
         username: {{ username | tojson }},
         startMode: {{ mode | tojson }},
@@ -32,7 +38,7 @@
         mockUpload: false,
       };
 
-      const app = createApp(App, props).mount('#app');
+      main(props);
 
     </script>
   </body>

web_view.py

@@ -1,12 +1,12 @@
 #!/usr/bin/python3.9
 
 from flask import (Flask, render_template, request, jsonify, redirect, url_for, g,
-                   flash, session, Response, stream_with_context)
+                   flash, session, Response, stream_with_context, abort, send_file)
 from sqlalchemy import func
 from sqlalchemy.sql.expression import update
 from matcher import (nominatim, model, database, commons, wikidata, wikidata_api,
                      osm_oauth, edit, mail, api, error_mail)
-from werkzeug.debug.tbtools import get_current_traceback
+# from werkzeug.debug.tbtools import get_current_traceback
 from matcher.data import property_map
 from time import time, sleep
 from requests_oauthlib import OAuth1Session
@@ -19,6 +19,7 @@ import json
 import GeoIP
 import re
 import maxminddb
+import sqlalchemy
 
 srid = 4326
 re_point = re.compile(r'^POINT\((.+) (.+)\)$')
@@ -54,27 +55,27 @@ def dict_repr_values(d):
     return {key: repr(value) for key, value in d.items()}
 
 
-@app.errorhandler(werkzeug.exceptions.InternalServerError)
+# @app.errorhandler(werkzeug.exceptions.InternalServerError)
-def exception_handler(e):
+# def exception_handler(e):
-    tb = get_current_traceback()
+#     tb = get_current_traceback()
-    last_frame = next(frame for frame in reversed(tb.frames) if not frame.is_library)
+#     last_frame = next(frame for frame in reversed(tb.frames) if not frame.is_library)
-    last_frame_args = inspect.getargs(last_frame.code)
+#     last_frame_args = inspect.getargs(last_frame.code)
-    if request.path.startswith("/api/"):
+#     if request.path.startswith("/api/"):
-        return cors_jsonify({
+#         return cors_jsonify({
-            "success": False,
+#             "success": False,
-            "error": tb.exception,
+#             "error": tb.exception,
-            "traceback": tb.plaintext,
+#             "traceback": tb.plaintext,
-            "locals": dict_repr_values(last_frame.locals),
+#             "locals": dict_repr_values(last_frame.locals),
-            "last_function": {
+#             "last_function": {
-                "name": tb.frames[-1].function_name,
+#                 "name": tb.frames[-1].function_name,
-                "args": repr(last_frame_args),
+#                 "args": repr(last_frame_args),
-            },
+#             },
-        }), 500
+#         }), 500
-
+# 
-    return render_template('show_error.html',
+#     return render_template('show_error.html',
-                           tb=tb,
+#                            tb=tb,
-                           last_frame=last_frame,
+#                            last_frame=last_frame,
-                           last_frame_args=last_frame_args), 500
+#                            last_frame_args=last_frame_args), 500
 
 def cors_jsonify(*args, **kwargs):
     response = jsonify(*args, **kwargs)
@@ -113,8 +114,8 @@ def geoip_user_record():
 
 def get_user_location():
     remote_ip = request.args.get('ip', request.remote_addr)
-    maxmind = maxminddb_reader.get(remote_ip)["location"]
+    maxmind = maxminddb_reader.get(remote_ip)
-    return maxmind["location"] if maxmind else None
+    return maxmind.get("location") if maxmind else None
 
 
 @app.route("/")
@@ -138,6 +139,12 @@ def isa_page(item_id):
     item = api.get_item(item_id)
 
     if request.method == "POST":
+        tag_or_key = request.form["tag_or_key"]
+        extra = model.ItemExtraKeys(item=item, tag_or_key=tag_or_key)
+        database.session.add(extra)
+        database.session.commit()
+        flash("extra OSM tag/key added")
+
         return redirect(url_for(request.endpoint, item_id=item_id))
 
     q = model.ItemExtraKeys.query.filter_by(item=item)
@@ -240,12 +247,19 @@ def identifier_page(pid):
 def map_start_page():
     loc = get_user_location()
 
+    if loc:
+        lat, lon = loc["latitude"], loc["longitude"]
+        radius = loc["accuracy_radius"]
+    else:
+        lat, lon = 42.2917, -85.5872
+        radius = 5
+
     return redirect(url_for(
         'map_location',
-        lat=f'{loc["latitude"]:.5f}',
+        lat=f'{lat:.5f}',
-        lon=f'{loc["longitude"]:.5f}',
+        lon=f'{lon:.5f}',
         zoom=16,
-        radius=loc["accuracy_radius"],
+        radius=radius,
         ip=request.args.get('ip'),
     ))
 
@@ -285,9 +299,22 @@ def search_page():
 @app.route("/map/<int:zoom>/<float(signed=True):lat>/<float(signed=True):lon>")
 def map_location(zoom, lat, lon):
     qid = request.args.get("item")
+    isa_param = request.args.get("isa")
     if qid:
         api.get_item(qid[1:])
 
+    isa_list = []
+    if isa_param:
+        for isa_qid in isa_param.split(";"):
+            isa = api.get_item(isa_qid[1:])
+            if not isa:
+                continue
+            cur = {
+                "qid": isa.qid,
+                "label": isa.label(),
+            }
+            isa_list.append(cur)
+
     return render_template(
         "map.html",
         active_tab="map",
@@ -298,9 +325,40 @@ def map_location(zoom, lat, lon):
         username=get_username(),
         mode="map",
         q=None,
+        item_type_filter=isa_list,
     )
 
 
+@app.route("/item/Q<int:item_id>")
+def lookup_item(item_id):
+    item = api.get_item(item_id)
+    if not item:
+        # TODO: show nicer page for Wikidata item not found
+        return abort(404)
+
+    try:
+        lat, lon = item.locations[0].get_lat_lon()
+    except IndexError:
+        # TODO: show nicer page for Wikidata item without coordinates
+        return abort(404)
+
+    return render_template(
+        "map.html",
+        active_tab="map",
+        zoom=16,
+        lat=lat,
+        lon=lon,
+        username=get_username(),
+        mode="map",
+        q=None,
+        qid=item.qid,
+        item_type_filter=[],
+    )
+
+    url = url_for("map_location", zoom=16, lat=lat, lon=lon, item=item.qid)
+    return redirect(url)
+
+
 @app.route("/search/map")
 def search_map_page():
     user_lat, user_lon = get_user_location() or (None, None)
@@ -394,6 +452,15 @@ def api_wikidata_items():
     t1 = time() - t0
     return cors_jsonify(success=True, duration=t1, **ret)
 
+@app.route("/api/1/place/<osm_type>/<int:osm_id>")
+def api_place_items(osm_type, osm_id):
+    t0 = time()
+
+    ret = api.get_place_items(osm_type, osm_id)
+
+    t1 = time() - t0
+    return cors_jsonify(success=True, duration=t1, **ret)
+
 
 @app.route("/api/1/osm")
 def api_osm_objects():
@@ -540,7 +607,11 @@ def api_search():
         hit["name"] = nominatim.get_hit_name(hit)
         hit["label"] = nominatim.get_hit_label(hit)
         hit["address"] = list(hit["address"].items())
+        if "osm_type" in hit and "osm_id" in hit:
             hit["identifier"] = f"{hit['osm_type']}/{hit['osm_id']}"
+        else:
+            print(hit)
+            print(q)
 
     return cors_jsonify(success=True, hits=hits)
 
@@ -805,6 +876,18 @@ def api_save_changeset(session_id):
     return api_call(session_id)
 
 
+@app.route("/sql", methods=["GET", "POST"])
+def run_sql():
+    if request.method != "POST":
+        return render_template("run_sql.html")
+
+    sql = request.form["sql"]
+    conn = database.session.connection()
+    result = conn.execute(sqlalchemy.text(sql))
+
+    return render_template("run_sql.html", result=result)
+
+
 def api_real_save_changeset(session_id):
     es = model.EditSession.query.get(session_id)