Merge branch 'main' of github.com:EdwardBetts/owl-map

2023-05-14 16:12:58 +02:00 · 2023-05-14 16:12:58 +02:00 · 33bda7f54b
parent c11794d3f2 b9b9728fc6
commit 33bda7f54b
12 changed files with 1335 additions and 281 deletions
--- a/matcher.style
+++ b/matcher.style
@ -0,0 +1,162 @@
+# This is the default osm2pgsql .style file that comes with osm2pgsql.
+#
+# A .style file has 4 columns that define how OSM objects end up in tables in
+# the database and what columns are created. It interacts with the command-line
+# hstore options.
+#
+# Columns
+# =======
+#
+# OsmType: This is either "node", "way" or "node,way" and indicates if this tag
+# applies to nodes, ways, or both.
+#
+# Tag: The tag
+#
+# DataType: The type of the column to be created. Normally "text"
+#
+# Flags: Flags that indicate what table the OSM object is moved into.
+#
+# There are 6 possible flags. These flags are used both to indicate if a column
+# should be created, and if ways with the tag are assumed to be areas. The area
+# assumptions can be overridden with an area=yes/no tag
+#
+# polygon - Create a column for this tag, and objects with the tag are areas
+#
+# linear - Create a column for this tag
+#
+# nocolumn - Override the above and don't create a column for the tag, but do
+# include objects with this tag
+#
+# phstore - Same as polygon,nocolumn for backward compatibility
+#
+# delete - Drop this tag completely and don't create a column for it. This also
+# prevents the tag from being added to hstore columns
+#
+# nocache - Deprecated and does nothing
+#
+# If an object has a tag that indicates it is an area or has area=yes/1,
+# osm2pgsql will try to turn it into an area. If it succeeds, it places it in
+# the polygon table. If it fails (e.g. not a closed way) it places it in the
+# line table.
+#
+# Nodes are never placed into the polygon or line table and are always placed in
+# the point table.
+#
+# Hstore
+# ======
+#
+# The options --hstore, --hstore-match-only, and --hstore-all interact with
+# the .style file.
+#
+# With --hstore any tags without a column will be added to the hstore column.
+# This will also cause all objects to be kept.
+#
+# With --hstore-match-only the behavior for tags is the same, but objects are
+# only kept if they have a non-NULL value in one of the columns.
+#
+# With --hstore-all all tags are added to the hstore column unless they appear
+# in the style file with a delete flag, causing duplication between the normal
+# columns and the hstore column.
+#
+# Special database columns
+# ========================
+#
+# There are some special database columns that if present in the .style file
+# will be populated by osm2pgsql.
+#
+# These are
+#
+# z_order - datatype int4
+#
+# way_area - datatype real. The area of the way, in the units of the projection
+# (e.g. square mercator meters). Only applies to areas
+#
+# osm_user - datatype text
+# osm_uid - datatype integer
+# osm_version - datatype integer
+# osm_changeset - datatype integer
+# osm_timestamp - datatype timestamptz(0).
+# Used with the --extra-attributes option to include metadata in the database.
+# If importing with both --hstore and --extra-attributes the meta-data will
+# end up in the tags hstore column regardless of the style file.
+
+# OsmType  Tag          DataType     Flags
+node,way   access       text         linear
+node,way   addr:housename      text  linear
+node,way   addr:housenumber    text  linear
+node,way   addr:interpolation  text  linear
+node,way   admin_level  text         linear
+node,way   aerialway    text         linear
+node,way   aeroway      text         polygon
+node,way   amenity      text         polygon
+node,way   area         text         polygon # hard coded support for area=1/yes => polygon is in osm2pgsql
+node,way   barrier      text         linear
+node,way   bicycle      text         linear
+node,way   brand        text         linear
+node,way   bridge       text         linear
+node,way   boundary     text         linear
+node,way   building     text         polygon
+node       capital      text         linear
+node,way   construction text         linear
+node,way   covered      text         linear
+node,way   culvert      text         linear
+node,way   cutting      text         linear
+node,way   denomination text         linear
+node,way   disused      text         linear
+node       ele          text         linear
+node,way   embankment   text         linear
+node,way   foot         text         linear
+node,way   generator:source    text  linear
+node,way   harbour      text         polygon
+node,way   highway      text         linear
+node,way   historic     text         polygon
+node,way   horse        text         linear
+node,way   intermittent text         linear
+node,way   junction     text         linear
+node,way   landuse      text         polygon
+node,way   layer        text         linear
+node,way   leisure      text         polygon
+node,way   lock         text         linear
+node,way   man_made     text         polygon
+node,way   military     text         polygon
+node,way   motorcar     text         linear
+node,way   name         text         linear
+node,way   natural      text         polygon  # natural=coastline tags are discarded by a hard coded rule in osm2pgsql
+node,way   office       text         polygon
+node,way   oneway       text         linear
+node,way   operator     text         linear
+node,way   place        text         polygon
+node,way   population   text         linear
+node,way   power        text         polygon
+node,way   power_source text         linear
+node,way   public_transport text     polygon
+node,way   railway      text         linear
+node,way   ref          text         linear
+node,way   religion     text         linear
+node,way   route        text         linear
+node,way   service      text         linear
+node,way   shop         text         polygon
+node,way   sport        text         polygon
+node,way   surface      text         linear
+node,way   toll         text         linear
+node,way   tourism      text         polygon
+node,way   tower:type   text         linear
+way        tracktype    text         linear
+node,way   tunnel       text         linear
+node,way   water        text         polygon
+node,way   waterway     text         polygon
+node,way   wetland      text         polygon
+node,way   width        text         linear
+node,way   wood         text         linear
+node,way   z_order      int4         linear # This is calculated during import
+way        way_area     real         linear # This is calculated during import
+
+# Area tags
+# We don't make columns for these tags, but objects with them are areas.
+# Mainly for use with hstore
+way         abandoned:aeroway       text    polygon,nocolumn
+way         abandoned:amenity       text    polygon,nocolumn
+way         abandoned:building      text    polygon,nocolumn
+way         abandoned:landuse       text    polygon,nocolumn
+way         abandoned:power         text    polygon,nocolumn
+way         area:highway            text    polygon,nocolumn
--- a/matcher/init.py
+++ b/matcher/init.py
@ -4,7 +4,9 @@
 CallParams = dict[str, str | int]

 user_agent = (
-    "osm-wikidata/0.1 (https://github.com/EdwardBetts/osm-wikidata; edward@4angle.com)"
+    "osm-wikidata/0.2"
+    + " (https://github.com/EdwardBetts/osm-wikidata;"
+    + " edward@4angle.com)"
 )


--- a/matcher/api.py
+++ b/matcher/api.py
@ -40,13 +40,14 @@ skip_tags = {
 }


-def get_country_iso3166_1(lat: float, lon: float) -> set[str]:
+def get_country_iso3166_1(lat, lon):
    """For a given lat/lon return a set of ISO country codes.

    Also cache the country code in the global object.

    Normally there should be only one country.
    """
+
    point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), srid)
    alpha2_codes = set()
    q = model.Polygon.query.filter(
@ -263,7 +264,7 @@ WHERE tags ? 'wikidata'
    conn = database.session.connection()
    result = conn.execute(text(sql))

-    print(sql)
+    # print(sql)

    point_sql = (
        f"""
@ -788,7 +789,7 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
    if limit:
        s = s.limit(limit)

-    print(s.compile(compile_kwargs={"literal_binds": True}))
+    # print(s.compile(compile_kwargs={"literal_binds": True}))

    conn = database.session.connection()
    nearby = []
@ -1031,7 +1032,7 @@ def isa_incremental_search(search_terms):
        func.length(en_label) < 20,
    )

-    print(q.statement.compile(compile_kwargs={"literal_binds": True}))
+    # print(q.statement.compile(compile_kwargs={"literal_binds": True}))

    ret = []
    for item in q:
--- a/matcher/mail.py
+++ b/matcher/mail.py
@ -1,10 +1,11 @@
-from flask import current_app, g, request, has_request_context
+import smtplib
+import sys
+import traceback
 from email.mime.text import MIMEText
 from email.utils import formatdate, make_msgid
 from pprint import pformat
-import smtplib
-import traceback
-import sys
+
+from flask import current_app, g, has_request_context, request


 def send_mail(subject, body, config=None):
@ -71,7 +72,7 @@ def open_changeset_error(session_id, changeset, r):
    username = g.user.username
    body = f"""
 user: {username}
-page: {url}
+page: {r.url}

 message user: https://www.openstreetmap.org/message/new/{username}

--- a/matcher/model.py
+++ b/matcher/model.py
@ -1,89 +1,159 @@
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.schema import ForeignKey, Column
-from sqlalchemy.orm import relationship, column_property, deferred, backref
-from sqlalchemy import func
-from sqlalchemy.types import Integer, String, Float, Boolean, DateTime, Text, BigInteger
-from sqlalchemy.dialects import postgresql
-from sqlalchemy.sql.expression import cast
-from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy.ext.declarative import declared_attr
-from geoalchemy2 import Geometry
-from collections import defaultdict
-from flask_login import UserMixin
-from .database import session, now_utc
-from . import wikidata, utils, mail
 import json
 import re
+import typing
+from collections import defaultdict
+from typing import Any
+
+from flask_login import UserMixin
+from geoalchemy2 import Geometry
+from sqlalchemy import func
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.ext.declarative import declarative_base, declared_attr
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy.orm import backref, column_property, deferred, relationship
+from sqlalchemy.orm.collections import attribute_mapped_collection
+from sqlalchemy.schema import Column, ForeignKey
+from sqlalchemy.sql.expression import cast
+from sqlalchemy.types import BigInteger, Boolean, DateTime, Float, Integer, String, Text
+
+from . import mail, utils, wikidata
+from .database import now_utc, session

 Base = declarative_base()
 Base.query = session.query_property()

-re_point = re.compile(r'^POINT\((.+) (.+)\)$')
+re_point = re.compile(r"^POINT\((.+) (.+)\)$")
+
+osm_type_enum = postgresql.ENUM(
+    "node", "way", "relation", name="osm_type_enum", metadata=Base.metadata
+)
+
+re_lau_code = re.compile(r"^[A-Z]{2}([^A-Z].+)$")  # 'LAU (local administrative unit)'
+
+property_map = [
+    ("P238", ["iata"], "IATA airport code"),
+    ("P239", ["icao"], "ICAO airport code"),
+    ("P240", ["faa", "ref"], "FAA airport code"),
+    ("P296", ["ref", "ref:train", "railway:ref"], "station code"),
+    ("P300", ["ISO3166-2"], "ISO 3166-2 code"),
+    ("P359", ["ref:rce"], "Rijksmonument ID"),
+    ("P590", ["ref:gnis", "GNISID", "gnis:id", "gnis:feature_id"], "USGS GNIS ID"),
+    ("P649", ["ref:nrhp"], "NRHP reference number"),
+    ("P722", ["uic_ref"], "UIC station code"),
+    ("P782", ["ref"], "LAU (local administrative unit)"),
+    ("P836", ["ref:gss"], "UK Government Statistical Service code"),
+    ("P856", ["website", "contact:website", "url"], "website"),
+    ("P882", ["nist:fips_code"], "FIPS 6-4 (US counties)"),
+    ("P901", ["ref:fips"], "FIPS 10-4 (countries and regions)"),
+    # A UIC id can be a IBNR, but not every IBNR is an UIC id
+    ("P954", ["uic_ref"], "IBNR ID"),
+    ("P981", ["ref:woonplaatscode"], "BAG code for Dutch residencies"),
+    ("P1216", ["HE_ref"], "National Heritage List for England number"),
+    ("P2253", ["ref:edubase"], "EDUBase URN"),
+    ("P2815", ["esr:user", "ref", "ref:train"], "ESR station code"),
+    ("P3425", ["ref", "ref:SIC"], "Natura 2000 site ID"),
+    ("P3562", ["seamark:light:reference"], "Admiralty number"),
+    (
+        "P4755",
+        ["ref", "ref:train", "ref:crs", "crs", "nat_ref"],
+        "UK railway station code",
+    ),
+    ("P4803", ["ref", "ref:train"], "Amtrak station code"),
+    ("P6082", ["nycdoitt:bin"], "NYC Building Identification Number"),
+    ("P5086", ["ref"], "FIPS 5-2 alpha code (US states)"),
+    ("P5087", ["ref:fips"], "FIPS 5-2 numeric code (US states)"),
+    ("P5208", ["ref:bag"], "BAG building ID for Dutch buildings"),
+]
+
+T = typing.TypeVar("T", bound="Item")

-osm_type_enum = postgresql.ENUM('node', 'way', 'relation',
-                                name='osm_type_enum',
-                                metadata=Base.metadata)

 class Item(Base):
+    """Wikidata item."""
+
    __tablename__ = "item"
    item_id = Column(Integer, primary_key=True, autoincrement=False)
    labels = Column(postgresql.JSONB)
    descriptions = Column(postgresql.JSONB)
    aliases = Column(postgresql.JSONB)
    sitelinks = Column(postgresql.JSONB)
-    claims = Column(postgresql.JSONB)
+    claims = Column(postgresql.JSONB, nullable=False)
    lastrevid = Column(Integer, nullable=False, unique=True)
-    locations = relationship("ItemLocation", cascade="all, delete-orphan", backref="item")
+    locations = relationship(
+        "ItemLocation", cascade="all, delete-orphan", backref="item"
+    )
    qid = column_property("Q" + cast(item_id, String))

+    wiki_extracts = relationship(
+        "Extract",
+        collection_class=attribute_mapped_collection("site"),
+        cascade="save-update, merge, delete, delete-orphan",
+        backref="item",
+    )
+    extracts = association_proxy("wiki_extracts", "extract")
+
    @classmethod
-    def get_by_qid(cls, qid):
+    def get_by_qid(cls: typing.Type[T], qid: str) -> T | None:
        if qid and len(qid) > 1 and qid[0].upper() == "Q" and qid[1:].isdigit():
-            return cls.query.get(qid[1:])
+            obj: T = cls.query.get(qid[1:])
+            return obj
+        return None

    @property
-    def wd_url(self):
+    def wd_url(self) -> str:
+        """Wikidata URL for item."""
        return f"https://www.wikidata.org/wiki/{self.qid}"

-    def get_claim(self, pid):
-        return [i["mainsnak"]["datavalue"]["value"] if "datavalue" in i["mainsnak"] else None
-                for i in self.claims.get(pid, [])]
+    def get_claim(self, pid: str) -> list[dict[str, Any] | None]:
+        """List of claims for given Wikidata property ID."""
+        claims = typing.cast(dict[str, list[dict[str, Any]]], self.claims)
+        return [
+            i["mainsnak"]["datavalue"]["value"]
+            if "datavalue" in i["mainsnak"]
+            else None
+            for i in claims.get(pid, [])
+        ]

-    def label(self, lang='en'):
-        if lang in self.labels:
-            return self.labels[lang]['value']
-        elif 'en' in self.labels:
-            return self.labels['en']['value']
+    def label(self, lang: str = "en") -> str:
+        """Label for this Wikidata item."""
+        labels = typing.cast(dict[str, dict[str, Any]], self.labels)
+        if lang in labels:
+            return typing.cast(str, labels[lang]["value"])
+        elif "en" in labels:
+            return typing.cast(str, labels["en"]["value"])

-        label_list = list(self.labels.values())
-        return label_list[0]['value'] if label_list else '[no label]'
+        label_list = list(labels.values())
+        return typing.cast(str, label_list[0]["value"]) if label_list else "[no label]"

-    def description(self, lang='en'):
-        if lang in self.descriptions:
-            return self.descriptions[lang]['value']
-        elif 'en' in self.descriptions:
-            return self.descriptions['en']['value']
-        return
+    def description(self, lang: str = "en") -> str | None:
+        """Return a description of the item."""
+        descriptions = typing.cast(dict[str, dict[str, Any]], self.descriptions)
+        if lang in descriptions:
+            return typing.cast(str, descriptions[lang]["value"])
+        elif "en" in descriptions:
+            return typing.cast(str, descriptions["en"]["value"])
+        return None

        d_list = list(self.descriptions.values())
        if d_list:
-            return d_list[0]['value']
+            return d_list[0]["value"]

-    def get_aliases(self, lang='en'):
+    def get_aliases(self, lang="en"):
        if lang not in self.aliases:
-            if 'en' not in self.aliases:
+            if "en" not in self.aliases:
                return []
-            lang = 'en'
-        return [a['value'] for a in self.aliases[lang]]
+            lang = "en"
+        return [a["value"] for a in self.aliases[lang]]

    def get_part_of_names(self):
        if not self.claims:
            return set()

        part_of_names = set()
-        for p361 in self.claims.get('P361', []):
+        for p361 in self.claims.get("P361", []):
            try:
-                part_of_id = p361['mainsnak']['datavalue']['value']['numeric-id']
+                part_of_id = p361["mainsnak"]["datavalue"]["value"]["numeric-id"]
            except KeyError:
                continue
            if part_of_id == self.item_id:
@ -98,7 +168,7 @@ class Item(Base):

    @property
    def entity(self):
-        keys = ['labels', 'aliases', 'descriptions', 'sitelinks', 'claims']
+        keys = ["labels", "aliases", "descriptions", "sitelinks", "claims"]
        return {key: getattr(self, key) for key in keys}

    def names(self, check_part_of=True):
@ -107,22 +177,24 @@ class Item(Base):
        d = wikidata.names_from_entity(self.entity) or defaultdict(list)

        for name, sources in list(d.items()):
-            if len(sources) == 1 and sources[0][0] == 'image':
+            if len(sources) == 1 and sources[0][0] == "image":
                continue
            for part_of_name in part_of_names:
                if not name.startswith(part_of_name):
                    continue
-                prefix_removed = name[len(part_of_name):].strip()
+                prefix_removed = name[len(part_of_name) :].strip()
                if prefix_removed not in d:
                    d[prefix_removed] = sources

        if self.claims:
-            for p6375 in self.claims.get('P6375', []):
+            for p6375 in self.claims.get("P6375", []):
                try:
-                    street_address = p6375['mainsnak']['datavalue']['value']
+                    street_address = p6375["mainsnak"]["datavalue"]["value"]
                except KeyError:
                    continue
-                d[street_address['text']].append(('P6375', street_address.get('language')))
+                d[street_address["text"]].append(
+                    ("P6375", street_address.get("language"))
+                )

        # A terrace of buildings can be illustrated with a photo of a single building.
        # We try to determine if this is the case and avoid using the filename of the
@ -131,8 +203,11 @@ class Item(Base):
        def has_digit(s):
            return any(c.isdigit() for c in s)

-        image_names = {name for name, sources in d.items()
-                       if len(sources) == 1 and sources[0][0] == 'image' and has_digit(name)}
+        image_names = {
+            name
+            for name, sources in d.items()
+            if len(sources) == 1 and sources[0][0] == "image" and has_digit(name)
+        }
        if not image_names:
            return dict(d) or None

@ -186,7 +261,6 @@ class Item(Base):
            "Q63565252",  # brook
            "Q12284",  # canal
            "Q55659167",  # natural watercourse
-
        }
        return bool(matching_types & set(isa_qids))

@ -195,19 +269,29 @@ class Item(Base):
        return self.is_street(isa_qids) or self.is_watercourse(isa_qids)

    def is_tram_stop(self):
-        return 'Q2175765' in self.get_isa_qids()
+        return "Q2175765" in self.get_isa_qids()

    def alert_admin_about_bad_time(self, v):
-        body = ("Wikidata item has an unsupported time precision\n\n"
-                + self.wd_url + "\n\n" + "Value:\n\n" + json.dumps(v, indent=2))
+        body = (
+            "Wikidata item has an unsupported time precision\n\n"
+            + self.wd_url
+            + "\n\n"
+            + "Value:\n\n"
+            + json.dumps(v, indent=2)
+        )
        mail.send_mail(f"OWL Map: bad time value in {self.qid}", body)

-    def closed(self):
+    def time_claim(self, pid):
        ret = []
-        for v in self.get_claim("P3999"):
+        for v in self.get_claim(pid):
            if not v:
                continue
+            try:
                t = utils.format_wikibase_time(v)
+            except Exception:
+                self.alert_admin_about_bad_time(v)
+                raise
+
            if t:
                ret.append(t)
            else:
@ -215,6 +299,84 @@ class Item(Base):

        return ret

+    def closed(self):
+        return self.time_claim("P3999")
+
+    def first_paragraph_language(self, lang):
+        if lang not in self.sitelinks():
+            return
+        extract = self.extracts.get(lang)
+        if not extract:
+            return
+
+        empty_list = [
+            "<p><span></span></p>",
+            "<p><span></span>\n</p>",
+            "<p><span></span>\n\n</p>",
+            "<p>\n<span></span>\n</p>",
+            "<p>\n\n<span></span>\n</p>",
+            "<p>.\n</p>",
+            "<p><br></p>",
+            '<p class="mw-empty-elt">\n</p>',
+            '<p class="mw-empty-elt">\n\n</p>',
+            '<p class="mw-empty-elt">\n\n\n</p>',
+        ]
+
+        text = extract.strip()
+        while True:
+            found_empty = False
+            for empty in empty_list:
+                if text.startswith(empty):
+                    text = text[len(empty) :].strip()
+                    found_empty = True
+            if not found_empty:
+                break
+
+        close_tag = "</p>"
+        first_end_p_tag = text.find(close_tag)
+        if first_end_p_tag == -1:
+            # FIXME: e-mail admin
+            return text
+
+        return text[: first_end_p_tag + len(close_tag)]
+
+    def get_identifiers_tags(self):
+        tags = defaultdict(list)
+        for claim, osm_keys, label in property_map:
+            values = [
+                i["mainsnak"]["datavalue"]["value"]
+                for i in self.claims.get(claim, [])
+                if "datavalue" in i["mainsnak"]
+            ]
+            if not values:
+                continue
+            if claim == "P782":
+                values += [
+                    m.group(1) for m in (re_lau_code.match(v) for v in values) if m
+                ]
+            for osm_key in osm_keys:
+                tags[osm_key].append((values, label))
+        return dict(tags)
+
+    def get_identifiers(self):
+        ret = {}
+        for claim, osm_keys, label in property_map:
+            values = [
+                i["mainsnak"]["datavalue"]["value"]
+                for i in self.claims.get(claim, [])
+                if "datavalue" in i["mainsnak"]
+            ]
+            if not values:
+                continue
+            if claim == "P782":
+                values += [
+                    m.group(1) for m in (re_lau_code.match(v) for v in values) if m
+                ]
+            for osm_key in osm_keys:
+                ret[label] = values
+        return ret
+
+
 # class Claim(Base):
 #     __tablename__ = "claim"
 #     item_id = Column(Integer, primary_key=True)
@ -222,13 +384,14 @@ class Item(Base):
 #     position = Column(Integer, primary_key=True)
 #     mainsnak = Column(postgresql.JSONB)

-class ItemIsA(Base):
-    __tablename__ = 'item_isa'
-    item_id = Column(Integer, ForeignKey('item.item_id'), primary_key=True)
-    isa_id = Column(Integer, ForeignKey('item.item_id'), primary_key=True)

-    item = relationship('Item', foreign_keys=[item_id])
-    isa = relationship('Item', foreign_keys=[isa_id])
+class ItemIsA(Base):
+    __tablename__ = "item_isa"
+    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
+    isa_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
+
+    item = relationship("Item", foreign_keys=[item_id])
+    isa = relationship("Item", foreign_keys=[isa_id])


 class ItemLocation(Base):
@ -241,18 +404,21 @@ class ItemLocation(Base):
    qid = column_property("Q" + cast(item_id, String))
    pid = column_property("P" + cast(item_id, String))

-    def get_lat_lon(self):
-        return session.query(func.ST_Y(self.location),
-                             func.ST_X(self.location)).one()
+    def get_lat_lon(self) -> tuple[float, float]:
+        """Get latitude and longitude of item."""
+        loc: tuple[float, float]
+        loc = session.query(func.ST_Y(self.location), func.ST_X(self.location)).one()
+        return loc
+

 def location_objects(coords):
    locations = []
    for pid, coord_list in coords.items():
        for num, coords in enumerate(coord_list):
            point = f"POINT({coords['longitude']} {coords['latitude']})"
-            loc = ItemLocation(property_id=int(pid[1:]),
-                               statement_order=num,
-                               location=point)
+            loc = ItemLocation(
+                property_id=int(pid[1:]), statement_order=num, location=point
+            )
            locations.append(loc)
    return locations

@ -282,8 +448,7 @@ class MapMixin:
    @declared_attr
    def geojson_str(cls):
        return column_property(
-            func.ST_AsGeoJSON(cls.way, maxdecimaldigits=6),
-            deferred=True
+            func.ST_AsGeoJSON(cls.way, maxdecimaldigits=6), deferred=True
        )

    @declared_attr
@ -292,17 +457,16 @@ class MapMixin:

    @hybrid_property
    def has_street_address(self):
-        return ("addr:housenumber" in self.tags
-                and "addr:street" in self.tags)
+        return "addr:housenumber" in self.tags and "addr:street" in self.tags

    def display_name(self):
-        for key in 'bridge:name', 'tunnel:name', 'lock_name':
+        for key in "bridge:name", "tunnel:name", "lock_name":
            if key in self.tags:
                return self.tags[key]

-        return (self.name
-                or self.tags.get("addr:housename")
-                or self.tags.get("inscription"))
+        return (
+            self.name or self.tags.get("addr:housename") or self.tags.get("inscription")
+        )

    def geojson(self):
        return json.loads(self.geojson_str)
@ -343,7 +507,7 @@ class Line(MapMixin, Base):

    @classmethod
    def get_osm(cls, osm_type, osm_id):
-        src_id = osm_id * {'way': 1, 'relation': -1}[osm_type]
+        src_id = osm_id * {"way": 1, "relation": -1}[osm_type]
        return cls.query.get(src_id)


@ -352,11 +516,12 @@ class Polygon(MapMixin, Base):

    @classmethod
    def get_osm(cls, osm_type, osm_id):
-        src_id = osm_id * {'way': 1, 'relation': -1}[osm_type]
+        src_id = osm_id * {"way": 1, "relation": -1}[osm_type]
        return cls.query.get(src_id)

    @property
-    def type(self):
+    def type(self) -> str:
+        """Polygon is either a way or a relation."""
        return "way" if self.src_id > 0 else "relation"

    @declared_attr
@ -364,12 +529,15 @@ class Polygon(MapMixin, Base):
        return column_property(func.ST_Area(cls.way, False), deferred=True)

    @hybrid_property
-    def area_in_sq_km(self):
+    def area_in_sq_km(self) -> float:
+        """Size of area in square km."""
        return self.area / (1000 * 1000)


 class User(Base, UserMixin):
-    __tablename__ = 'user'
+    """User."""
+
+    __tablename__ = "user"
    id = Column(Integer, primary_key=True)
    username = Column(String)
    password = Column(String)
@ -392,23 +560,27 @@ class User(Base, UserMixin):
    osm_oauth_token = Column(String)
    osm_oauth_token_secret = Column(String)

-    def is_active(self):
+    def is_active(self) -> bool:
+        """User is active."""
        return self.active

+
 class EditSession(Base):
-    __tablename__ = 'edit_session'
+    __tablename__ = "edit_session"
    id = Column(Integer, primary_key=True)
    user_id = Column(Integer, ForeignKey(User.id))
    created = Column(DateTime, default=now_utc(), nullable=False)
    edit_list = Column(postgresql.JSONB)
    comment = Column(String)

-    user = relationship('User')
-    changeset = relationship('Changeset', back_populates='edit_session', uselist=False)
+    user = relationship("User")
+    changeset = relationship("Changeset", back_populates="edit_session", uselist=False)


 class Changeset(Base):
-    __tablename__ = 'changeset'
+    """An OSM Changeset generated by this tool."""
+
+    __tablename__ = "changeset"
    id = Column(BigInteger, primary_key=True)
    created = Column(DateTime)
    comment = Column(String)
@ -416,38 +588,62 @@ class Changeset(Base):
    update_count = Column(Integer, nullable=False)
    edit_session_id = Column(Integer, ForeignKey(EditSession.id))

-    user = relationship('User',
-                        backref=backref('changesets',
-                                        lazy='dynamic',
-                                        order_by='Changeset.created.desc()'))
+    user = relationship(
+        "User",
+        backref=backref(
+            "changesets", lazy="dynamic", order_by="Changeset.created.desc()"
+        ),
+    )

-    edit_session = relationship('EditSession', back_populates='changeset')
+    edit_session = relationship("EditSession", back_populates="changeset")


 class ChangesetEdit(Base):
-    __tablename__ = 'changeset_edit'
+    """Record details of edits within a changeset."""

-    changeset_id = Column(BigInteger,
-                          ForeignKey('changeset.id'),
-                          primary_key=True)
+    __tablename__ = "changeset_edit"
+
+    changeset_id = Column(BigInteger, ForeignKey("changeset.id"), primary_key=True)
    item_id = Column(Integer, primary_key=True)
    osm_id = Column(BigInteger, primary_key=True)
    osm_type = Column(osm_type_enum, primary_key=True)
    saved = Column(DateTime, default=now_utc(), nullable=False)

-    changeset = relationship('Changeset',
-                             backref=backref('edits', lazy='dynamic'))
+    changeset = relationship("Changeset", backref=backref("edits", lazy="dynamic"))
+

 class SkipIsA(Base):
-    __tablename__ = 'skip_isa'
-    item_id = Column(Integer, ForeignKey('item.item_id'), primary_key=True)
+    """Ignore this item type when walking the Wikidata subclass graph."""
+
+    __tablename__ = "skip_isa"
+    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
+    qid = column_property("Q" + cast(item_id, String))
+
+    item = relationship("Item")

-    item = relationship('Item')

 class ItemExtraKeys(Base):
-    __tablename__ = 'item_extra_keys'
-    item_id = Column(Integer, ForeignKey('item.item_id'), primary_key=True)
+    """Extra tag or key to consider for an Wikidata item type."""
+
+    __tablename__ = "item_extra_keys"
+    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
    tag_or_key = Column(String, primary_key=True)
    note = Column(String)
+    qid = column_property("Q" + cast(item_id, String))

-    item = relationship('Item')
+    item = relationship("Item")
+
+
+class Extract(Base):
+    """First paragraph from Wikipedia."""
+
+    __tablename__ = "extract"
+
+    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
+    site = Column(String, primary_key=True)
+    extract = Column(String, nullable=False)
+
+    def __init__(self, site: str, extract: str):
+        """Initialise the object."""
+        self.site = site
+        self.extract = extract
--- a/406
+++ b/406
@ -0,0 +1,406 @@
+# vim: spell:tw=80 ft=markdown
+
+Extracted items from data dump that include "P625" with the quotes. There are 8,398,490 matching items.
+
+Nearest-Neighbour Searching
+
+https://postgis.net/workshops/postgis-intro/knn.html
+
+---
+Use recent changes API to update local Wikidata entity mirror.
+
+Need to handle new item, edit, delete, and undelete.
+
+For now we're just interested in items with coordinates, later we might care
+about languages, and classes.
+
+At some point we might keep track of redirects.
+
+Deletes
+-------
+Is the item in our database? If not then ignore it, if yes then delete it.
+
+New
+---
+Download full entity, check if it contains coordinates, if yes, then add to database, if not then ignore.
+
+Make a note of item ID and revid. Avoid downloading item again during update.
+
+Edits
+-----
+If the item is in our database and lastrevid is larger than the revid of the change then skip.
+
+Download full entity.
+
+If in our database and latest revision includes coordinates update item in
+database. If no coordinates then delete from our database.
+
+
+======
+Currently we have geographic objects represented by the Item class. We also want
+information about the type of object, languages and countries.
+
+How about a hierarchy with Item as the base class and GeoItem as a subclass for
+geographical objects. We can also have IsA, Language, and Country classes that
+derive from Item.
+
+Countries are a subclass of GeoItem.
+
+With the current design the Item table represents a cached copy of the latest
+version of the Wikidata item, no history is stored locally. This makes it had to
+keep track of changes over time.
+
+The same is true of the OSM data, we just keeping a copy of the most recent
+version.
+
+Instead we could store multiple revisions of Wikidata items. We want the latest
+version and any that has been considered part of a match with OSM.
+
+Which Wikidata revisions do we keep?
+
+1. latest revision
+2. revision used to generate match
+3. revision used in match checked by user
+
+Maybe a separate item revision table is too complex. We could just store JSON
+from a match in a table of OSM user uploads.
+
+===
+All countries have a P625 statement
+
+===
+cable-stayed bridge (Q158555)
+
+There are 786 bridges on OSM tagged with bridge:structure=cable-stayed. Some of
+these have a Wikidata tag but aren't tagged as a cable-stayed bridge in
+Wikidata. The Wikidata could be updated to tag them as a cable-stayed bridge.
+Something similar could be applied to other types.
+
+===
+Lots of items with coordinates don\'t have OSM tags/keys, either because they
+don\'t belong on the map or there isn\'t enough info in Wikidata.
+
+Need to search different properties for OSM tags, at least 'instance of',
+'use', 'sport' and 'religion'.
+
+Should start from items with an OSM tag first. Download all items with OSM tag,
+then walk subclass tree and download.
+
+===
+Test out a new design.
+
+===
+Make a status page that shows the size of each database table.
+===
+What should URLs look like, say I want to show lakes in lapland?
+
+https://osm.wikidata.link/matches?isa=Lake&location=Lapland
+
+===
+OSM & Wikidata pin map TODO list
+
+IsA list should support filtering
+
+===
+2021-06-17
+
+Candidate list should show street address. For example:
+
+https://alpha.osm.wikidata.link/map/17/40.01461/-105.28196?item=Q42384818
+---
+Preset could be more specific. For example mosque instead of place of worship.
+
+id-tagging-schema/data/presets/amenity/place_of_worship/muslim.json
+---
+candidates list should show object tags
+
+===
+2021-06-19
+
+* Rename from 'alpha' to 'v2'.
+* Use Flask-Babel to for i18n. Get translations from
+  https://www.microsoft.com/en-us/language/
+* Show total number of items
+* Show place name
+* Show aliases
+
+===
+2021-06-23
+
+Planning to update user IP location code. Should grab items within city or
+region. Need to handle IP that only resolves to a whole country. For example
+archive.org is 207.241.224.2, and just returns USA. The USA is too big to apply
+the matcher interface to.
+
+When trying to match the whole USA we should show the whole country and
+encourage the user to zoom in. Once the 
+
+---
+Map thoughts. Questions:
+
+What do we show to the user when the site loads?
+What happens when the user drags the map?
+What happens when the user changes zoom?
+How does searching change things?
+
+Starting scenarios:
+
+User enters IP that resolves to a big country, say USA. We show a map of the
+whole USA and ask them to zoom in. Once they've zoomed in we can show the total
+number of items and the item type facets.
+
+Find item type within Cambridge:
+
+``` SQL
+select jsonb_path_query(claims, '$.P31[*].mainsnak.datavalue.value.id') as isa, count(*) as num
+from item, item_location, planet_osm_polygon
+where item.item_id = item_location.item_id and osm_id=-295355 and ST_Covers(way, location) group by isa order by num;
+```
+
+Also need a to show a facet for items where item type is empty
+
+Find item type within California:
+``` SQL
+select jsonb_path_query(claims, '$.P31[*].mainsnak.datavalue.value.id') as isa, count(*) as num
+from item, item_location, planet_osm_polygon
+where item.item_id = item_location.item_id and osm_id=-165475 and ST_Intersects(way, location)
+group by isa order by num desc limit 20;
+```
+This query takes 26.5 seconds.
+
+England item count takes 1.5 seconds.
+
+``` SQL
+select count(distinct item_id)
+from item_location, planet_osm_polygon
+where osm_id=-58447 and ST_Covers(way, location);
+```
+
+===
+2021-06-25
+
+Library buildings (Q856584) in England. Query takes 3 seconds
+
+``` SQL
+select count(*)
+from item, item_location, planet_osm_polygon as loc
+where loc.osm_id=-58447
+    and jsonb_path_query_array(claims, '$.P31[*].mainsnak.datavalue.value.id') ? 'Q856584'
+    and item.item_id = item_location.item_id
+    and item_location.location && loc.way;
+```
+===
+2021-07-04
+
+TODO
+* Better error page than just 500 Internal Server Error.
+* Improve handling of Wikidata items without coordinates. Use different colour
+  for OSM Pin. Explain situation on item detail page. No need to look for matches.
+* DONE: Show spinner when looking for nearby OSM candidate matches.
+* DONE: Show message if no matches found.
+* Add 'building only match' switch
+* Two item pins on top of each other is a problem.
+
+2021-07-05
+
+Sometimes the selected OSM matches are incorrect. For example:
+
+https://v2.osm.wikidata.link/map/15/37.31390/-121.86338?item=Q83645632
+
+The item is linked to a node, a way and a relation. The node shows as a pin on
+the map, but isn't in the list of possible nearby matches. The way and relation
+both show in the list, but aren't selected.
+
+2021-07-07
+
+Logout link should come back to the same map location. Need to record the
+location somewhere. Could be in a cookie, constant updating of the logout
+URL, or have JavaScript that runs when the user follows the logout link.
+
+Search
+Should show a spinner so the user knows something is happening.
+Trigger search after first three characters have been entered.
+DONE: Style search hits so not so close to search box
+
+Highlight chosen search result.
+Close button to hide search results.
+DONE: Zoom to result bounds instead of zoom level 16.
+Should you be allowed to search while editing?
+
+DONE: Hide OSM candidate checkboxes if user not logged in.
+
+2021-07-10
+
+Exclude ways that are part of a boundary. Example:
+
+https://v2.osm.wikidata.link/map/18/42.37903/-71.11136?item=Q14715848
+
+2021-07-16
+
+Need better handling for OSM with wikidata tag but item has no coordinates. 
+
+Viewing a street shows too many yellow pins.
+https://v2.osm.wikidata.link/map/15/37.31221/-121.88869?item=Q89545422
+
+2021-07-17
+Could match on just name
+https://v2.osm.wikidata.link/map/18/50.21789/-5.28079?item=Q5288904
+
+2021-07-18
+Florida State Road 922 (Q2433226) is stored as multiple lines in the osm2pgsql
+database. Need to rebuild the database with the --multi-geometry so there is
+only one.
+
+2021-07-19
+After a save clicking on another item without closing edit panel causes
+problems. Need to trigger close_edit_list when opening item if upload_state is
+set to 'done'
+
+2021-07-22
+
+Example of a long road: Collins Avenue (Q652775)
+https://v2.osm.wikidata.link/map/19/25.86222/-80.12032?item=Q652775
+
+2021-08-04
+Use https://vue-select.org/ for item type filter.
+Show alert with spinner while count is running.
+Maybe we want to supply the item type filter as JSON and filter in the browser,
+no need to hit the server and database.
+Write documentation for the API.
+Speed up the item detail OSM nearby option.
+Use the sidebar to show list of items in the current view, so the user can
+go through the list and check them.
+OSM object polygon size is broken
+
+2021-08-05
+
+IsA search
+
+```sql
+SELECT 'Q' || item.item_id, item.labels->'en'->>'value' FROM item WHERE
+item.claims ? 'P1282' AND lower(jsonb_extract_path_text(item.labels, 'en',
+'value')) LIKE lower('%hotel%') AND length(jsonb_extract_path_text(item.labels,
+'en', 'value')) < 20;
+```
+
+2021-09-11
+
+Notes from Pixel 2
+
+Pin at the centroid of a polygon is to busy, especially with an item that links
+to multiple OSM objects. Object outline already on map, just need to connect
+outline to Wikidata markers. Could try and work out corners of rectangular
+buildings. Should link to ends nearest node for linear objects.
+
+Show warning when navigating away from map with edits.
+
+See WindowEventHandlers.onbeforeunload
+
+Option to clear edit list.
+
+---
+Ignore coordinates with a Google Maps reference. Example:
+
+https://www.wikidata.org/w/index.php?title=Q66228733&oldid=992964237
+
+---
+Check history for previous wikidata tags to warn mappers if a wikidata tag
+they're adding has previously been removed.
+
+Examples:
+    https://v2.osm.wikidata.link/map/17/52.18211/0.17756?item=Q6717455
+    and https://www.openstreetmap.org/way/143741201
+    https://www.openstreetmap.org/way/684624781
+
+---
+What happens when we moved the map?
+
+First we check the area visible on the map. If it is too large then there is
+nothing we can do, we give up and tell the user they need to zoom in.
+
+Otherwise we send the server a request for a count of the number of items in the
+current view. If the count is too high we abort and tell the user to zoom in.
+
+Once we know the area isn't too big and doesn't have too many items we want to
+make three requests to the server. First we make requests for the Wikidata items
+on the map another request for OSM objects with a Wikidata tag on the map. Both
+requests run at the same time. Once both requests complete we make another
+request to check for missing Wikidata items that were linked from OSM objects.
+
+---
+This is done
+
+https://v2.osm.wikidata.link/map/18/52.23270/0.21560?item=Q55099320
+should match: https://www.openstreetmap.org/node/2000849525
+
+Look for Tag:abandoned:railway=station
+
+---
+Need better handling for Wikidata redirects.
+
+Example: https://www.openstreetmap.org/way/130458959
+https://v2.osm.wikidata.link/map/18/51.36973/-2.81079?item=Q5117357
+
+---
+Consider 'OS grid reference' 
+https://www.wikidata.org/w/index.php?title=Q27082051&oldid=1336630735
+
+---
+Check for OpenStreetMap relation ID (P402) in Wikidata
+
+Display on details page. Highlight matching relation.
+
+example: https://www.wikidata.org/wiki/Q78078847
+
+---
+TODO
+
+* DONE: Add special code for matching watercourses that works like street matching
+* DONE: Frontend should catch API errors and show them
+* DONE: API calls should return errors in JSON
+
+* Run update code from systemd
+* Stop Wikidata update code from crashing when it hits an error
+* Add an option for 'select all' for linear features
+* Add a note to details page explaining street matching
+* Upload code to GitHub
+* Candidates list jumps when first object is selected, because message appears
+    at the top the list. Can be fixed by having a message there and replacing
+    it.
+
+IsA pages
+* Flesh out IsA pages
+* Allow users to add extra tags to IsA
+* Add option to update IsA
+
+Type filter
+* Include type filter QIDs in URL
+* Move type filter to modal box
+* Show item type description
+
+---
+Show note about relations for tram stops and windfarms
+
+---
+Show  dissolved, abolished or demolished date (P576) 
+https://map.osm.wikidata.link/map/18/40.74610/-73.99652?item=Q14707174
+
+---
+Get subclasses for one item type
+
+``` SQL
+select item_id, labels->'en'->'value' from item where jsonb_path_query_array(claims, '$."P279"[*]."mainsnak"."datavalue"."value"."id"'::jsonpath) ?| '{"Q718893"}';
+```
+
+Get subclasses for items with OSM tag/key
+
+``` SQL
+select item_id, labels->'en'->'value'
+	from item
+	where jsonb_path_query_array(claims, '$."P279"[*]."mainsnak"."datavalue"."value"."id"'::jsonpath)
+		?| array(select 'Q' || item_id from item where claims ? 'P1282');
+```
+
+---
+Shipyard results shouldn't include place=city
+https://map.osm.wikidata.link/map/18/50.89540/-1.38243?item=Q551401
--- a/snowpack.config.mjs
+++ b/snowpack.config.mjs
@ -4,7 +4,15 @@ export default {
    public: {url: '/', static: true},
    frontend: {url: '/dist'},
  },
-  plugins: ['@snowpack/plugin-vue', '@snowpack/plugin-dotenv'],
+  plugins: [
+    '@snowpack/plugin-vue',
+    '@snowpack/plugin-dotenv',
+    ['snowpack-plugin-cdn-import', {
+        dependencies: pkg.dependencies,
+        enableInDevMode: true,
+        baseUrl: 'https://unpkg.com',
+    }]
+  ],
  routes: [
    /* Enable an SPA Fallback in development: */
    // {"match": "routes", "src": ".*", "dest": "/index.html"},
--- a/templates/flash_msg.html
+++ b/templates/flash_msg.html
@ -0,0 +1,12 @@
+{% with messages = get_flashed_messages() %}
+  {% if messages %}
+    {% for message in messages %}
+      <div class="alert alert-success alert-dismissible" role="alert">
+        <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+          <span aria-hidden="true">&times;</span>
+        </button>
+        {{ message }}
+      </div>
+    {% endfor %}
+  {% endif %}
+{% endwith %}
--- a/templates/show_error.html
+++ b/templates/show_error.html
@ -0,0 +1,36 @@
+{% extends "base.html" %}
+
+{% block style %}
+<link rel="stylesheet" href="{{url_for('static', filename='css/exception.css')}}" />
+{% endblock %}
+
+{% block content %}
+
+<div class="container my-2">
+<div class="row">
+<div class="col">
+
+<h1>Software error: {{ tb.exception_type }}</h1>
+<div>
+  <pre>{{ tb.exception }}</pre>
+</div>
+
+{% set body %}
+URL: {{ request.url }}
+
+{{ tb.plaintext | safe }}
+{% endset %}
+
+<p><a class="btn btn-primary btn-lg" role="button" href="https://github.com/EdwardBetts/osm-wikidata/issues/new?title={{ tb.exception | urlencode }}&body={{ body | urlencode }}">Submit as an issue on GitHub</a> (requires an account with GitHub)</p>
+
+<h2 class="traceback">Traceback <em>(most recent call last)</em></h2>
+{{ tb.render_summary(include_title=False) | safe }}
+
+<p>Error in function "{{ tb.frames[-1].function_name }}": {{ last_frame_args | pprint }}</p>
+<pre>{{ last_frame.locals | pprint }}</pre>
+
+</div>
+</div>
+</div>
+
+{% endblock %}
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@ -12,3 +12,6 @@ def test_format_wikibase_time_century():

    v = {"time": "+1950-00-00T00:00:00Z", "precision": 7}
    assert utils.format_wikibase_time(v) == "20th century"
+
+    v = {"time": "+1868-01-09T00:00:00Z", "precision": 11}
+    assert utils.format_wikibase_time(v) == "9 January 1868"
--- a/update.py
+++ b/update.py
@ -0,0 +1,159 @@
+#!/usr/bin/python3
+
+"""Download Wikidata recent changes and update items in local database."""
+
+import json
+import typing
+from time import sleep
+
+from matcher import database, model, wikidata, wikidata_api
+
+DB_URL = "postgresql:///matcher"
+database.init_db(DB_URL)
+
+entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"}
+
+
+class Change(typing.TypedDict):
+    """Dict representing an edit in recent changes."""
+
+    title: str
+    timestamp: str
+    redirect: dict[str, typing.Any] | None
+    revid: int
+
+
+def handle_new(change: Change) -> None:
+    """Handle a new Wikidata item from the recent changes feed."""
+    qid = change["title"]
+    ts = change["timestamp"]
+    if change["redirect"]:
+        print(f"{ts}: new item {qid}, since replaced with redirect")
+        return
+    item = model.Item.query.get(qid[1:])  # check if item is already loaded
+    if item:
+        return handle_edit(change)
+
+    entity = wikidata_api.get_entity(qid)
+    if entity["id"] != qid:
+        print(f'redirect {qid} -> {entity["id"]}')
+        return
+
+    if "claims" not in entity:
+        print(qid)
+        print(entity)
+    coords = wikidata.get_entity_coords(entity["claims"])
+    if not coords:
+        print(f"{ts}: new item {qid} without coordinates")
+        return
+
+    print(f"{ts}: new item {qid} with coordinates")
+
+    item_id = int(qid[1:])
+    obj = {k: v for k, v in entity.items() if k in entity_keys}
+    try:
+        item = model.Item(item_id=item_id, **obj)
+    except TypeError:
+        print(qid)
+        print(f'{entity["pageid"]=} {entity["ns"]=} {entity["type"]=}')
+        print(entity.keys())
+        raise
+    item.locations = model.location_objects(coords)
+    database.session.add(item)
+
+
+def coords_equal(a: dict[str, typing.Any], b: dict[str, typing.Any]) -> bool:
+    """Deep equality comparison of nested dicts."""
+    return json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True)
+
+
+def handle_edit(change: Change) -> None:
+    """Process an edit from recent changes."""
+    qid = change["title"]
+    item = model.Item.query.get(qid[1:])
+    if not item:
+        return  # item isn't in our database so it probably has no coordinates
+
+    ts = change["timestamp"]
+
+    if item.lastrevid >= change["revid"]:
+        print(f"{ts}: no need to update {qid}")
+        return
+
+    entity = wikidata_api.get_entity(qid)
+    entity_qid = entity.pop("id")
+    if entity_qid != qid:
+        print(f"{ts}: item {qid} replaced with redirect")
+        database.session.delete(item)
+        database.session.commit()
+        return
+
+    assert entity_qid == qid
+    existing_coords = wikidata.get_entity_coords(item.claims)
+    if "claims" not in entity:
+        return
+    coords = wikidata.get_entity_coords(entity["claims"])
+
+    if not coords_equal(existing_coords, coords):
+        print(f"{ts}: update item {qid}, including coordinates")
+        item.locations = model.location_objects(coords)
+    else:
+        print(f"{ts}: update item {qid}, no change to coordinates")
+
+    for key in entity_keys:
+        setattr(item, key, entity[key])
+
+
+def update_timestamp(timestamp: str) -> None:
+    """Save timestamp to rc_timestamp."""
+    out = open("rc_timestamp", "w")
+    print(timestamp, file=out)
+    out.close()
+
+
+def update_database() -> None:
+    """Check recent changes and apply updates to local mirror of Wikidata."""
+    with open("rc_timestamp") as f:
+        start = f.read().strip()
+
+    rccontinue = None
+    seen = set()
+    while True:
+        r = wikidata_api.get_recent_changes(rcstart=start, rccontinue=rccontinue)
+
+        reply = r.json()
+        for change in reply["query"]["recentchanges"]:
+            rctype = change["type"]
+            timestamp = change["timestamp"]
+            qid = change["title"]
+            if qid in seen:
+                continue
+
+            if rctype == "new":
+                handle_new(change)
+                seen.add(qid)
+            if rctype == "edit":
+                handle_edit(change)
+                seen.add(qid)
+
+        update_timestamp(timestamp)
+        print("commit")
+        database.session.commit()
+
+        if "continue" not in reply:
+            break
+
+        rccontinue = reply["continue"]["rccontinue"]
+    database.session.commit()
+    print("finished")
+
+
+def main() -> None:
+    """Infinite loop."""
+    while True:
+        update_database()
+        sleep(60)
+
+
+if __name__ == "__main__":
+    main()
--- a/web_view.py
+++ b/web_view.py
@ -1,37 +1,61 @@
 #!/usr/bin/python3

-from flask import (Flask, render_template, request, jsonify, redirect, url_for, g,
-                   flash, session, Response, stream_with_context, abort, send_file)
+import json
+import re
+from time import sleep, time
+
+import flask_login
+import GeoIP
+import maxminddb
+import requests
+import sqlalchemy
+from flask import (
+    Flask,
+    Response,
+    abort,
+    flash,
+    g,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    session,
+    stream_with_context,
+    url_for,
+)
+from lxml import etree
+from requests_oauthlib import OAuth1Session
 from sqlalchemy import func
 from sqlalchemy.sql.expression import update
-from matcher import (nominatim, model, database, commons, wikidata, wikidata_api,
-                     osm_oauth, edit, mail, api, error_mail)
-# from werkzeug.debug.tbtools import get_current_traceback
+
+from matcher import (
+    api,
+    commons,
+    database,
+    edit,
+    error_mail,
+    mail,
+    model,
+    nominatim,
+    osm_oauth,
+    wikidata,
+    wikidata_api,
+)
 from matcher.data import property_map
-from time import time, sleep
-from requests_oauthlib import OAuth1Session
-from lxml import etree
-import werkzeug.exceptions
-import inspect
-import flask_login
-import requests
-import json
-import GeoIP
-import re
-import maxminddb
-import sqlalchemy
+
+# from werkzeug.debug.tbtools import get_current_traceback

 srid = 4326
-re_point = re.compile(r'^POINT\((.+) (.+)\)$')
+re_point = re.compile(r"^POINT\((.+) (.+)\)$")

 app = Flask(__name__)
 app.debug = True
-app.config.from_object('config.default')
+app.config.from_object("config.default")
 error_mail.setup_error_mail(app)

 login_manager = flask_login.LoginManager(app)
-login_manager.login_view = 'login_route'
-osm_api_base = 'https://api.openstreetmap.org/api/0.6'
+login_manager.login_view = "login_route"
+osm_api_base = "https://api.openstreetmap.org/api/0.6"

 maxminddb_reader = maxminddb.open_database(app.config["GEOLITE2"])

@ -39,7 +63,7 @@ DB_URL = "postgresql:///matcher"
 database.init_db(DB_URL)
 entity_keys = {"labels", "sitelinks", "aliases", "claims", "descriptions", "lastrevid"}

-re_qid = re.compile(r'^Q\d+$')
+re_qid = re.compile(r"^Q\d+$")


@app.teardown_appcontext
@ -51,6 +75,7 @@ def shutdown_session(exception=None):
 def global_user():
    g.user = flask_login.current_user._get_current_object()

+
 def dict_repr_values(d):
    return {key: repr(value) for key, value in d.items()}

@ -77,11 +102,13 @@ def dict_repr_values(d):
 #                            last_frame=last_frame,
 #                            last_frame_args=last_frame_args), 500

+
 def cors_jsonify(*args, **kwargs):
    response = jsonify(*args, **kwargs)
    response.headers["Access-Control-Allow-Origin"] = "*"
    return response

+
 def check_for_tagged_qids(qids):
    tagged = set()
    for qid in qids:
@ -108,12 +135,12 @@ def check_for_tagged_qid(qid):
 def geoip_user_record():
    gi = GeoIP.open(app.config["GEOIP_DATA"], GeoIP.GEOIP_STANDARD)

-    remote_ip = request.get('ip', request.remote_addr)
+    remote_ip = request.get("ip", request.remote_addr)
    return gi.record_by_addr(remote_ip)


 def get_user_location():
-    remote_ip = request.args.get('ip', request.remote_addr)
+    remote_ip = request.args.get("ip", request.remote_addr)
    maxmind = maxminddb_reader.get(remote_ip)
    return maxmind.get("location") if maxmind else None

@ -154,13 +181,15 @@ def isa_page(item_id):
    subclass_list = []
    for s in item.get_claim(subclass_property):
        subclass = api.get_item(s["numeric-id"])
-        subclass_list.append({
+        subclass_list.append(
+            {
                "qid": s["id"],
                "item_id": s["numeric-id"],
                "label": subclass.label(),
                "description": subclass.description(),
                "isa_page_url": url_for("isa_page", item_id=s["numeric-id"]),
-        })
+            }
+        )

    tags = api.get_tags_for_isa_item(item)

@ -254,14 +283,16 @@ def map_start_page():
        lat, lon = 42.2917, -85.5872
        radius = 5

-    return redirect(url_for(
-        'map_location',
-        lat=f'{lat:.5f}',
-        lon=f'{lon:.5f}',
+    return redirect(
+        url_for(
+            "map_location",
+            lat=f"{lat:.5f}",
+            lon=f"{lon:.5f}",
            zoom=16,
            radius=radius,
-        ip=request.args.get('ip'),
-    ))
+            ip=request.args.get("ip"),
+        )
+    )


@app.route("/documentation")
@ -270,16 +301,14 @@ def documentation_page():
    username = user.username if user.is_authenticated else None

    return render_template(
-        "documentation.html",
-        active_tab="documentation",
-        username=username
+        "documentation.html", active_tab="documentation", username=username
    )


@app.route("/search")
 def search_page():
    loc = get_user_location()
-    q = request.args.get('q')
+    q = request.args.get("q")

    user = flask_login.current_user
    username = user.username if user.is_authenticated else None
@ -296,6 +325,7 @@ def search_page():
        q=q,
    )

+
@app.route("/map/<int:zoom>/<float(signed=True):lat>/<float(signed=True):lon>")
 def map_location(zoom, lat, lon):
    qid = request.args.get("item")
@ -359,6 +389,36 @@ def lookup_item(item_id):
    return redirect(url)


+@app.route("/item/Q<int:item_id>")
+def lookup_item(item_id):
+    item = api.get_item(item_id)
+    if not item:
+        # TODO: show nicer page for Wikidata item not found
+        return abort(404)
+
+    try:
+        lat, lon = item.locations[0].get_lat_lon()
+    except IndexError:
+        # TODO: show nicer page for Wikidata item without coordinates
+        return abort(404)
+
+    return render_template(
+        "map.html",
+        active_tab="map",
+        zoom=16,
+        lat=lat,
+        lon=lon,
+        username=get_username(),
+        mode="map",
+        q=None,
+        qid=item.qid,
+        item_type_filter=[],
+    )
+
+    url = url_for("map_location", zoom=16, lat=lat, lon=lon, item=item.qid)
+    return redirect(url)
+
+
@app.route("/search/map")
 def search_map_page():
    user_lat, user_lon = get_user_location() or (None, None)
@ -398,10 +458,12 @@ def old_search_page():
 def read_bounds_param():
    return [float(i) for i in request.args["bounds"].split(",")]

+
 def read_isa_filter_param():
-    isa_param = request.args.get('isa')
+    isa_param = request.args.get("isa")
    if isa_param:
-        return set(qid.strip() for qid in isa_param.upper().split(','))
+        return set(qid.strip() for qid in isa_param.upper().split(","))
+

@app.route("/api/1/location")
 def show_user_location():
@ -417,6 +479,7 @@ def api_wikidata_items_count():
    t1 = time() - t0
    return cors_jsonify(success=True, count=count, duration=t1)

+
@app.route("/api/1/isa_search")
 def api_isa_search():
    t0 = time()
@ -452,6 +515,7 @@ def api_wikidata_items():
    t1 = time() - t0
    return cors_jsonify(success=True, duration=t1, **ret)

+
@app.route("/api/1/place/<osm_type>/<int:osm_id>")
 def api_place_items(osm_type, osm_id):
    t0 = time()
@ -478,9 +542,7 @@ def api_get_item(item_id):
    detail = api.item_detail(item)
    t1 = time() - t0

-    return cors_jsonify(success=True,
-                        duration=t1,
-                        **detail)
+    return cors_jsonify(success=True, duration=t1, **detail)


@app.route("/api/1/item/Q<int:item_id>/tags")
@ -491,25 +553,23 @@ def api_get_item_tags(item_id):
    osm_list = sorted(tags.keys())
    t1 = time() - t0

-    return cors_jsonify(success=True,
-                        qid=item.qid,
-                        tag_or_key_list=osm_list,
-                        tag_src=tags,
-                        duration=t1)
+    return cors_jsonify(
+        success=True, qid=item.qid, tag_or_key_list=osm_list, tag_src=tags, duration=t1
+    )


 def expand_street_name(from_names):
    ret = set(from_names)
    for name in from_names:
-        if any(name.startswith(st) for st in ('St ', 'St. ')):
-            first_space = name.find(' ')
+        if any(name.startswith(st) for st in ("St ", "St. ")):
+            first_space = name.find(" ")
            ret.add("Saint" + name[first_space:])

-        if ', ' in name:
+        if ", " in name:
            for n in set(ret):
                comma = n.find(", ")
                ret.add(name[:comma])
-        elif '/' in name:
+        elif "/" in name:
            for n in set(ret):
                ret.extend(part.strip() for part in n.split("/"))

@ -522,14 +582,12 @@ def api_find_osm_candidates(item_id):
    t0 = time()
    item = model.Item.query.get(item_id)
    if not item:
-        return cors_jsonify(success=True,
-                            qid=f'Q{item_id}',
-                            error="item doesn't exist")
+        return cors_jsonify(success=True, qid=f"Q{item_id}", error="item doesn't exist")

    if not item.locations:
-        return cors_jsonify(success=True,
-                            qid=f'Q{item_id}',
-                            error="item has no coordinates")
+        return cors_jsonify(
+            success=True, qid=f"Q{item_id}", error="item has no coordinates"
+        )

    label = item.label()
    item_is_street = item.is_street()
@ -547,17 +605,15 @@ def api_find_osm_candidates(item_id):
        max_distance = 1_000
        limit = 40
        names = None
-    nearby = api.find_osm_candidates(item,
-                                     limit=limit,
-                                     max_distance=max_distance,
-                                     names=names)
+    nearby = api.find_osm_candidates(
+        item, limit=limit, max_distance=max_distance, names=names
+    )

    if (item_is_street or item_is_watercourse) and not nearby:
        # nearby = [osm for osm in nearby if street_name_match(label, osm)]

        # try again without name filter
-        nearby = api.find_osm_candidates(item, limit=100,
-                                         max_distance=1_000)
+        nearby = api.find_osm_candidates(item, limit=100, max_distance=1_000)

    t1 = time() - t0
    return cors_jsonify(
@ -565,7 +621,7 @@ def api_find_osm_candidates(item_id):
        qid=item.qid,
        nearby=nearby,
        duration=t1,
-        max_distance=max_distance
+        max_distance=max_distance,
    )


@ -574,10 +630,12 @@ def api_missing_wikidata_items():
    t0 = time()
    qids_arg = request.args.get("qids")
    if not qids_arg:
-        return cors_jsonify(success=False,
+        return cors_jsonify(
+            success=False,
            error="required parameter 'qids' is missing",
            items=[],
-                           isa_count=[])
+            isa_count=[],
+        )

    qids = []
    for qid in qids_arg.upper().split(","):
@ -593,10 +651,7 @@ def api_missing_wikidata_items():

    ret = api.missing_wikidata_items(qids, lat, lon)
    t1 = time() - t0
-    return cors_jsonify(
-        success=True,
-        duration=t1,
-        **ret)
+    return cors_jsonify(success=True, duration=t1, **ret)


@app.route("/api/1/search")
@ -615,20 +670,20 @@ def api_search():

    return cors_jsonify(success=True, hits=hits)

+
@app.route("/api/1/polygon/<osm_type>/<int:osm_id>")
 def api_polygon(osm_type, osm_id):
    obj = model.Polygon.get_osm(osm_type, osm_id)
-    return cors_jsonify(successful=True,
-                        osm_type=osm_type,
-                        osm_id=osm_id,
-                        geojson=obj.geojson())
+    return cors_jsonify(
+        successful=True, osm_type=osm_type, osm_id=osm_id, geojson=obj.geojson()
+    )


@app.route("/refresh/Q<int:item_id>")
 def refresh_item(item_id):
    assert not model.Item.query.get(item_id)

-    qid = f'Q{item_id}'
+    qid = f"Q{item_id}"
    entity = wikidata_api.get_entity(qid)
    entity_qid = entity.pop("id")
    assert qid == entity_qid
@ -643,100 +698,110 @@ def refresh_item(item_id):
    database.session.add(item)
    database.session.commit()

-    return 'done'
+    return "done"

-@app.route('/login')
+
+@app.route("/login")
 def login_openstreetmap():
-    return redirect(url_for('start_oauth',
-                            next=request.args.get('next')))
+    return redirect(url_for("start_oauth", next=request.args.get("next")))

-@app.route('/logout')
+
+@app.route("/logout")
 def logout():
-    next_url = request.args.get('next') or url_for('map_start_page')
+    next_url = request.args.get("next") or url_for("map_start_page")
    flask_login.logout_user()
-    flash('you are logged out')
+    flash("you are logged out")
    return redirect(next_url)

-@app.route('/done/')
+
+@app.route("/done/")
 def done():
-    flash('login successful')
-    return redirect(url_for('map_start_page'))
+    flash("login successful")
+    return redirect(url_for("map_start_page"))

-@app.route('/oauth/start')
+
+@app.route("/oauth/start")
 def start_oauth():
-    next_page = request.args.get('next')
+    next_page = request.args.get("next")
    if next_page:
-        session['next'] = next_page
+        session["next"] = next_page

-    client_key = app.config['CLIENT_KEY']
-    client_secret = app.config['CLIENT_SECRET']
+    client_key = app.config["CLIENT_KEY"]
+    client_secret = app.config["CLIENT_SECRET"]

-    request_token_url = 'https://www.openstreetmap.org/oauth/request_token'
+    request_token_url = "https://www.openstreetmap.org/oauth/request_token"

-    callback = url_for('oauth_callback', _external=True)
+    callback = url_for("oauth_callback", _external=True)

-    oauth = OAuth1Session(client_key,
-                          client_secret=client_secret,
-                          callback_uri=callback)
+    oauth = OAuth1Session(
+        client_key, client_secret=client_secret, callback_uri=callback
+    )
    fetch_response = oauth.fetch_request_token(request_token_url)

-    session['owner_key'] = fetch_response.get('oauth_token')
-    session['owner_secret'] = fetch_response.get('oauth_token_secret')
+    session["owner_key"] = fetch_response.get("oauth_token")
+    session["owner_secret"] = fetch_response.get("oauth_token_secret")

-    base_authorization_url = 'https://www.openstreetmap.org/oauth/authorize'
-    authorization_url = oauth.authorization_url(base_authorization_url,
-                                                oauth_consumer_key=client_key)
+    base_authorization_url = "https://www.openstreetmap.org/oauth/authorize"
+    authorization_url = oauth.authorization_url(
+        base_authorization_url, oauth_consumer_key=client_key
+    )
    return redirect(authorization_url)

+
@login_manager.user_loader
 def load_user(user_id):
    return model.User.query.get(user_id)

+
@app.route("/oauth/callback", methods=["GET"])
 def oauth_callback():
-    client_key = app.config['CLIENT_KEY']
-    client_secret = app.config['CLIENT_SECRET']
+    client_key = app.config["CLIENT_KEY"]
+    client_secret = app.config["CLIENT_SECRET"]

-    oauth = OAuth1Session(client_key,
+    oauth = OAuth1Session(
+        client_key,
        client_secret=client_secret,
-                          resource_owner_key=session['owner_key'],
-                          resource_owner_secret=session['owner_secret'])
+        resource_owner_key=session["owner_key"],
+        resource_owner_secret=session["owner_secret"],
+    )

    oauth_response = oauth.parse_authorization_response(request.url)
-    verifier = oauth_response.get('oauth_verifier')
-    access_token_url = 'https://www.openstreetmap.org/oauth/access_token'
-    oauth = OAuth1Session(client_key,
+    verifier = oauth_response.get("oauth_verifier")
+    access_token_url = "https://www.openstreetmap.org/oauth/access_token"
+    oauth = OAuth1Session(
+        client_key,
        client_secret=client_secret,
-                          resource_owner_key=session['owner_key'],
-                          resource_owner_secret=session['owner_secret'],
-                          verifier=verifier)
+        resource_owner_key=session["owner_key"],
+        resource_owner_secret=session["owner_secret"],
+        verifier=verifier,
+    )

    oauth_tokens = oauth.fetch_access_token(access_token_url)
-    session['owner_key'] = oauth_tokens.get('oauth_token')
-    session['owner_secret'] = oauth_tokens.get('oauth_token_secret')
+    session["owner_key"] = oauth_tokens.get("oauth_token")
+    session["owner_secret"] = oauth_tokens.get("oauth_token_secret")

-    r = oauth.get(osm_api_base + '/user/details')
+    r = oauth.get(osm_api_base + "/user/details")
    info = osm_oauth.parse_userinfo_call(r.content)

-    user = model.User.query.filter_by(osm_id=info['id']).one_or_none()
+    user = model.User.query.filter_by(osm_id=info["id"]).one_or_none()

    if user:
-        user.osm_oauth_token = oauth_tokens.get('oauth_token')
-        user.osm_oauth_token_secret = oauth_tokens.get('oauth_token_secret')
+        user.osm_oauth_token = oauth_tokens.get("oauth_token")
+        user.osm_oauth_token_secret = oauth_tokens.get("oauth_token_secret")
    else:
        user = model.User(
-            username=info['username'],
-            description=info['description'],
-            img=info['img'],
-            osm_id=info['id'],
-            osm_account_created=info['account_created'],
+            username=info["username"],
+            description=info["description"],
+            img=info["img"],
+            osm_id=info["id"],
+            osm_account_created=info["account_created"],
            mock_upload=False,
        )
        database.session.add(user)
    database.session.commit()
    flask_login.login_user(user)

-    next_page = session.get('next') or url_for('map_start_page')
+    next_page = session.get("next") or url_for("map_start_page")
    return redirect(next_page)


@ -744,14 +809,13 @@ def validate_edit_list(edits):
    for e in edits:
        assert model.Item.get_by_qid(e["qid"])
        assert e["op"] in {"add", "remove", "change"}
-        osm_type, _, osm_id = e['osm'].partition('/')
+        osm_type, _, osm_id = e["osm"].partition("/")
        osm_id = int(osm_id)
-        if osm_type == 'node':
+        if osm_type == "node":
            assert model.Point.query.get(osm_id)
        else:
            src_id = osm_id if osm_type == "way" else -osm_id
-            assert (model.Line.query.get(src_id)
-                    or model.Polygon.query.get(src_id))
+            assert model.Line.query.get(src_id) or model.Polygon.query.get(src_id)


@app.route("/api/1/edit", methods=["POST"])
@ -760,9 +824,9 @@ def api_new_edit_session():
    incoming = request.json

    validate_edit_list(incoming["edit_list"])
-    es = model.EditSession(user=user,
-                           edit_list=incoming['edit_list'],
-                           comment=incoming['comment'])
+    es = model.EditSession(
+        user=user, edit_list=incoming["edit_list"], comment=incoming["comment"]
+    )
    database.session.add(es)
    database.session.commit()

@ -770,13 +834,14 @@ def api_new_edit_session():

    return cors_jsonify(success=True, session_id=session_id)

+
@app.route("/api/1/edit/<int:session_id>", methods=["POST"])
 def api_edit_session(session_id):
    es = model.EditSession.query.get(session_id)
    assert flask_login.current_user.id == es.user_id
    incoming = request.json

-    for f in 'edit_list', 'comment':
+    for f in "edit_list", "comment":
        if f not in incoming:
            continue
        setattr(es, f, incoming[f])
@ -784,21 +849,24 @@ def api_edit_session(session_id):

    return cors_jsonify(success=True, session_id=session_id)

+
 class VersionMismatch(Exception):
    pass

+
 def osm_object(osm_type, osm_id):
    if osm_type == "node":
        return model.Point.query.get(osm_id)

-    src_id = int(osm_id) * {'way': 1, 'relation': -1}[osm_type]
+    src_id = int(osm_id) * {"way": 1, "relation": -1}[osm_type]
    for cls in model.Line, model.Polygon:
        obj = cls.query.get(src_id)
        if obj:
            return obj

+
 def process_edit(changeset_id, e):
-    osm_type, _, osm_id = e['osm'].partition('/')
+    osm_type, _, osm_id = e["osm"].partition("/")
    qid = e["qid"]
    item_id = qid[1:]

@ -851,9 +919,7 @@ def process_edit(changeset_id, e):

    cls = type(osm)
    database.session.execute(
-            update(cls).
-            where(cls.src_id == osm.src_id).
-            values(tags=new_tags)
+        update(cls).where(cls.src_id == osm.src_id).values(tags=new_tags)
    )

    db_edit = model.ChangesetEdit(
@ -867,6 +933,7 @@ def process_edit(changeset_id, e):

    return "saved"

+
@app.route("/api/1/save/<int:session_id>")
 def api_save_changeset(session_id):
    assert g.user.is_authenticated
@ -938,7 +1005,8 @@ def api_real_save_changeset(session_id):
        edit.close_changeset(changeset_id)
        yield send("done")

-    return Response(stream_with_context(stream(g.user)), mimetype='text/event-stream')
+    return Response(stream_with_context(stream(g.user)), mimetype="text/event-stream")
+

 def api_mock_save_changeset(session_id):
    es = model.EditSession.query.get(session_id)
@ -948,7 +1016,7 @@ def api_mock_save_changeset(session_id):
        return f"data: {json.dumps(data)}\n\n"

    def stream(user):
-        print('stream')
+        print("stream")
        changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar()
        sleep(1)
        yield send("open", id=changeset_id)
@ -956,12 +1024,12 @@ def api_mock_save_changeset(session_id):

        update_count = 0

-        print('record_changeset', changeset_id)
+        print("record_changeset", changeset_id)
        edit.record_changeset(
            id=changeset_id, user=user, comment=es.comment, update_count=update_count
        )

-        print('edits')
+        print("edits")

        for num, e in enumerate(es.edit_list):
            print(num, e)
@ -970,12 +1038,12 @@ def api_mock_save_changeset(session_id):
            yield send("saved", edit=e, num=num)
            sleep(1)

-        print('closing')
+        print("closing")
        yield send("closing")
        sleep(1)
        yield send("done")

-    return Response(stream(g.user), mimetype='text/event-stream')
+    return Response(stream(g.user), mimetype="text/event-stream")


 if __name__ == "__main__":