Add types and docstrings + upgrade to SQLAlchmey 2
This commit is contained in:
		
							parent
							
								
									82671959bb
								
							
						
					
					
						commit
						5e8d1a99b0
					
				| 
						 | 
					@ -8,6 +8,8 @@ import flask
 | 
				
			||||||
import geoalchemy2
 | 
					import geoalchemy2
 | 
				
			||||||
import sqlalchemy
 | 
					import sqlalchemy
 | 
				
			||||||
from sqlalchemy import and_, or_
 | 
					from sqlalchemy import and_, or_
 | 
				
			||||||
 | 
					from sqlalchemy.dialects import postgresql
 | 
				
			||||||
 | 
					from sqlalchemy.orm import Mapped
 | 
				
			||||||
from sqlalchemy.sql import select
 | 
					from sqlalchemy.sql import select
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from matcher import database, model, wikidata, wikidata_api
 | 
					from matcher import database, model, wikidata, wikidata_api
 | 
				
			||||||
| 
						 | 
					@ -91,16 +93,22 @@ def make_envelope(bounds: list[float]) -> geoalchemy2.functions.ST_MakeEnvelope:
 | 
				
			||||||
    return sqlalchemy.func.ST_MakeEnvelope(*bounds, srid)
 | 
					    return sqlalchemy.func.ST_MakeEnvelope(*bounds, srid)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					def parse_point(point: str) -> tuple[str, str]:
 | 
				
			||||||
 | 
					    """Parse point from PostGIS."""
 | 
				
			||||||
 | 
					    m = re_point.match(point)
 | 
				
			||||||
 | 
					    assert m
 | 
				
			||||||
 | 
					    lon, lat = m.groups()
 | 
				
			||||||
 | 
					    assert lon and lat
 | 
				
			||||||
 | 
					    return (lon, lat)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_bbox_centroid(bbox: list[float]) -> tuple[str, str]:
 | 
					def get_bbox_centroid(bbox: list[float]) -> tuple[str, str]:
 | 
				
			||||||
    """Get centroid of bounding box."""
 | 
					    """Get centroid of bounding box."""
 | 
				
			||||||
    bbox = make_envelope(bbox)
 | 
					    bbox = make_envelope(bbox)
 | 
				
			||||||
    centroid = database.session.query(
 | 
					    centroid = database.session.query(
 | 
				
			||||||
        sqlalchemy.func.ST_AsText(sqlalchemy.func.ST_Centroid(bbox))
 | 
					        sqlalchemy.func.ST_AsText(sqlalchemy.func.ST_Centroid(bbox))
 | 
				
			||||||
    ).scalar()
 | 
					    ).scalar()
 | 
				
			||||||
    m = re_point.match(centroid)
 | 
					    lon, lat = parse_point(centroid)
 | 
				
			||||||
    assert m
 | 
					 | 
				
			||||||
    lon, lat = m.groups()
 | 
					 | 
				
			||||||
    assert lon and lat
 | 
					 | 
				
			||||||
    return (lat, lon)
 | 
					    return (lat, lon)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -117,26 +125,17 @@ def make_envelope_around_point(
 | 
				
			||||||
    s = select(
 | 
					    s = select(
 | 
				
			||||||
        [
 | 
					        [
 | 
				
			||||||
            sqlalchemy.func.ST_AsText(
 | 
					            sqlalchemy.func.ST_AsText(
 | 
				
			||||||
                sqlalchemy.func.ST_Project(p, distance, sqlalchemy.func.radians(0))
 | 
					                sqlalchemy.func.ST_Project(p, distance, sqlalchemy.func.radians(deg))
 | 
				
			||||||
            ),
 | 
					            )
 | 
				
			||||||
            sqlalchemy.func.ST_AsText(
 | 
					            for deg in (0, 90, 180, 270)
 | 
				
			||||||
                sqlalchemy.func.ST_Project(p, distance, sqlalchemy.func.radians(90))
 | 
					 | 
				
			||||||
            ),
 | 
					 | 
				
			||||||
            sqlalchemy.func.ST_AsText(
 | 
					 | 
				
			||||||
                sqlalchemy.func.ST_Project(p, distance, sqlalchemy.func.radians(180))
 | 
					 | 
				
			||||||
            ),
 | 
					 | 
				
			||||||
            sqlalchemy.func.ST_AsText(
 | 
					 | 
				
			||||||
                sqlalchemy.func.ST_Project(p, distance, sqlalchemy.func.radians(270))
 | 
					 | 
				
			||||||
            ),
 | 
					 | 
				
			||||||
        ]
 | 
					        ]
 | 
				
			||||||
    )
 | 
					    )
 | 
				
			||||||
    row = conn.execute(s).fetchone()
 | 
					    coords = [parse_point(i) for i in conn.execute(s).fetchone()]
 | 
				
			||||||
    coords = [[float(v) for v in re_point.match(i).groups()] for i in row]
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
    north = coords[0][1]
 | 
					    north = float(coords[0][1])
 | 
				
			||||||
    east = coords[1][0]
 | 
					    east = float(coords[1][0])
 | 
				
			||||||
    south = coords[2][1]
 | 
					    south = float(coords[2][1])
 | 
				
			||||||
    west = coords[3][0]
 | 
					    west = float(coords[3][0])
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    return sqlalchemy.func.ST_MakeEnvelope(west, south, east, north, srid)
 | 
					    return sqlalchemy.func.ST_MakeEnvelope(west, south, east, north, srid)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -148,10 +147,15 @@ def drop_way_area(tags: TagsType) -> TagsType:
 | 
				
			||||||
    return tags
 | 
					    return tags
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_part_of(table_name, src_id, bbox):
 | 
					def get_part_of(
 | 
				
			||||||
 | 
					    table_name: str, src_id: int, bbox: geoalchemy2.functions.ST_MakeEnvelope
 | 
				
			||||||
 | 
					) -> list[dict[str, typing.Any]]:
 | 
				
			||||||
 | 
					    """Get part of."""
 | 
				
			||||||
    table_map = {"point": point, "line": line, "polygon": polygon}
 | 
					    table_map = {"point": point, "line": line, "polygon": polygon}
 | 
				
			||||||
    table_alias = table_map[table_name].alias()
 | 
					    table_alias = table_map[table_name].alias()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    tags: Mapped[postgresql.HSTORE] = polygon.c.tags
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    s = (
 | 
					    s = (
 | 
				
			||||||
        select(
 | 
					        select(
 | 
				
			||||||
            [
 | 
					            [
 | 
				
			||||||
| 
						 | 
					@ -165,11 +169,8 @@ def get_part_of(table_name, src_id, bbox):
 | 
				
			||||||
                sqlalchemy.func.ST_Intersects(bbox, polygon.c.way),
 | 
					                sqlalchemy.func.ST_Intersects(bbox, polygon.c.way),
 | 
				
			||||||
                sqlalchemy.func.ST_Covers(polygon.c.way, table_alias.c.way),
 | 
					                sqlalchemy.func.ST_Covers(polygon.c.way, table_alias.c.way),
 | 
				
			||||||
                table_alias.c.osm_id == src_id,
 | 
					                table_alias.c.osm_id == src_id,
 | 
				
			||||||
                polygon.c.tags.has_key("name"),
 | 
					                tags.has_key("name"),
 | 
				
			||||||
                or_(
 | 
					                or_(tags.has_key("landuse"), tags.has_key("amenity")),
 | 
				
			||||||
                    polygon.c.tags.has_key("landuse"),
 | 
					 | 
				
			||||||
                    polygon.c.tags.has_key("amenity"),
 | 
					 | 
				
			||||||
                ),
 | 
					 | 
				
			||||||
            )
 | 
					            )
 | 
				
			||||||
        )
 | 
					        )
 | 
				
			||||||
        .group_by(polygon.c.osm_id, polygon.c.tags)
 | 
					        .group_by(polygon.c.osm_id, polygon.c.tags)
 | 
				
			||||||
| 
						 | 
					@ -228,6 +229,7 @@ def get_isa_count(items: list[model.Item]) -> list[tuple[str, int]]:
 | 
				
			||||||
            if not isa:
 | 
					            if not isa:
 | 
				
			||||||
                print("missing IsA:", item.qid)
 | 
					                print("missing IsA:", item.qid)
 | 
				
			||||||
                continue
 | 
					                continue
 | 
				
			||||||
 | 
					            assert isinstance(isa, dict) and isinstance(isa["id"], str)
 | 
				
			||||||
            isa_count[isa["id"]] += 1
 | 
					            isa_count[isa["id"]] += 1
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    return isa_count.most_common()
 | 
					    return isa_count.most_common()
 | 
				
			||||||
| 
						 | 
					@ -920,7 +922,7 @@ def find_osm_candidates(item, limit=80, max_distance=450, names=None):
 | 
				
			||||||
            "geojson": json.loads(geojson),
 | 
					            "geojson": json.loads(geojson),
 | 
				
			||||||
            "presets": get_presets_from_tags(shape, tags),
 | 
					            "presets": get_presets_from_tags(shape, tags),
 | 
				
			||||||
            "address_list": address_list,
 | 
					            "address_list": address_list,
 | 
				
			||||||
            "centroid": list(reversed(re_point.match(centroid).groups())),
 | 
					            "centroid": list(reversed(parse_point(centroid))),
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
        if area is not None:
 | 
					        if area is not None:
 | 
				
			||||||
            cur["area"] = area
 | 
					            cur["area"] = area
 | 
				
			||||||
| 
						 | 
					@ -980,23 +982,23 @@ def check_is_street_number_first(latlng):
 | 
				
			||||||
    flask.g.street_number_first = is_street_number_first(*latlng)
 | 
					    flask.g.street_number_first = is_street_number_first(*latlng)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class ItemDetailType(typing.TypedDict):
 | 
					class ItemDetailType(typing.TypedDict, total=False):
 | 
				
			||||||
    """Details of an item as a dict."""
 | 
					    """Details of an item as a dict."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    qid: str
 | 
					    qid: str
 | 
				
			||||||
    label: str
 | 
					    label: str
 | 
				
			||||||
    description: str
 | 
					    description: str | None
 | 
				
			||||||
    markers: list[dict[str, float]]
 | 
					    markers: list[dict[str, float]]
 | 
				
			||||||
    image_list: list[str]
 | 
					    image_list: list[str]
 | 
				
			||||||
    street_address: list[str]
 | 
					    street_address: list[str]
 | 
				
			||||||
    isa_list: list[dict[str, str]]
 | 
					    isa_list: list[dict[str, str]]
 | 
				
			||||||
    closed: bool
 | 
					    closed: list[str]
 | 
				
			||||||
    inception: str
 | 
					    inception: str
 | 
				
			||||||
    p1619: str
 | 
					    p1619: str
 | 
				
			||||||
    p576: str
 | 
					    p576: str
 | 
				
			||||||
    heritage_designation: str
 | 
					    heritage_designation: str
 | 
				
			||||||
    wikipedia: dict[str, str]
 | 
					    wikipedia: list[dict[str, str]]
 | 
				
			||||||
    identifiers: list[str]
 | 
					    identifiers: dict[str, list[str]]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def item_detail(item: model.Item) -> ItemDetailType:
 | 
					def item_detail(item: model.Item) -> ItemDetailType:
 | 
				
			||||||
| 
						 | 
					@ -1036,7 +1038,7 @@ def item_detail(item: model.Item) -> ItemDetailType:
 | 
				
			||||||
        if site.endswith("wiki") and len(site) < 8
 | 
					        if site.endswith("wiki") and len(site) < 8
 | 
				
			||||||
    ]
 | 
					    ]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    d = {
 | 
					    d: ItemDetailType = {
 | 
				
			||||||
        "qid": item.qid,
 | 
					        "qid": item.qid,
 | 
				
			||||||
        "label": item.label(),
 | 
					        "label": item.label(),
 | 
				
			||||||
        "description": item.description(),
 | 
					        "description": item.description(),
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -1,4 +1,6 @@
 | 
				
			||||||
"""Database functions."""
 | 
					"""Database."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					from datetime import datetime
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import flask
 | 
					import flask
 | 
				
			||||||
import sqlalchemy
 | 
					import sqlalchemy
 | 
				
			||||||
| 
						 | 
					@ -8,7 +10,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
 | 
				
			||||||
 | 
					
 | 
				
			||||||
session: sqlalchemy.orm.scoping.scoped_session = scoped_session(sessionmaker())
 | 
					session: sqlalchemy.orm.scoping.scoped_session = scoped_session(sessionmaker())
 | 
				
			||||||
 | 
					
 | 
				
			||||||
timeout = 20_000  # 20 seconds
 | 
					timeout = 2_000  # 20 seconds
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def init_db(db_url: str, echo: bool = False) -> None:
 | 
					def init_db(db_url: str, echo: bool = False) -> None:
 | 
				
			||||||
| 
						 | 
					@ -17,7 +19,7 @@ def init_db(db_url: str, echo: bool = False) -> None:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_engine(db_url: str, echo: bool = False) -> sqlalchemy.engine.base.Engine:
 | 
					def get_engine(db_url: str, echo: bool = False) -> sqlalchemy.engine.base.Engine:
 | 
				
			||||||
    """Create an engine objcet."""
 | 
					    """Create an engine object."""
 | 
				
			||||||
    return create_engine(
 | 
					    return create_engine(
 | 
				
			||||||
        db_url,
 | 
					        db_url,
 | 
				
			||||||
        pool_recycle=3600,
 | 
					        pool_recycle=3600,
 | 
				
			||||||
| 
						 | 
					@ -40,9 +42,10 @@ def init_app(app: flask.app.Flask, echo: bool = False) -> None:
 | 
				
			||||||
    session.configure(bind=get_engine(db_url, echo=echo))
 | 
					    session.configure(bind=get_engine(db_url, echo=echo))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @app.teardown_appcontext
 | 
					    @app.teardown_appcontext
 | 
				
			||||||
    def shutdown_session(exception: Exception | None = None) -> None:
 | 
					    def shutdown_session(exception: BaseException | None = None) -> None:
 | 
				
			||||||
        session.remove()
 | 
					        session.remove()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def now_utc():
 | 
					def now_utc() -> sqlalchemy.sql.functions.Function[datetime]:
 | 
				
			||||||
 | 
					    """Now with UTC timezone."""
 | 
				
			||||||
    return func.timezone("utc", func.now())
 | 
					    return func.timezone("utc", func.now())
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -1,14 +1,17 @@
 | 
				
			||||||
from flask import g
 | 
					 | 
				
			||||||
from . import user_agent_headers, database, osm_oauth, mail
 | 
					 | 
				
			||||||
from .model import Changeset
 | 
					 | 
				
			||||||
import requests
 | 
					 | 
				
			||||||
import html
 | 
					import html
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import requests
 | 
				
			||||||
 | 
					from flask import g
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					from . import database, mail, osm_oauth, user_agent_headers
 | 
				
			||||||
 | 
					from .model import Changeset
 | 
				
			||||||
 | 
					
 | 
				
			||||||
really_save = True
 | 
					really_save = True
 | 
				
			||||||
osm_api_base = "https://api.openstreetmap.org/api/0.6"
 | 
					osm_api_base = "https://api.openstreetmap.org/api/0.6"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def new_changeset(comment: str) -> str:
 | 
					def new_changeset(comment: str) -> str:
 | 
				
			||||||
 | 
					    """XML for a new changeset."""
 | 
				
			||||||
    return f"""
 | 
					    return f"""
 | 
				
			||||||
<osm>
 | 
					<osm>
 | 
				
			||||||
  <changeset>
 | 
					  <changeset>
 | 
				
			||||||
| 
						 | 
					@ -18,11 +21,12 @@ def new_changeset(comment: str) -> str:
 | 
				
			||||||
</osm>"""
 | 
					</osm>"""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def osm_request(path, **kwargs):
 | 
					def osm_request(path, **kwargs) -> requests.Response:
 | 
				
			||||||
    return osm_oauth.api_put_request(path, **kwargs)
 | 
					    return osm_oauth.api_put_request(path, **kwargs)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def create_changeset(changeset):
 | 
					def create_changeset(changeset: str) -> requests.Response:
 | 
				
			||||||
 | 
					    """Create new changeset."""
 | 
				
			||||||
    try:
 | 
					    try:
 | 
				
			||||||
        return osm_request("/changeset/create", data=changeset.encode("utf-8"))
 | 
					        return osm_request("/changeset/create", data=changeset.encode("utf-8"))
 | 
				
			||||||
    except requests.exceptions.HTTPError as r:
 | 
					    except requests.exceptions.HTTPError as r:
 | 
				
			||||||
| 
						 | 
					@ -31,11 +35,15 @@ def create_changeset(changeset):
 | 
				
			||||||
        raise
 | 
					        raise
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def close_changeset(changeset_id):
 | 
					def close_changeset(changeset_id: int) -> requests.Response:
 | 
				
			||||||
 | 
					    """Close changeset."""
 | 
				
			||||||
    return osm_request(f"/changeset/{changeset_id}/close")
 | 
					    return osm_request(f"/changeset/{changeset_id}/close")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def save_element(osm_type, osm_id, element_data):
 | 
					def save_element(
 | 
				
			||||||
 | 
					    osm_type: str, osm_id: int, element_data: str
 | 
				
			||||||
 | 
					) -> requests.Response | None:
 | 
				
			||||||
 | 
					    """Upload new version of object to OSM map."""
 | 
				
			||||||
    osm_path = f"/{osm_type}/{osm_id}"
 | 
					    osm_path = f"/{osm_type}/{osm_id}"
 | 
				
			||||||
    r = osm_request(osm_path, data=element_data)
 | 
					    r = osm_request(osm_path, data=element_data)
 | 
				
			||||||
    reply = r.text.strip()
 | 
					    reply = r.text.strip()
 | 
				
			||||||
| 
						 | 
					@ -56,9 +64,12 @@ error:
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    mail.send_mail(subject, body)
 | 
					    mail.send_mail(subject, body)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    return None
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def record_changeset(**kwargs):
 | 
					
 | 
				
			||||||
    change = Changeset(created=database.now_utc(), **kwargs)
 | 
					def record_changeset(**kwargs: str) -> Changeset:
 | 
				
			||||||
 | 
					    """Record changeset in the database."""
 | 
				
			||||||
 | 
					    change: Changeset = Changeset(created=database.now_utc(), **kwargs)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    database.session.add(change)
 | 
					    database.session.add(change)
 | 
				
			||||||
    database.session.commit()
 | 
					    database.session.commit()
 | 
				
			||||||
| 
						 | 
					@ -66,6 +77,7 @@ def record_changeset(**kwargs):
 | 
				
			||||||
    return change
 | 
					    return change
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_existing(osm_type, osm_id):
 | 
					def get_existing(osm_type: str, osm_id: int) -> requests.Response:
 | 
				
			||||||
 | 
					    """Get existing OSM object."""
 | 
				
			||||||
    url = f"{osm_api_base}/{osm_type}/{osm_id}"
 | 
					    url = f"{osm_api_base}/{osm_type}/{osm_id}"
 | 
				
			||||||
    return requests.get(url, headers=user_agent_headers())
 | 
					    return requests.get(url, headers=user_agent_headers())
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
							
								
								
									
										148
									
								
								matcher/model.py
									
									
									
									
									
								
							
							
						
						
									
										148
									
								
								matcher/model.py
									
									
									
									
									
								
							| 
						 | 
					@ -1,3 +1,4 @@
 | 
				
			||||||
 | 
					import abc
 | 
				
			||||||
import json
 | 
					import json
 | 
				
			||||||
import re
 | 
					import re
 | 
				
			||||||
import typing
 | 
					import typing
 | 
				
			||||||
| 
						 | 
					@ -12,13 +13,21 @@ from sqlalchemy.dialects import postgresql
 | 
				
			||||||
from sqlalchemy.ext.associationproxy import association_proxy
 | 
					from sqlalchemy.ext.associationproxy import association_proxy
 | 
				
			||||||
from sqlalchemy.ext.declarative import declared_attr
 | 
					from sqlalchemy.ext.declarative import declared_attr
 | 
				
			||||||
from sqlalchemy.ext.hybrid import hybrid_property
 | 
					from sqlalchemy.ext.hybrid import hybrid_property
 | 
				
			||||||
from sqlalchemy.orm import backref, column_property, deferred, registry, relationship
 | 
					from sqlalchemy.orm import (
 | 
				
			||||||
 | 
					    Mapped,
 | 
				
			||||||
 | 
					    QueryPropertyDescriptor,
 | 
				
			||||||
 | 
					    backref,
 | 
				
			||||||
 | 
					    column_property,
 | 
				
			||||||
 | 
					    deferred,
 | 
				
			||||||
 | 
					    registry,
 | 
				
			||||||
 | 
					    relationship,
 | 
				
			||||||
 | 
					)
 | 
				
			||||||
from sqlalchemy.orm.collections import attribute_mapped_collection
 | 
					from sqlalchemy.orm.collections import attribute_mapped_collection
 | 
				
			||||||
from sqlalchemy.orm.decl_api import DeclarativeMeta
 | 
					from sqlalchemy.orm.decl_api import DeclarativeMeta
 | 
				
			||||||
from sqlalchemy.schema import Column, ForeignKey
 | 
					from sqlalchemy.schema import Column, ForeignKey
 | 
				
			||||||
from sqlalchemy.types import BigInteger, Boolean, DateTime, Float, Integer, String, Text
 | 
					from sqlalchemy.types import BigInteger, Boolean, DateTime, Float, Integer, String, Text
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from . import mail, utils, wikidata
 | 
					from . import mail, utils, wikidata, wikidata_api
 | 
				
			||||||
from .database import now_utc, session
 | 
					from .database import now_utc, session
 | 
				
			||||||
 | 
					
 | 
				
			||||||
mapper_registry = registry()
 | 
					mapper_registry = registry()
 | 
				
			||||||
| 
						 | 
					@ -36,7 +45,7 @@ class Base(metaclass=DeclarativeMeta):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    registry = mapper_registry
 | 
					    registry = mapper_registry
 | 
				
			||||||
    metadata = mapper_registry.metadata
 | 
					    metadata = mapper_registry.metadata
 | 
				
			||||||
    query = session.query_property()
 | 
					    query: QueryPropertyDescriptor = session.query_property()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    __init__ = mapper_registry.constructor
 | 
					    __init__ = mapper_registry.constructor
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -98,12 +107,12 @@ class Item(Base):
 | 
				
			||||||
    sitelinks = Column(postgresql.JSONB)
 | 
					    sitelinks = Column(postgresql.JSONB)
 | 
				
			||||||
    claims = Column(postgresql.JSONB, nullable=False)
 | 
					    claims = Column(postgresql.JSONB, nullable=False)
 | 
				
			||||||
    lastrevid = Column(Integer, nullable=False, unique=True)
 | 
					    lastrevid = Column(Integer, nullable=False, unique=True)
 | 
				
			||||||
    locations = relationship(
 | 
					    locations: Mapped[list["ItemLocation"]] = relationship(
 | 
				
			||||||
        "ItemLocation", cascade="all, delete-orphan", backref="item"
 | 
					        "ItemLocation", cascade="all, delete-orphan", backref="item"
 | 
				
			||||||
    )
 | 
					    )
 | 
				
			||||||
    qid = column_property("Q" + cast_to_string(item_id))
 | 
					    qid: Mapped[str] = column_property("Q" + cast_to_string(item_id))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    wiki_extracts = relationship(
 | 
					    wiki_extracts: Mapped[list["Extract"]] = relationship(
 | 
				
			||||||
        "Extract",
 | 
					        "Extract",
 | 
				
			||||||
        collection_class=attribute_mapped_collection("site"),
 | 
					        collection_class=attribute_mapped_collection("site"),
 | 
				
			||||||
        cascade="save-update, merge, delete, delete-orphan",
 | 
					        cascade="save-update, merge, delete, delete-orphan",
 | 
				
			||||||
| 
						 | 
					@ -158,14 +167,15 @@ class Item(Base):
 | 
				
			||||||
        if d_list:
 | 
					        if d_list:
 | 
				
			||||||
            return d_list[0]["value"]
 | 
					            return d_list[0]["value"]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def get_aliases(self, lang="en"):
 | 
					    def get_aliases(self, lang: str = "en") -> list[str]:
 | 
				
			||||||
 | 
					        """Get aliases."""
 | 
				
			||||||
        if lang not in self.aliases:
 | 
					        if lang not in self.aliases:
 | 
				
			||||||
            if "en" not in self.aliases:
 | 
					            if "en" not in self.aliases:
 | 
				
			||||||
                return []
 | 
					                return []
 | 
				
			||||||
            lang = "en"
 | 
					            lang = "en"
 | 
				
			||||||
        return [a["value"] for a in self.aliases[lang]]
 | 
					        return [a["value"] for a in self.aliases[lang]]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def get_part_of_names(self):
 | 
					    def get_part_of_names(self) -> set[str]:
 | 
				
			||||||
        if not self.claims:
 | 
					        if not self.claims:
 | 
				
			||||||
            return set()
 | 
					            return set()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -186,11 +196,14 @@ class Item(Base):
 | 
				
			||||||
        return part_of_names
 | 
					        return part_of_names
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @property
 | 
					    @property
 | 
				
			||||||
    def entity(self):
 | 
					    def entity(self) -> wikidata_api.EntityType:
 | 
				
			||||||
 | 
					        """Entity."""
 | 
				
			||||||
        keys = ["labels", "aliases", "descriptions", "sitelinks", "claims"]
 | 
					        keys = ["labels", "aliases", "descriptions", "sitelinks", "claims"]
 | 
				
			||||||
        return {key: getattr(self, key) for key in keys}
 | 
					        return typing.cast(
 | 
				
			||||||
 | 
					            wikidata_api.EntityType, {key: getattr(self, key) for key in keys}
 | 
				
			||||||
 | 
					        )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def names(self, check_part_of=True):
 | 
					    def names(self, check_part_of: bool = True) -> dict[str, list[str]] | None:
 | 
				
			||||||
        part_of_names = self.get_part_of_names() if check_part_of else set()
 | 
					        part_of_names = self.get_part_of_names() if check_part_of else set()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        d = wikidata.names_from_entity(self.entity) or defaultdict(list)
 | 
					        d = wikidata.names_from_entity(self.entity) or defaultdict(list)
 | 
				
			||||||
| 
						 | 
					@ -258,7 +271,8 @@ class Item(Base):
 | 
				
			||||||
        """Get QIDs of items listed instance of (P31) property."""
 | 
					        """Get QIDs of items listed instance of (P31) property."""
 | 
				
			||||||
        return [typing.cast(str, isa["id"]) for isa in self.get_isa()]
 | 
					        return [typing.cast(str, isa["id"]) for isa in self.get_isa()]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def is_street(self, isa_qids=None):
 | 
					    def is_street(self, isa_qids: typing.Collection[str] | None = None) -> bool:
 | 
				
			||||||
 | 
					        """Item represents a street."""
 | 
				
			||||||
        if isa_qids is None:
 | 
					        if isa_qids is None:
 | 
				
			||||||
            isa_qids = self.get_isa_qids()
 | 
					            isa_qids = self.get_isa_qids()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -272,7 +286,8 @@ class Item(Base):
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
        return bool(matching_types & set(isa_qids))
 | 
					        return bool(matching_types & set(isa_qids))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def is_watercourse(self, isa_qids=None):
 | 
					    def is_watercourse(self, isa_qids: typing.Collection[str] | None = None) -> bool:
 | 
				
			||||||
 | 
					        """Item represents a watercourse."""
 | 
				
			||||||
        if isa_qids is None:
 | 
					        if isa_qids is None:
 | 
				
			||||||
            isa_qids = self.get_isa_qids()
 | 
					            isa_qids = self.get_isa_qids()
 | 
				
			||||||
        matching_types = {
 | 
					        matching_types = {
 | 
				
			||||||
| 
						 | 
					@ -368,7 +383,7 @@ class Item(Base):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        return text[: first_end_p_tag + len(close_tag)]
 | 
					        return text[: first_end_p_tag + len(close_tag)]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def get_identifiers_tags(self):
 | 
					    def get_identifiers_tags(self) -> dict[str, list[tuple[list[str], str]]]:
 | 
				
			||||||
        tags = defaultdict(list)
 | 
					        tags = defaultdict(list)
 | 
				
			||||||
        for claim, osm_keys, label in property_map:
 | 
					        for claim, osm_keys, label in property_map:
 | 
				
			||||||
            values = [
 | 
					            values = [
 | 
				
			||||||
| 
						 | 
					@ -386,7 +401,7 @@ class Item(Base):
 | 
				
			||||||
                tags[osm_key].append((values, label))
 | 
					                tags[osm_key].append((values, label))
 | 
				
			||||||
        return dict(tags)
 | 
					        return dict(tags)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def get_identifiers(self):
 | 
					    def get_identifiers(self) -> dict[str, list[str]]:
 | 
				
			||||||
        ret = {}
 | 
					        ret = {}
 | 
				
			||||||
        for claim, osm_keys, label in property_map:
 | 
					        for claim, osm_keys, label in property_map:
 | 
				
			||||||
            values = [
 | 
					            values = [
 | 
				
			||||||
| 
						 | 
					@ -420,8 +435,8 @@ class ItemIsA(Base):
 | 
				
			||||||
    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
 | 
					    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
 | 
				
			||||||
    isa_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
 | 
					    isa_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    item = relationship("Item", foreign_keys=[item_id])
 | 
					    item: Mapped[Item] = relationship("Item", foreign_keys=[item_id])
 | 
				
			||||||
    isa = relationship("Item", foreign_keys=[isa_id])
 | 
					    isa: Mapped[Item] = relationship("Item", foreign_keys=[isa_id])
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class ItemLocation(Base):
 | 
					class ItemLocation(Base):
 | 
				
			||||||
| 
						 | 
					@ -458,7 +473,9 @@ def location_objects(
 | 
				
			||||||
    return locations
 | 
					    return locations
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class MapMixin:
 | 
					class MapBase(Base):
 | 
				
			||||||
 | 
					    """Map base class."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @declared_attr
 | 
					    @declared_attr
 | 
				
			||||||
    def __tablename__(cls):
 | 
					    def __tablename__(cls):
 | 
				
			||||||
        return "planet_osm_" + cls.__name__.lower()
 | 
					        return "planet_osm_" + cls.__name__.lower()
 | 
				
			||||||
| 
						 | 
					@ -468,7 +485,7 @@ class MapMixin:
 | 
				
			||||||
    admin_level = Column(String)
 | 
					    admin_level = Column(String)
 | 
				
			||||||
    boundary = Column(String)
 | 
					    boundary = Column(String)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    tags = Column(postgresql.HSTORE)
 | 
					    tags: Mapped[postgresql.HSTORE]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @declared_attr
 | 
					    @declared_attr
 | 
				
			||||||
    def way(cls):
 | 
					    def way(cls):
 | 
				
			||||||
| 
						 | 
					@ -477,67 +494,92 @@ class MapMixin:
 | 
				
			||||||
        )
 | 
					        )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @declared_attr
 | 
					    @declared_attr
 | 
				
			||||||
    def kml(cls):
 | 
					    def kml(cls) -> sqlalchemy.orm.properties.ColumnProperty:
 | 
				
			||||||
 | 
					        """Get object in KML format."""
 | 
				
			||||||
        return column_property(func.ST_AsKML(cls.way), deferred=True)
 | 
					        return column_property(func.ST_AsKML(cls.way), deferred=True)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @declared_attr
 | 
					    @declared_attr
 | 
				
			||||||
    def geojson_str(cls):
 | 
					    def geojson_str(cls) -> sqlalchemy.orm.properties.ColumnProperty:
 | 
				
			||||||
 | 
					        """Get object as GeoJSON string."""
 | 
				
			||||||
        return column_property(
 | 
					        return column_property(
 | 
				
			||||||
            func.ST_AsGeoJSON(cls.way, maxdecimaldigits=6), deferred=True
 | 
					            func.ST_AsGeoJSON(cls.way, maxdecimaldigits=6), deferred=True
 | 
				
			||||||
        )
 | 
					        )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @declared_attr
 | 
					    @declared_attr
 | 
				
			||||||
    def as_EWKT(cls):
 | 
					    def as_EWKT(cls) -> sqlalchemy.orm.properties.ColumnProperty:
 | 
				
			||||||
 | 
					        """As EWKT."""
 | 
				
			||||||
        return column_property(func.ST_AsEWKT(cls.way), deferred=True)
 | 
					        return column_property(func.ST_AsEWKT(cls.way), deferred=True)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @hybrid_property
 | 
					    @hybrid_property
 | 
				
			||||||
    def has_street_address(self):
 | 
					    def has_street_address(self) -> bool:
 | 
				
			||||||
 | 
					        """Object has street address."""
 | 
				
			||||||
        return "addr:housenumber" in self.tags and "addr:street" in self.tags
 | 
					        return "addr:housenumber" in self.tags and "addr:street" in self.tags
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def display_name(self):
 | 
					    def display_name(self) -> str:
 | 
				
			||||||
 | 
					        """Name for display."""
 | 
				
			||||||
        for key in "bridge:name", "tunnel:name", "lock_name":
 | 
					        for key in "bridge:name", "tunnel:name", "lock_name":
 | 
				
			||||||
            if key in self.tags:
 | 
					            if key in self.tags:
 | 
				
			||||||
                return self.tags[key]
 | 
					                return typing.cast(str, self.tags[key])
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        return (
 | 
					        return typing.cast(
 | 
				
			||||||
            self.name or self.tags.get("addr:housename") or self.tags.get("inscription")
 | 
					            str,
 | 
				
			||||||
 | 
					            self.name
 | 
				
			||||||
 | 
					            or self.tags.get("addr:housename")
 | 
				
			||||||
 | 
					            or self.tags.get("inscription"),
 | 
				
			||||||
        )
 | 
					        )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def geojson(self):
 | 
					    def geojson(self) -> dict[str, typing.Any]:
 | 
				
			||||||
        return json.loads(self.geojson_str)
 | 
					        """Object GeoJSON parsed into Python data structure."""
 | 
				
			||||||
 | 
					        return typing.cast(dict[str, typing.Any], json.loads(self.geojson_str))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def get_centroid(self):
 | 
					    def get_centroid(self) -> tuple[float, float]:
 | 
				
			||||||
 | 
					        """Centroid."""
 | 
				
			||||||
        centroid = session.query(func.ST_AsText(func.ST_Centroid(self.way))).scalar()
 | 
					        centroid = session.query(func.ST_AsText(func.ST_Centroid(self.way))).scalar()
 | 
				
			||||||
        lon, lat = re_point.match(centroid).groups()
 | 
					        assert centroid
 | 
				
			||||||
 | 
					        assert (m := re_point.match(centroid))
 | 
				
			||||||
 | 
					        lon, lat = m.groups()
 | 
				
			||||||
        return (float(lat), float(lon))
 | 
					        return (float(lat), float(lon))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @classmethod
 | 
					    @classmethod
 | 
				
			||||||
    def coords_within(cls, lat, lon):
 | 
					    def coords_within(cls, lat: float, lon: float):
 | 
				
			||||||
        point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326)
 | 
					        point = func.ST_SetSRID(func.ST_MakePoint(lon, lat), 4326)
 | 
				
			||||||
        return cls.query.filter(
 | 
					        return cls.query.filter(
 | 
				
			||||||
            cls.admin_level.isnot(None), func.ST_Within(point, cls.way)
 | 
					            cls.admin_level.isnot(None), func.ST_Within(point, cls.way)
 | 
				
			||||||
        ).order_by(cls.area)
 | 
					        ).order_by(cls.area)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @property
 | 
					    @property
 | 
				
			||||||
    def id(self):
 | 
					    def id(self) -> int:
 | 
				
			||||||
 | 
					        """OSM id."""
 | 
				
			||||||
        return abs(self.src_id)  # relations have negative IDs
 | 
					        return abs(self.src_id)  # relations have negative IDs
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @property
 | 
					    @property
 | 
				
			||||||
    def identifier(self):
 | 
					    @abc.abstractmethod
 | 
				
			||||||
 | 
					    def type(self) -> str:
 | 
				
			||||||
 | 
					        """OSM type."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @property
 | 
				
			||||||
 | 
					    def identifier(self) -> str:
 | 
				
			||||||
 | 
					        """OSM identifier."""
 | 
				
			||||||
        return f"{self.type}/{self.id}"
 | 
					        return f"{self.type}/{self.id}"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @property
 | 
					    @property
 | 
				
			||||||
    def osm_url(self):
 | 
					    def osm_url(self):
 | 
				
			||||||
 | 
					        """OSM URL."""
 | 
				
			||||||
        return f"https://www.openstreetmap.org/{self.type}/{self.id}"
 | 
					        return f"https://www.openstreetmap.org/{self.type}/{self.id}"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class Point(MapMixin, Base):
 | 
					class Point(MapBase):
 | 
				
			||||||
 | 
					    """OSM planet point."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    type = "node"
 | 
					    type = "node"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class Line(MapMixin, Base):
 | 
					class Line(MapBase):
 | 
				
			||||||
 | 
					    """OSM planet line."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @property
 | 
					    @property
 | 
				
			||||||
    def type(self):
 | 
					    def type(self) -> str:
 | 
				
			||||||
 | 
					        """OSM type."""
 | 
				
			||||||
        return "way" if self.src_id > 0 else "relation"
 | 
					        return "way" if self.src_id > 0 else "relation"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @classmethod
 | 
					    @classmethod
 | 
				
			||||||
| 
						 | 
					@ -546,7 +588,9 @@ class Line(MapMixin, Base):
 | 
				
			||||||
        return cls.query.get(src_id)
 | 
					        return cls.query.get(src_id)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class Polygon(MapMixin, Base):
 | 
					class Polygon(MapBase):
 | 
				
			||||||
 | 
					    """OSM planet polygon."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    way_area = Column(Float)
 | 
					    way_area = Column(Float)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @classmethod
 | 
					    @classmethod
 | 
				
			||||||
| 
						 | 
					@ -560,13 +604,15 @@ class Polygon(MapMixin, Base):
 | 
				
			||||||
        return "way" if self.src_id > 0 else "relation"
 | 
					        return "way" if self.src_id > 0 else "relation"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @declared_attr
 | 
					    @declared_attr
 | 
				
			||||||
    def area(cls):
 | 
					    def area(cls) -> sqlalchemy.orm.properties.ColumnProperty:
 | 
				
			||||||
 | 
					        """Polygon area."""
 | 
				
			||||||
        return column_property(func.ST_Area(cls.way, False), deferred=True)
 | 
					        return column_property(func.ST_Area(cls.way, False), deferred=True)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @hybrid_property
 | 
					    @hybrid_property
 | 
				
			||||||
    def area_in_sq_km(self) -> float:
 | 
					    def area_in_sq_km(self) -> float:
 | 
				
			||||||
        """Size of area in square km."""
 | 
					        """Size of area in square km."""
 | 
				
			||||||
        return self.area / (1000 * 1000)
 | 
					        area: float = self.area
 | 
				
			||||||
 | 
					        return area / (1000 * 1000)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class User(Base, UserMixin):
 | 
					class User(Base, UserMixin):
 | 
				
			||||||
| 
						 | 
					@ -601,6 +647,8 @@ class User(Base, UserMixin):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class EditSession(Base):
 | 
					class EditSession(Base):
 | 
				
			||||||
 | 
					    """Edit session."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    __tablename__ = "edit_session"
 | 
					    __tablename__ = "edit_session"
 | 
				
			||||||
    id = Column(Integer, primary_key=True)
 | 
					    id = Column(Integer, primary_key=True)
 | 
				
			||||||
    user_id = Column(Integer, ForeignKey(User.id))
 | 
					    user_id = Column(Integer, ForeignKey(User.id))
 | 
				
			||||||
| 
						 | 
					@ -608,8 +656,10 @@ class EditSession(Base):
 | 
				
			||||||
    edit_list = Column(postgresql.JSONB)
 | 
					    edit_list = Column(postgresql.JSONB)
 | 
				
			||||||
    comment = Column(String)
 | 
					    comment = Column(String)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    user = relationship("User")
 | 
					    user: Mapped[User] = relationship("User")
 | 
				
			||||||
    changeset = relationship("Changeset", back_populates="edit_session", uselist=False)
 | 
					    changeset: Mapped["Changeset"] = relationship(
 | 
				
			||||||
 | 
					        "Changeset", back_populates="edit_session", uselist=False
 | 
				
			||||||
 | 
					    )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class Changeset(Base):
 | 
					class Changeset(Base):
 | 
				
			||||||
| 
						 | 
					@ -623,14 +673,16 @@ class Changeset(Base):
 | 
				
			||||||
    update_count = Column(Integer, nullable=False)
 | 
					    update_count = Column(Integer, nullable=False)
 | 
				
			||||||
    edit_session_id = Column(Integer, ForeignKey(EditSession.id))
 | 
					    edit_session_id = Column(Integer, ForeignKey(EditSession.id))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    user = relationship(
 | 
					    user: Mapped[User] = relationship(
 | 
				
			||||||
        "User",
 | 
					        "User",
 | 
				
			||||||
        backref=backref(
 | 
					        backref=backref(
 | 
				
			||||||
            "changesets", lazy="dynamic", order_by="Changeset.created.desc()"
 | 
					            "changesets", lazy="dynamic", order_by="Changeset.created.desc()"
 | 
				
			||||||
        ),
 | 
					        ),
 | 
				
			||||||
    )
 | 
					    )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    edit_session = relationship("EditSession", back_populates="changeset")
 | 
					    edit_session: Mapped[EditSession] = relationship(
 | 
				
			||||||
 | 
					        "EditSession", back_populates="changeset"
 | 
				
			||||||
 | 
					    )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class ChangesetEdit(Base):
 | 
					class ChangesetEdit(Base):
 | 
				
			||||||
| 
						 | 
					@ -644,7 +696,9 @@ class ChangesetEdit(Base):
 | 
				
			||||||
    osm_type = Column(osm_type_enum, primary_key=True)
 | 
					    osm_type = Column(osm_type_enum, primary_key=True)
 | 
				
			||||||
    saved = Column(DateTime, default=now_utc(), nullable=False)
 | 
					    saved = Column(DateTime, default=now_utc(), nullable=False)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    changeset = relationship("Changeset", backref=backref("edits", lazy="dynamic"))
 | 
					    changeset: Mapped[Changeset] = relationship(
 | 
				
			||||||
 | 
					        "Changeset", backref=backref("edits", lazy="dynamic")
 | 
				
			||||||
 | 
					    )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class SkipIsA(Base):
 | 
					class SkipIsA(Base):
 | 
				
			||||||
| 
						 | 
					@ -654,7 +708,7 @@ class SkipIsA(Base):
 | 
				
			||||||
    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
 | 
					    item_id = Column(Integer, ForeignKey("item.item_id"), primary_key=True)
 | 
				
			||||||
    qid = column_property("Q" + cast_to_string(item_id))
 | 
					    qid = column_property("Q" + cast_to_string(item_id))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    item = relationship("Item")
 | 
					    item: Mapped[Item] = relationship("Item")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class ItemExtraKeys(Base):
 | 
					class ItemExtraKeys(Base):
 | 
				
			||||||
| 
						 | 
					@ -666,7 +720,7 @@ class ItemExtraKeys(Base):
 | 
				
			||||||
    note = Column(String)
 | 
					    note = Column(String)
 | 
				
			||||||
    qid = column_property("Q" + cast_to_string(item_id))
 | 
					    qid = column_property("Q" + cast_to_string(item_id))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    item = relationship("Item")
 | 
					    item: Mapped[Item] = relationship("Item")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class Extract(Base):
 | 
					class Extract(Base):
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -1,17 +1,24 @@
 | 
				
			||||||
from collections import OrderedDict
 | 
					"""Nominatim."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import json
 | 
					import json
 | 
				
			||||||
 | 
					import typing
 | 
				
			||||||
 | 
					from collections import OrderedDict
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import requests
 | 
					import requests
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					from . import CallParams
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					Hit = dict[str, typing.Any]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class SearchError(Exception):
 | 
					class SearchError(Exception):
 | 
				
			||||||
    pass
 | 
					    """Search error."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def lookup_with_params(**kwargs):
 | 
					def lookup_with_params(**kwargs: str) -> list[Hit]:
 | 
				
			||||||
    url = "http://nominatim.openstreetmap.org/search"
 | 
					    url = "http://nominatim.openstreetmap.org/search"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    params = {
 | 
					    params: CallParams = {
 | 
				
			||||||
        "format": "jsonv2",
 | 
					        "format": "jsonv2",
 | 
				
			||||||
        "addressdetails": 1,
 | 
					        "addressdetails": 1,
 | 
				
			||||||
        "extratags": 1,
 | 
					        "extratags": 1,
 | 
				
			||||||
| 
						 | 
					@ -26,21 +33,24 @@ def lookup_with_params(**kwargs):
 | 
				
			||||||
        raise SearchError
 | 
					        raise SearchError
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    try:
 | 
					    try:
 | 
				
			||||||
        return json.loads(r.text, object_pairs_hook=OrderedDict)
 | 
					        reply: list[Hit] = json.loads(r.text, object_pairs_hook=OrderedDict)
 | 
				
			||||||
 | 
					        return reply
 | 
				
			||||||
    except json.decoder.JSONDecodeError:
 | 
					    except json.decoder.JSONDecodeError:
 | 
				
			||||||
        raise SearchError(r)
 | 
					        raise SearchError(r)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def lookup(q):
 | 
					def lookup(q: str) -> list[Hit]:
 | 
				
			||||||
 | 
					    """Nominatim search."""
 | 
				
			||||||
    return lookup_with_params(q=q)
 | 
					    return lookup_with_params(q=q)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_us_county(county, state):
 | 
					def get_us_county(county: str, state: str) -> Hit | None:
 | 
				
			||||||
 | 
					    """Search for US county and return resulting hit."""
 | 
				
			||||||
    if " " not in county and "county" not in county:
 | 
					    if " " not in county and "county" not in county:
 | 
				
			||||||
        county += " county"
 | 
					        county += " county"
 | 
				
			||||||
    results = lookup(q="{}, {}".format(county, state))
 | 
					    results = lookup(q="{}, {}".format(county, state))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def pred(hit):
 | 
					    def pred(hit: Hit) -> typing.TypeGuard[Hit]:
 | 
				
			||||||
        return (
 | 
					        return (
 | 
				
			||||||
            "osm_type" in hit
 | 
					            "osm_type" in hit
 | 
				
			||||||
            and hit["osm_type"] != "node"
 | 
					            and hit["osm_type"] != "node"
 | 
				
			||||||
| 
						 | 
					@ -50,7 +60,8 @@ def get_us_county(county, state):
 | 
				
			||||||
    return next(filter(pred, results), None)
 | 
					    return next(filter(pred, results), None)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_us_city(name, state):
 | 
					def get_us_city(name: str, state: str) -> Hit | None:
 | 
				
			||||||
 | 
					    """Search for US city and return resulting hit."""
 | 
				
			||||||
    results = lookup_with_params(city=name, state=state)
 | 
					    results = lookup_with_params(city=name, state=state)
 | 
				
			||||||
    if len(results) != 1:
 | 
					    if len(results) != 1:
 | 
				
			||||||
        results = [
 | 
					        results = [
 | 
				
			||||||
| 
						 | 
					@ -58,29 +69,32 @@ def get_us_city(name, state):
 | 
				
			||||||
        ]
 | 
					        ]
 | 
				
			||||||
        if len(results) != 1:
 | 
					        if len(results) != 1:
 | 
				
			||||||
            print("more than one")
 | 
					            print("more than one")
 | 
				
			||||||
            return
 | 
					            return None
 | 
				
			||||||
    hit = results[0]
 | 
					    hit = results[0]
 | 
				
			||||||
    if hit["type"] not in ("administrative", "city"):
 | 
					    if hit["type"] not in ("administrative", "city"):
 | 
				
			||||||
        print("not a city")
 | 
					        print("not a city")
 | 
				
			||||||
        return
 | 
					        return None
 | 
				
			||||||
    if hit["osm_type"] == "node":
 | 
					    if hit["osm_type"] == "node":
 | 
				
			||||||
        print("node")
 | 
					        print("node")
 | 
				
			||||||
        return
 | 
					        return None
 | 
				
			||||||
    if not hit["display_name"].startswith(name):
 | 
					    if not hit["display_name"].startswith(name):
 | 
				
			||||||
        print("wrong name")
 | 
					        print("wrong name")
 | 
				
			||||||
        return
 | 
					        return None
 | 
				
			||||||
    assert "osm_type" in hit and "osm_id" in hit and "geotext" in hit
 | 
					    assert "osm_type" in hit and "osm_id" in hit and "geotext" in hit
 | 
				
			||||||
    return hit
 | 
					    return hit
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_hit_name(hit):
 | 
					def get_hit_name(hit: Hit) -> str:
 | 
				
			||||||
 | 
					    """Get name from hit."""
 | 
				
			||||||
    address = hit.get("address")
 | 
					    address = hit.get("address")
 | 
				
			||||||
    if not address:
 | 
					    if not address:
 | 
				
			||||||
 | 
					        assert isinstance(hit["display_name"], str)
 | 
				
			||||||
        return hit["display_name"]
 | 
					        return hit["display_name"]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    address_values = list(address.values())
 | 
					    address_values = list(address.values())
 | 
				
			||||||
    n1 = address_values[0]
 | 
					    n1 = address_values[0]
 | 
				
			||||||
    if len(address) == 1:
 | 
					    if len(address) == 1:
 | 
				
			||||||
 | 
					        assert isinstance(n1, str)
 | 
				
			||||||
        return n1
 | 
					        return n1
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    country = address.pop("country", None)
 | 
					    country = address.pop("country", None)
 | 
				
			||||||
| 
						 | 
					@ -102,13 +116,15 @@ def get_hit_name(hit):
 | 
				
			||||||
        return f"{n1}, {n2}, {country}"
 | 
					        return f"{n1}, {n2}, {country}"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_hit_label(hit):
 | 
					def get_hit_label(hit: Hit) -> str:
 | 
				
			||||||
 | 
					    """Parse hit and generate label."""
 | 
				
			||||||
    tags = hit["extratags"]
 | 
					    tags = hit["extratags"]
 | 
				
			||||||
    designation = tags.get("designation")
 | 
					    designation = tags.get("designation")
 | 
				
			||||||
    category = hit["category"]
 | 
					    category = hit["category"]
 | 
				
			||||||
    hit_type = hit["type"]
 | 
					    hit_type = hit["type"]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    if designation:
 | 
					    if designation:
 | 
				
			||||||
 | 
					        assert isinstance(designation, str)
 | 
				
			||||||
        return designation.replace("_", " ")
 | 
					        return designation.replace("_", " ")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    if category == "boundary" and hit_type == "administrative":
 | 
					    if category == "boundary" and hit_type == "administrative":
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -1,14 +1,15 @@
 | 
				
			||||||
from flask import current_app, session
 | 
					"""OSM Authentication."""
 | 
				
			||||||
from requests_oauthlib import OAuth1Session
 | 
					
 | 
				
			||||||
from urllib.parse import urlencode
 | 
					import typing
 | 
				
			||||||
from datetime import datetime
 | 
					from datetime import datetime
 | 
				
			||||||
from flask import g
 | 
					from urllib.parse import urlencode
 | 
				
			||||||
 | 
					 | 
				
			||||||
from .model import User
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
from . import user_agent_headers
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
import lxml.etree
 | 
					import lxml.etree
 | 
				
			||||||
 | 
					from flask import current_app, g, session
 | 
				
			||||||
 | 
					from requests_oauthlib import OAuth1Session
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					from . import user_agent_headers
 | 
				
			||||||
 | 
					from .model import User
 | 
				
			||||||
 | 
					
 | 
				
			||||||
osm_api_base = "https://api.openstreetmap.org/api/0.6"
 | 
					osm_api_base = "https://api.openstreetmap.org/api/0.6"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					@ -67,11 +68,12 @@ def parse_userinfo_call(xml):
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def get_username():
 | 
					def get_username() -> str | None:
 | 
				
			||||||
 | 
					    """Get username of current user."""
 | 
				
			||||||
    if "user_id" not in session:
 | 
					    if "user_id" not in session:
 | 
				
			||||||
        return  # not authorized
 | 
					        return None  # not authorized
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    user_id = session["user_id"]
 | 
					    user_id = session["user_id"]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    user = User.query.get(user_id)
 | 
					    user = User.query.get(user_id)
 | 
				
			||||||
    return user.username
 | 
					    return typing.cast(str, user.username)
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -1,6 +1,8 @@
 | 
				
			||||||
 | 
					"""Wikidata API."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import json
 | 
					import json
 | 
				
			||||||
import typing
 | 
					import typing
 | 
				
			||||||
from typing import Any, cast
 | 
					from typing import cast
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import requests
 | 
					import requests
 | 
				
			||||||
import simplejson.errors
 | 
					import simplejson.errors
 | 
				
			||||||
| 
						 | 
					@ -9,7 +11,26 @@ from . import CallParams, user_agent_headers
 | 
				
			||||||
 | 
					
 | 
				
			||||||
wd_api_url = "https://www.wikidata.org/w/api.php"
 | 
					wd_api_url = "https://www.wikidata.org/w/api.php"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
EntityType = dict[str, Any]
 | 
					Claims = dict[str, list[dict[str, typing.Any]]]
 | 
				
			||||||
 | 
					Sitelinks = dict[str, dict[str, typing.Any]]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class EntityType(typing.TypedDict, total=False):
 | 
				
			||||||
 | 
					    """Wikidata Entity."""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    id: str
 | 
				
			||||||
 | 
					    ns: str
 | 
				
			||||||
 | 
					    type: str
 | 
				
			||||||
 | 
					    pageid: int
 | 
				
			||||||
 | 
					    title: str
 | 
				
			||||||
 | 
					    labels: dict[str, typing.Any]
 | 
				
			||||||
 | 
					    descriptions: dict[str, typing.Any]
 | 
				
			||||||
 | 
					    claims: Claims
 | 
				
			||||||
 | 
					    lastrevid: int
 | 
				
			||||||
 | 
					    sitelinks: Sitelinks
 | 
				
			||||||
 | 
					    modified: str
 | 
				
			||||||
 | 
					    redirects: dict[str, typing.Any]
 | 
				
			||||||
 | 
					    aliases: dict[str, list[dict[str, typing.Any]]]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def api_get(params: CallParams) -> requests.Response:
 | 
					def api_get(params: CallParams) -> requests.Response:
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
							
								
								
									
										13
									
								
								requirements.txt
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										13
									
								
								requirements.txt
									
									
									
									
									
										Normal file
									
								
							| 
						 | 
					@ -0,0 +1,13 @@
 | 
				
			||||||
 | 
					flask
 | 
				
			||||||
 | 
					-e git+https://github.com/maxcountryman/flask-login.git#egg=Flask-Login
 | 
				
			||||||
 | 
					GeoIP
 | 
				
			||||||
 | 
					lxml
 | 
				
			||||||
 | 
					maxminddb
 | 
				
			||||||
 | 
					requests
 | 
				
			||||||
 | 
					sqlalchemy
 | 
				
			||||||
 | 
					requests_oauthlib
 | 
				
			||||||
 | 
					geoalchemy2
 | 
				
			||||||
 | 
					simplejson
 | 
				
			||||||
 | 
					user_agents
 | 
				
			||||||
 | 
					num2words
 | 
				
			||||||
 | 
					psycopg2
 | 
				
			||||||
		Loading…
	
		Reference in a new issue