openstreetmap-tools/tests/test_pt_search.py

"""Offline tests for public transport relation search and its JSON endpoint."""

import importlib.util
import json
import sys
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs

import pytest
import requests
import responses

from osm_geojson.pt import core


def relation(relation_id: int = 1, **tags: str) -> dict[str, Any]:
    """Build a minimal tagged Overpass relation."""
    return {"type": "relation", "id": relation_id, "tags": {"type": "route", "route": "bus", **tags}}


@pytest.fixture()
def app(monkeypatch: Any) -> Any:
    """Load the web application independently of Python's import path."""
    spec = importlib.util.spec_from_file_location("pt_web_app", Path(__file__).parents[1] / "web/app.py")
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    monkeypatch.setitem(sys.modules, spec.name, module)
    spec.loader.exec_module(module)
    module.app.config["TESTING"] = True
    return module.app


@responses.activate
def test_search_metadata_and_query() -> None:
    """Return routes and masters, exclude other relations, and prefer exact refs."""
    responses.post(core.OVERPASS_URL, json={"elements": [
        relation(2, name="Airport shuttle", ref="X1", operator="City Bus", **{"from": "Airport", "to": "Centre"}),
        relation(3, name="M11", type="route_master", route_master="subway", ref="M11"),
        relation(1, name="Metro M11", route="subway", ref="M11"),
        relation(4, route="hiking"),
        {"type": "node", "id": 5},
    ]})
    result = core.search_routes(" M11 ")
    assert [item["id"] for item in result["routes"]] == [3, 1, 2]
    assert result["routes"][0]["type"] == "route_master"
    assert result["routes"][2]["operator"] == "City Bus"
    assert result["truncated"] is False
    request = responses.calls[0].request
    query = parse_qs(request.body)["data"][0]
    assert '["ref"~"^M11$",i]' in query
    assert '["route_master"~' in query
    assert 'out tags 51;' in query
    assert request.headers["User-Agent"] == "osm-pt-geojson/1.0"


@responses.activate
def test_literal_search_and_mode_filter() -> None:
    """User text stays a literal regex and cannot inject Overpass selectors."""
    responses.post(core.OVERPASS_URL, json={"elements": []})
    text = 'İstanbul [A.*] "\\);out;'
    assert core.search_routes(text, "subway")["routes"] == []
    query = parse_qs(responses.calls[0].request.body)["data"][0]
    escaped = 'İstanbul \\[A\\.\\*\\] "\\\\\\);out;'
    assert f'["name"~{json.dumps(escaped, ensure_ascii=False)},i]' in query
    assert '["route"~"^(subway)$"]' in query
    assert '["route_master"~"^(subway)$"]' in query


@pytest.mark.parametrize("query,mode", [("", ""), ("x", ""), ("x" * 101, ""), ("a\nb", ""), ("M11", "hiking")])
@responses.activate
def test_invalid_search(query: str, mode: str) -> None:
    """Reject invalid input without contacting Overpass."""
    with pytest.raises(core.OsmError) as error:
        core.search_routes(query, mode)
    assert error.value.status_code == 400
    assert len(responses.calls) == 0


@responses.activate
def test_search_limit() -> None:
    """Tell the UI when more than fifty relations matched."""
    responses.post(core.OVERPASS_URL, json={"elements": [relation(i) for i in range(51)]})
    result = core.search_routes("Airport")
    assert len(result["routes"]) == 50
    assert result["truncated"] is True


@pytest.mark.parametrize("payload", [{"remark": "runtime error: Query timed out", "elements": []}, {}, []])
@responses.activate
def test_incomplete_search(payload: Any) -> None:
    """A timed-out or malformed response must not look like an empty result."""
    responses.post(core.OVERPASS_URL, json=payload)
    with pytest.raises(core.OsmError) as error:
        core.search_routes("M11")
    assert error.value.status_code == 502


@pytest.mark.parametrize("failure", [requests.Timeout("timeout"), "invalid JSON", 429, 504])
@responses.activate
def test_upstream_search_failure(failure: Any) -> None:
    """Network, rate-limit and decoding failures become library errors."""
    if isinstance(failure, int):
        responses.post(core.OVERPASS_URL, status=failure)
    else:
        responses.post(core.OVERPASS_URL, body=failure)
    with pytest.raises(core.OsmError) as error:
        core.search_routes("M11")
    assert error.value.status_code == 502


@responses.activate
def test_search_api(app: Any) -> None:
    """The endpoint handles results, empty matches, invalid inputs and upstream errors."""
    responses.post(core.OVERPASS_URL, json={"elements": [relation(15083963, name="M11", ref="M11")]})
    client = app.test_client()
    result = client.get("/api/search?q=M11&mode=subway")
    assert result.status_code == 200
    assert result.json["routes"][0]["id"] == 15083963
    responses.post(core.OVERPASS_URL, json={"elements": []})
    assert client.get("/api/search?q=Unknown").json == {"routes": [], "truncated": False}
    for path in ("/api/search", "/api/search?q=M11&mode=hiking"):
        result = client.get(path)
        assert result.status_code == 400
        assert result.json["error"] == "invalid_query"
    responses.post(core.OVERPASS_URL, status=503)
    result = client.get("/api/search?q=M11")
    assert result.status_code == 502
    assert result.json["error"] == "osm_error"


@pytest.mark.parametrize("width", [1280, 390])
def test_search_browser(app: Any, width: int, monkeypatch: Any) -> None:
    """Search results load routes on desktop and mobile, with safe text and errors."""
    import threading
    from werkzeug.serving import make_server

    browser_api = pytest.importorskip("playwright.sync_api")
    monkeypatch.setitem(app.view_functions["route_page"].__globals__, "fetch_relation_name", lambda _: "Metro M11")
    server = make_server("127.0.0.1", 0, app)
    thread = threading.Thread(target=server.serve_forever, daemon=True)
    thread.start()
    try:
        with browser_api.sync_playwright() as playwright:
            browser = playwright.chromium.launch()
            page = browser.new_page(viewport={"width": width, "height": 844})
            errors: list[str] = []
            location_requests: list[str] = []
            page.on("pageerror", lambda error: errors.append(str(error)))
            page.on('request', lambda request: location_requests.append(request.url) if '/api/locations?' in request.url else None)
            page.route("**/tile.openstreetmap.org/**", lambda route: route.abort())
            page.route("**/api/locations?*", lambda route: route.fulfill(json={"locations": [
                {"label": "Bristol, UK", "bounds": [51.39, -2.73, 51.55, -2.51]},
                {"label": "Bristol, USA", "bounds": [41.6, -72.9, 41.8, -72.8]},
            ]}))
            page.route("**/api/search?*", lambda route: route.fulfill(json={
                "routes": [{"id": 15083963, "name": "Metro M11 <script>", "ref": "M11",
                            "route": "subway", "type": "route", "from": "Airport", "to": "Centre",
                            "operator": "Metro"}], "truncated": True,
            }))
            page.route("**/api/route/15083963", lambda route: route.fulfill(json={
                "name": "Metro M11", "ref": "M11", "other_directions": [],
                "stops": [{"name": "Airport", "lat": 41.2, "lon": 28.8}],
                "geojson": {"type": "FeatureCollection", "features": [{
                    "type": "Feature", "properties": {},
                    "geometry": {"type": "LineString", "coordinates": [[28.8, 41.2], [28.9, 41.3]]},
                }]},
            }))
            page.goto(f"http://127.0.0.1:{server.server_port}/")
            page.get_by_label("Find a public transport route").fill("24")
            page.get_by_label("Transport type").select_option("bus")
            page.get_by_label("Location (optional)").fill("Bristol")
            page.get_by_role("button", name="Search", exact=True).click()
            browser_api.expect(page.locator("#search-status")).to_contain_text("Choose a location")
            with page.expect_request("**/api/search?*") as request:
                page.get_by_role("button", name="Bristol, UK", exact=True).click()
            params = parse_qs(request.value.url.split('?')[1])
            assert params['q'] == ['24'] and params['mode'] == ['bus']
            assert params['south'] == ['51.39'] and params['east'] == ['-2.51']
            browser_api.expect(page.locator('#search-area-label')).to_contain_text('Bristol, UK')
            assert page.evaluate('map.getBounds().contains(L.latLngBounds([51.39, -2.73], [51.55, -2.51]))')
            assert page.evaluate('map.getZoom()') > 2
            assert page.evaluate('searchAreaLayer !== null')
            browser_api.expect(page.locator("#search-status")).to_contain_text("More matches exist")
            shared_url = page.url
            shared_params = parse_qs(shared_url.split('?')[1])
            assert shared_params['location'] == ['Bristol']
            assert shared_params['place'] == ['Bristol, UK']
            assert shared_params['q'] == ['24'] and shared_params['mode'] == ['bus']
            assert len(location_requests) == 1
            page.set_viewport_size({'width': 390 if width == 1280 else 1280, 'height': 844})
            with page.expect_request('**/api/search?*') as request:
                page.reload()
            assert parse_qs(request.value.url.split('?')[1]) == params
            browser_api.expect(page.locator('#search-status')).to_contain_text('More matches exist')
            browser_api.expect(page.get_by_label('Find a public transport route')).to_have_value('24')
            browser_api.expect(page.get_by_label('Transport type')).to_have_value('bus')
            browser_api.expect(page.get_by_label('Location (optional)')).to_have_value('Bristol')
            browser_api.expect(page.locator('#search-area-label')).to_contain_text('Bristol, UK')
            assert page.url == shared_url
            assert len(location_requests) == 1
            page.set_viewport_size({'width': width, 'height': 844})
            if width == 390:
                page.evaluate('openPanel()')
            link = page.locator("#search-results a")
            browser_api.expect(link).to_contain_text("Metro M11 <script>")
            assert page.locator("#search-results script").count() == 0
            page.get_by_label('Search this map area').check()
            browser_api.expect(page.get_by_label('Location (optional)')).to_be_disabled()
            page.evaluate('map.setView([51.45, -2.59], 13, {animate: false})')
            with page.expect_request('**/api/search?*') as request:
                page.get_by_role('button', name='Search', exact=True).click()
            map_params = parse_qs(request.value.url.split('?')[1])
            assert float(map_params['south'][0]) < 51.45 < float(map_params['north'][0])
            assert float(map_params['west'][0]) < -2.59 < float(map_params['east'][0])
            browser_api.expect(page.locator('#search-status')).to_contain_text('More matches exist')
            shared_map_url = page.url
            assert parse_qs(shared_map_url.split('?')[1])['area'] == ['map']
            page.set_viewport_size({'width': 390 if width == 1280 else 1280, 'height': 844})
            with page.expect_request('**/api/search?*') as request:
                page.reload()
            assert parse_qs(request.value.url.split('?')[1]) == map_params
            browser_api.expect(page.locator('#search-status')).to_contain_text('More matches exist')
            browser_api.expect(page.get_by_label('Search this map area')).to_be_checked()
            browser_api.expect(page.get_by_label('Location (optional)')).to_be_disabled()
            assert page.url == shared_map_url
            page.set_viewport_size({'width': width, 'height': 844})
            if width == 390:
                page.evaluate('openPanel()')
            page.evaluate('map.setView([51.5, -2.7], 13, {animate: false})')
            with page.expect_request('**/api/search?*') as request:
                page.get_by_role('button', name='Search', exact=True).click()
            moved_params = parse_qs(request.value.url.split('?')[1])
            assert moved_params['west'] != map_params['west']
            browser_api.expect(page.locator('#search-status')).to_contain_text('More matches exist')
            with page.expect_request('**/api/search?*') as request:
                page.go_back()
            assert parse_qs(request.value.url.split('?')[1]) == map_params
            browser_api.expect(page.locator('#search-status')).to_contain_text('More matches exist')
            with page.expect_request('**/api/search?*') as request:
                page.go_forward()
            assert parse_qs(request.value.url.split('?')[1]) == moved_params
            browser_api.expect(page.locator('#search-status')).to_contain_text('More matches exist')
            page.get_by_label('Search this map area').uncheck()
            browser_api.expect(page.get_by_label('Location (optional)')).to_be_enabled()
            page.get_by_label('Location (optional)').fill('Bristol')
            page.get_by_role('button', name='Search', exact=True).click()
            browser_api.expect(page.locator('#search-status')).to_contain_text('Choose a location')
            with page.expect_request('**/api/search?*') as request:
                page.get_by_role('button', name='Bristol, UK', exact=True).click()
            assert parse_qs(request.value.url.split('?')[1])['south'] == ['51.39']
            browser_api.expect(page.locator('#search-status')).to_contain_text('More matches exist')
            link.click()
            browser_api.expect(page.locator("#route-name")).to_have_text("Metro M11")
            browser_api.expect(page).to_have_url(f"http://127.0.0.1:{server.server_port}/15083963")
            page.route("**/api/search?*", lambda route: route.fulfill(json={"routes": [], "truncated": False}))
            page.get_by_label("Find a public transport route").fill("Unknown")
            page.get_by_role("button", name="Search", exact=True).click()
            browser_api.expect(page.locator("#search-status")).to_contain_text("No routes found")
            assert page.locator("#search-results a").count() == 0
            world_url = page.url
            assert parse_qs(world_url.split('?')[1]) == {'q': ['Unknown']}
            with page.expect_request('**/api/search?*') as request:
                page.reload()
            assert parse_qs(request.value.url.split('?')[1]) == {'q': ['Unknown']}
            browser_api.expect(page.locator('#search-status')).to_contain_text('No routes found')
            page.route("**/api/search?*", lambda route: route.fulfill(status=502, json={"message": "Search unavailable"}))
            page.get_by_role("button", name="Search", exact=True).click()
            browser_api.expect(page.locator("#search-status")).to_have_text("Search unavailable")
            browser_api.expect(page.get_by_role("button", name="Search", exact=True)).to_be_enabled()
            page.route('**/api/locations?*', lambda route: route.fulfill(json={'locations': []}))
            page.get_by_label('Location (optional)').fill('Unknown place')
            page.get_by_role('button', name='Search', exact=True).click()
            browser_api.expect(page.locator('#search-status')).to_contain_text('No location found')
            page.route('**/api/locations?*', lambda route: route.fulfill(status=502, json={'message': 'Location unavailable'}))
            page.get_by_role('button', name='Search', exact=True).click()
            browser_api.expect(page.locator('#search-status')).to_have_text('Location unavailable')
            page.goto(f'http://127.0.0.1:{server.server_port}/?q=24&mode=bus&south=nan&west=-3&north=52&east=-2')
            browser_api.expect(page.locator('#search-status')).to_contain_text('Invalid search area')
            assert errors == []
            browser.close()
    finally:
        server.shutdown()
        thread.join()


@pytest.fixture()
def location_cache(monkeypatch: Any) -> None:
    """Give each location test an empty cache and no pending rate-limit delay."""
    core._find_locations_cached.cache_clear()
    monkeypatch.setattr(core, '_last_location_request', 0.0)


@responses.activate
def test_locations_cache_and_bounds(location_cache: None) -> None:
    """Geocode worldwide, preserve candidate areas, and reuse successful lookups."""
    responses.get(core.NOMINATIM_URL, json=[
        {"display_name": "Bristol, UK", "boundingbox": ["51.39", "51.55", "-2.73", "-2.51"]},
        {"display_name": "Bristol, USA", "boundingbox": ["41.6", "41.8", "-72.9", "-72.8"]},
        {"display_name": "Invalid", "boundingbox": ["nan", "52", "-3", "-2"]},
    ])
    locations = core.find_locations(' Bristol ')
    assert locations[0] == {"label": "Bristol, UK", "bounds": (51.39, -2.73, 51.55, -2.51)}
    assert len(locations) == 2
    assert core.find_locations('Bristol') == locations
    assert len(responses.calls) == 1
    assert 'countrycodes' not in responses.calls[0].request.url
    assert 'openstreetmap.tools' in responses.calls[0].request.headers['User-Agent']


@responses.activate
def test_location_api(app: Any, location_cache: None) -> None:
    """Location resolution returns candidates or explicit errors, never global routes."""
    client = app.test_client()
    responses.get(core.NOMINATIM_URL, json=[])
    result = client.get('/api/locations?q=Nowhere')
    assert result.status_code == 200
    assert result.json == {"locations": []}
    assert client.get('/api/locations').status_code == 400
    assert client.get('/api/locations?q=x').status_code == 400


@pytest.mark.parametrize('payload', [{}, 'broken JSON', requests.Timeout('timeout'), 429])
@responses.activate
def test_location_failure(payload: Any, location_cache: None) -> None:
    """Upstream errors and invalid location payloads become OsmError."""
    if isinstance(payload, int):
        responses.get(core.NOMINATIM_URL, status=payload)
    elif isinstance(payload, dict):
        responses.get(core.NOMINATIM_URL, json=payload)
    else:
        responses.get(core.NOMINATIM_URL, body=payload)
    with pytest.raises(core.OsmError) as error:
        core.find_locations('Bristol')
    assert error.value.status_code == 502


@responses.activate
def test_area_query_and_parent_masters() -> None:
    """Scope routes before text filtering and include masters of local routes."""
    responses.post(core.OVERPASS_URL, json={"elements": [relation(24, ref='24')]})
    result = core.search_routes('24', 'bus', (51.39, -2.73, 51.55, -2.51))
    assert result['routes'][0]['ref'] == '24'
    query = parse_qs(responses.calls[0].request.body)['data'][0]
    assert 'rel(51.3900000,-2.7300000,51.5500000,-2.5100000)' in query
    assert 'rel(br.areaRoutes)["type"="route_master"]' in query
    assert 'rel.inArea["type"="route"]["route"~"^(bus)$"]["ref"~"^24$",i]' in query
    assert 'rel.inArea["type"="route_master"]' in query


@pytest.mark.parametrize('suffix', [
    '&south=51', '&south=x&west=-3&north=52&east=-2',
    '&south=nan&west=-3&north=52&east=-2',
    '&south=51&west=-3&north=inf&east=-2',
    '&south=52&west=-3&north=51&east=-2',
    '&south=51&west=179&north=52&east=-179',
    '&south=-90&west=-180&north=90&east=180',
])
@responses.activate
def test_invalid_area_api(app: Any, suffix: str) -> None:
    """Reject partial, invalid, broad and date-line-crossing bounds without HTTP."""
    result = app.test_client().get('/api/search?q=24' + suffix)
    assert result.status_code == 400
    assert result.json['error'] == 'invalid_query'
    assert len(responses.calls) == 0


@responses.activate
def test_area_api(app: Any) -> None:
    """Parse and pass all four map coordinates through to Overpass."""
    responses.post(core.OVERPASS_URL, json={"elements": []})
    result = app.test_client().get('/api/search?q=24&mode=bus&south=51.39&west=-2.73&north=51.55&east=-2.51')
    assert result.status_code == 200
    assert 'rel(51.3900000,-2.7300000,51.5500000,-2.5100000)' in parse_qs(responses.calls[0].request.body)['data'][0]


@responses.activate
def test_location_rate_limit(location_cache: None, monkeypatch: Any) -> None:
    """Distinct lookups are spaced one second apart; cached ones need no delay."""
    delays: list[float] = []
    monkeypatch.setattr(core.time, 'monotonic', lambda: 10.0)
    monkeypatch.setattr(core.time, 'sleep', delays.append)
    responses.get(core.NOMINATIM_URL, json=[])
    core.find_locations('Bristol')
    core.find_locations('London')
    core.find_locations('Bristol')
    assert delays == [1.0]
    assert len(responses.calls) == 2