Cache provisional weekday timetables

This commit is contained in:
Edward Betts 2026-05-21 11:31:17 +01:00
parent 378d2484d0
commit bc7cb9cffa
6 changed files with 686 additions and 58 deletions

410
app.py
View file

@ -96,6 +96,117 @@ VALID_NR_CLASSES = {'walkon', 'advance_std', 'advance_1st'}
VALID_ES_CLASSES = {'standard', 'plus'}
DEFAULT_NR_CLASS = 'walkon'
DEFAULT_ES_CLASS = 'standard'
NR_TIMETABLE_PERIODS = [
(date(2026, 5, 17), date(2026, 12, 12), "2026-05-17_2026-12-12"),
]
def _weekday_for(section_date: str) -> str:
return date.fromisoformat(section_date).strftime("%a").lower()
def _month_for(section_date: str) -> str:
return date.fromisoformat(section_date).strftime("%Y-%m")
def _nr_timetable_period_key(section_date: str) -> str:
dt = date.fromisoformat(section_date)
for start, end, key in NR_TIMETABLE_PERIODS:
if start <= dt <= end:
return key
return dt.strftime("%Y-%m")
def _nr_exact_cache_key(direction: str, station_crs: str, section_date: str) -> str:
return f"rtt_{direction}_{station_crs}_{section_date}"
def _nr_weekday_cache_key(direction: str, station_crs: str, section_date: str) -> str:
return (
f"weekday_rtt_{direction}_{station_crs}_"
f"{_nr_timetable_period_key(section_date)}_{_weekday_for(section_date)}"
)
def _eurostar_exact_cache_key(direction: str, section_date: str, destination: str) -> str:
return f"eurostar_{direction}_{section_date}_{destination}"
def _eurostar_weekday_cache_key(direction: str, section_date: str, destination: str) -> str:
return (
f"weekday_eurostar_{direction}_{destination}_"
f"{_month_for(section_date)}_{_weekday_for(section_date)}"
)
def _eurostar_return_exact_cache_key(travel_date: str, return_date: str, destination: str) -> str:
return f"eurostar_return_{travel_date}_{return_date}_{destination}"
def _eurostar_return_weekday_cache_key(travel_date: str, return_date: str, destination: str) -> str:
return (
f"weekday_eurostar_return_{destination}_"
f"{_month_for(travel_date)}_{_weekday_for(travel_date)}_"
f"{_month_for(return_date)}_{_weekday_for(return_date)}"
)
def _strip_nr_timetable(trains):
keys = {
"depart_bristol",
"arrive_paddington",
"depart_paddington",
"arrive_destination",
"arrive_platform",
"headcode",
}
return [{k: train[k] for k in keys if k in train} for train in trains]
def _strip_eurostar_timetable(services):
keys = {
"depart_st_pancras",
"arrive_destination",
"depart_destination",
"arrive_st_pancras",
"destination",
"train_number",
}
return [{k: service[k] for k in keys if k in service} for service in services]
def _strip_eurostar_return_timetable(es_return):
if not isinstance(es_return, dict):
return {"outbound": [], "inbound": []}
return {
"outbound": _strip_eurostar_timetable(es_return.get("outbound", [])),
"inbound": _strip_eurostar_timetable(es_return.get("inbound", [])),
}
def _timetable_signature(data) -> str:
return json.dumps(data, sort_keys=True, separators=(",", ":"))
def _eurostar_prices_by_row(section_id: str, direction: str, services):
prices = {}
for service in services:
key = service.get("depart_st_pancras") if direction == "outbound" else service.get("depart_destination")
if not key:
continue
prices[f"{section_id}:{key}"] = {
"es_standard": (
{"price": service.get("price"), "seats": service.get("seats")}
if service.get("price") is not None
else None
),
"es_plus": (
{"price": service.get("plus_price"), "seats": service.get("plus_seats")}
if service.get("plus_price") is not None
else None
),
}
return prices
def _get_defaults():
@ -255,6 +366,11 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
):
dt = date.fromisoformat(travel_date)
travel_date_display = dt.strftime("%A %-d %B %Y")
return_date_display = (
date.fromisoformat(return_date).strftime("%A %-d %B %Y")
if return_date
else None
)
full_args = dict(request.args)
full_args.pop("progressive", None)
full_args.pop("journey_type", None)
@ -267,6 +383,7 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
journey_type=journey_type,
travel_date_display=travel_date_display,
return_date=return_date,
return_date_display=return_date_display,
full_results_url=_results_url(
station_crs=station_crs,
slug=slug,
@ -281,6 +398,7 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
user_agent = request.headers.get("User-Agent", rtt_scraper.DEFAULT_UA)
error_messages = []
from_cache_parts = []
provisional_timetable = False
def cached_fetch(key, ttl, fetcher, label):
cached = get_cached(key, ttl=ttl)
@ -295,14 +413,38 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
error_messages.append(f"Could not fetch {label}: {e}")
return [] if label != "GWR fares" else {}
def cached_timetable_fetch(exact_key, weekday_key, fetcher, label, stripper, ttl=None):
nonlocal provisional_timetable
cached = get_cached(exact_key, ttl=ttl)
if cached is not None:
from_cache_parts.append(exact_key)
return cached, False
weekday_cached = get_cached(weekday_key)
if weekday_cached is not None:
from_cache_parts.append(weekday_key)
provisional_timetable = True
return weekday_cached, True
try:
data = fetcher()
set_cached(exact_key, data)
set_cached(weekday_key, stripper(data))
return data, False
except Exception as e:
error_messages.append(f"Could not fetch {label}: {e}")
if label == "Eurostar return times":
return {"outbound": [], "inbound": []}, False
return [], False
es_return = None
es_return_provisional = False
if journey_type == "return":
es_return_key = f"eurostar_return_{travel_date}_{return_date}_{destination}"
es_return = cached_fetch(
es_return_key,
24 * 3600,
es_return, es_return_provisional = cached_timetable_fetch(
_eurostar_return_exact_cache_key(travel_date, return_date, destination),
_eurostar_return_weekday_cache_key(travel_date, return_date, destination),
lambda: eurostar_scraper.fetch_return(destination, travel_date, return_date),
"Eurostar times",
"Eurostar return times",
_strip_eurostar_return_timetable,
24 * 3600,
)
if not isinstance(es_return, dict):
es_return = {"outbound": [], "inbound": []}
@ -314,50 +456,63 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
section_min_connection = INBOUND_MIN_CONNECTION_MINUTES
section_max_connection = INBOUND_MAX_CONNECTION_MINUTES
rtt_direction = "to_paddington" if direction == "outbound" else "from_paddington"
rtt_cache_key = f"rtt_{rtt_direction}_{station_crs}_{section_date}"
rtt_cache_key = _nr_exact_cache_key(rtt_direction, station_crs, section_date)
rtt_weekday_cache_key = _nr_weekday_cache_key(rtt_direction, station_crs, section_date)
gwr_cache_key = f"gwr_fares_{rtt_direction}_{station_crs}_{section_date}"
advance_cache_key = f"gwr_advance_{rtt_direction}_{station_crs}_{section_date}"
if direction == "outbound":
trains = cached_fetch(
trains, nr_provisional = cached_timetable_fetch(
rtt_cache_key,
None,
rtt_weekday_cache_key,
lambda: rtt_scraper.fetch(section_date, user_agent, station_crs),
"GWR trains",
_strip_nr_timetable,
)
else:
trains = cached_fetch(
trains, nr_provisional = cached_timetable_fetch(
rtt_cache_key,
None,
rtt_weekday_cache_key,
lambda: rtt_scraper.fetch_from_paddington(section_date, user_agent, station_crs),
"GWR trains",
_strip_nr_timetable,
)
es_provisional = es_return_provisional if journey_type == "return" else False
if eurostar_services is None:
es_cache_key = f"eurostar_{direction}_{section_date}_{destination}"
es_cache_key = _eurostar_exact_cache_key(direction, section_date, destination)
es_weekday_cache_key = _eurostar_weekday_cache_key(direction, section_date, destination)
es_fetcher = (
(lambda: eurostar_scraper.fetch(destination, section_date))
if direction == "outbound"
else (lambda: eurostar_scraper.fetch(destination, section_date, direction=direction))
)
eurostar_services = cached_fetch(
eurostar_services, es_provisional = cached_timetable_fetch(
es_cache_key,
24 * 3600,
es_weekday_cache_key,
es_fetcher,
"Eurostar times",
_strip_eurostar_timetable,
24 * 3600,
)
fare_direction = "to_paddington" if direction == "outbound" else "from_paddington"
gwr_fares = cached_fetch(
gwr_cache_key,
30 * 24 * 3600,
(
(lambda: gwr_fares_scraper.fetch(station_crs, section_date))
if fare_direction == "to_paddington"
else (lambda: gwr_fares_scraper.fetch(station_crs, section_date, direction=fare_direction))
),
"GWR fares",
)
gwr_fares = get_cached(gwr_cache_key, ttl=30 * 24 * 3600)
if gwr_fares is not None:
from_cache_parts.append(gwr_cache_key)
elif nr_provisional or es_provisional:
gwr_fares = {}
else:
gwr_fares = cached_fetch(
gwr_cache_key,
30 * 24 * 3600,
(
(lambda: gwr_fares_scraper.fetch(station_crs, section_date))
if fare_direction == "to_paddington"
else (lambda: gwr_fares_scraper.fetch(station_crs, section_date, direction=fare_direction))
),
"GWR fares",
)
cached_advance = get_cached(advance_cache_key, ttl=24 * 3600)
if direction == "outbound":
@ -424,7 +579,12 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
row["eurostar_seats"] = es.get("seats")
row["eurostar_plus_price"] = es.get("plus_price")
row["eurostar_plus_seats"] = es.get("plus_seats")
row["row_key"] = f"{section_id}:{key}"
row["eurostar_key"] = f"{section_id}:{key}"
if row.get("row_type") == "trip":
nr_key = row.get("depart_bristol") or row.get("depart_paddington")
row["row_key"] = f"{section_id}:{nr_key}:{key}"
else:
row["row_key"] = f"{section_id}:unreachable:{key}"
dt = date.fromisoformat(section_date)
return {
@ -438,7 +598,14 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
"eurostar_count": len(eurostar_services),
"min_connection": section_min_connection,
"max_connection": section_max_connection,
"provisional_timetable": nr_provisional or es_provisional,
"advance_fares": cached_advance,
"walkon_api_url": url_for(
"api_walkon_fares",
station_crs=station_crs,
travel_date=section_date,
direction=fare_direction,
),
"advance_api_url": url_for(
"api_advance_fares",
station_crs=station_crs,
@ -475,6 +642,14 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
prev_date = (dt - timedelta(days=1)).isoformat()
next_date = (dt + timedelta(days=1)).isoformat()
travel_date_display = dt.strftime("%A %-d %B %Y")
return_date_display = None
prev_return_date = return_date
next_return_date = return_date
if return_date:
return_dt = date.fromisoformat(return_date)
return_date_display = return_dt.strftime("%A %-d %B %Y")
prev_return_date = (return_dt - timedelta(days=1)).isoformat()
next_return_date = (return_dt + timedelta(days=1)).isoformat()
eurostar_url = eurostar_scraper.search_url(
destination, travel_date, direction=journey_type, return_date=return_date
@ -498,13 +673,13 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
station_crs,
slug,
prev_date,
**common_url_args,
**{**common_url_args, "return_date": prev_return_date},
)
next_results_url = _results_url(
station_crs,
slug,
next_date,
**common_url_args,
**{**common_url_args, "return_date": next_return_date},
)
destination_links = [
(
@ -529,10 +704,12 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
trip_fares = {}
advance_fares = {}
walkon_api_urls = {}
advance_api_urls = {}
advance_stream_urls = {}
for section in sections:
advance_fares[section["id"]] = section["advance_fares"]
walkon_api_urls[section["id"]] = section["walkon_api_url"]
advance_api_urls[section["id"]] = section["advance_api_url"]
advance_stream_urls[section["id"]] = section["advance_stream_url"]
for row in section["rows"]:
@ -555,6 +732,7 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
)
trip_fares[row["row_key"]] = {
"section": section["id"],
"eurostar_key": row.get("eurostar_key"),
"advance_key": row.get("depart_bristol") or row.get("depart_paddington"),
"walkon": walkon,
"es_standard": es_std,
@ -562,6 +740,23 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
"circle_fare": circle_fare,
}
if journey_type == "return":
timetable_refresh_url = url_for(
"api_return_results_refresh",
station_crs=station_crs,
slug=slug,
travel_date=travel_date,
return_date=return_date,
)
else:
timetable_refresh_url = url_for(
"api_results_refresh",
station_crs=station_crs,
slug=slug,
travel_date=travel_date,
journey_type=journey_type if journey_type == "inbound" else None,
)
return render_template(
"results.html",
sections=sections,
@ -583,9 +778,11 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
destination_links=destination_links,
results_base_url=results_base_url,
travel_date_display=travel_date_display,
return_date_display=return_date_display,
gwr_count=sum(section["gwr_count"] for section in sections),
eurostar_count=sum(section["eurostar_count"] for section in sections),
from_cache=bool(from_cache_parts),
provisional_timetable=provisional_timetable,
error="; ".join(error_messages) if error_messages else None,
no_prices_note=no_prices_note,
eurostar_url=eurostar_url,
@ -603,8 +800,10 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
url_es_class=url_es,
trip_fares_json=json.dumps(trip_fares),
advance_fares_json=json.dumps(advance_fares),
walkon_api_urls_json=json.dumps(walkon_api_urls),
advance_api_urls_json=json.dumps(advance_api_urls),
advance_stream_urls_json=json.dumps(advance_stream_urls),
timetable_refresh_url=timetable_refresh_url,
advance_fares_api_url=url_for("api_advance_fares", station_crs=station_crs, travel_date=travel_date),
advance_fares_stream_url=url_for("api_advance_fares_stream", station_crs=station_crs, travel_date=travel_date),
valid_min_connections=sorted(valid_min),
@ -612,6 +811,165 @@ def _results(station_crs, slug, travel_date, journey_type, return_date):
)
def _fetch_exact_nr_timetable(station_crs, section_date, direction, user_agent):
rtt_direction = "to_paddington" if direction == "outbound" else "from_paddington"
exact_key = _nr_exact_cache_key(rtt_direction, station_crs, section_date)
weekday_key = _nr_weekday_cache_key(rtt_direction, station_crs, section_date)
trains = (
rtt_scraper.fetch(section_date, user_agent, station_crs)
if direction == "outbound"
else rtt_scraper.fetch_from_paddington(section_date, user_agent, station_crs)
)
set_cached(exact_key, trains)
set_cached(weekday_key, _strip_nr_timetable(trains))
return trains
def _fetch_exact_eurostar_single(destination, section_date, direction):
exact_key = _eurostar_exact_cache_key(direction, section_date, destination)
weekday_key = _eurostar_weekday_cache_key(direction, section_date, destination)
services = (
eurostar_scraper.fetch(destination, section_date)
if direction == "outbound"
else eurostar_scraper.fetch(destination, section_date, direction=direction)
)
set_cached(exact_key, services)
set_cached(weekday_key, _strip_eurostar_timetable(services))
return services
def _fetch_exact_eurostar_return(destination, travel_date, return_date):
exact_key = _eurostar_return_exact_cache_key(travel_date, return_date, destination)
weekday_key = _eurostar_return_weekday_cache_key(travel_date, return_date, destination)
services = eurostar_scraper.fetch_return(destination, travel_date, return_date)
set_cached(exact_key, services)
set_cached(weekday_key, _strip_eurostar_return_timetable(services))
return services
@app.route("/api/walkon_fares/<station_crs>/<travel_date>")
def api_walkon_fares(station_crs, travel_date):
if station_crs not in STATION_BY_CRS:
abort(404)
direction = request.args.get("direction", "to_paddington")
if direction not in {"to_paddington", "from_paddington"}:
direction = "to_paddington"
cache_key = f"gwr_fares_{direction}_{station_crs}_{travel_date}"
cached = get_cached(cache_key, ttl=30 * 24 * 3600)
if cached is not None:
return jsonify(cached)
try:
fares = (
gwr_fares_scraper.fetch(station_crs, travel_date)
if direction == "to_paddington"
else gwr_fares_scraper.fetch(station_crs, travel_date, direction=direction)
)
set_cached(cache_key, fares)
return jsonify(fares)
except Exception as e:
return jsonify({"error": str(e)}), 500
@app.route("/api/results_refresh/<station_crs>/<slug>/<travel_date>")
def api_results_refresh(station_crs, slug, travel_date):
return _api_results_refresh(station_crs, slug, travel_date, request.args.get("return_date"))
@app.route("/api/results_refresh/<station_crs>/<slug>/<travel_date>/return/<return_date>")
def api_return_results_refresh(station_crs, slug, travel_date, return_date):
return _api_results_refresh(station_crs, slug, travel_date, return_date, "return")
def _api_results_refresh(station_crs, slug, travel_date, return_date=None, path_journey_type=None):
if station_crs not in STATION_BY_CRS:
abort(404)
destination = DESTINATIONS.get(slug)
if not destination:
abort(404)
journey_type = path_journey_type or request.args.get("journey_type", "outbound")
if journey_type not in VALID_JOURNEY_TYPES:
journey_type = "outbound"
if return_date is None:
return_date = request.args.get("return_date")
if journey_type == "return" and not return_date:
abort(400)
user_agent = request.headers.get("User-Agent", rtt_scraper.DEFAULT_UA)
def generate():
try:
old_es_weekdays = {}
if journey_type == "return":
old_es_weekdays["return"] = get_cached(
_eurostar_return_weekday_cache_key(travel_date, return_date, destination)
)
es_return = _fetch_exact_eurostar_return(destination, travel_date, return_date)
sections = [
("outbound", "outbound", travel_date, es_return.get("outbound", [])),
("inbound", "inbound", return_date, es_return.get("inbound", [])),
]
else:
direction = journey_type
section_date = travel_date
old_es_weekdays["main"] = get_cached(
_eurostar_weekday_cache_key(direction, section_date, destination)
)
es_services = _fetch_exact_eurostar_single(destination, section_date, direction)
sections = [("main", direction, section_date, es_services)]
reload_needed = False
eurostar_prices = {}
for section_id, direction, section_date, es_services in sections:
nr_weekday_key = _nr_weekday_cache_key(
"to_paddington" if direction == "outbound" else "from_paddington",
station_crs,
section_date,
)
old_nr_weekday = get_cached(nr_weekday_key)
exact_nr = _fetch_exact_nr_timetable(station_crs, section_date, direction, user_agent)
if old_nr_weekday is not None and _timetable_signature(old_nr_weekday) != _timetable_signature(_strip_nr_timetable(exact_nr)):
reload_needed = True
old_es_weekday = old_es_weekdays["return"] if journey_type == "return" else old_es_weekdays[section_id]
exact_es_timetable = (
_strip_eurostar_return_timetable(es_return)
if journey_type == "return"
else _strip_eurostar_timetable(es_services)
)
if old_es_weekday is not None and _timetable_signature(old_es_weekday) != _timetable_signature(exact_es_timetable):
reload_needed = True
eurostar_prices.update(_eurostar_prices_by_row(section_id, direction, es_services))
if reload_needed:
yield f"data: {json.dumps({'type': 'reload'})}\n\n"
yield f"data: {json.dumps({'type': 'done'})}\n\n"
return
if eurostar_prices:
yield f"data: {json.dumps({'type': 'eurostar_prices', 'prices': eurostar_prices})}\n\n"
for section_id, direction, section_date, _es_services in sections:
fare_direction = "to_paddington" if direction == "outbound" else "from_paddington"
cache_key = f"gwr_fares_{fare_direction}_{station_crs}_{section_date}"
cached = get_cached(cache_key, ttl=30 * 24 * 3600)
if cached is None:
cached = (
gwr_fares_scraper.fetch(station_crs, section_date)
if fare_direction == "to_paddington"
else gwr_fares_scraper.fetch(station_crs, section_date, direction=fare_direction)
)
set_cached(cache_key, cached)
yield f"data: {json.dumps({'type': 'walkon_fares', 'section': section_id, 'fares': cached})}\n\n"
except Exception as e:
yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
return
yield f"data: {json.dumps({'type': 'done'})}\n\n"
return Response(stream_with_context(generate()), mimetype="text/event-stream")
@app.route("/api/advance_fares/<station_crs>/<travel_date>")
def api_advance_fares(station_crs, travel_date):
if station_crs not in STATION_BY_CRS:

View file

@ -1,6 +1,7 @@
import json
import os
import time
import uuid
from config.default import CACHE_DIR # overridden by app config after import
@ -13,15 +14,21 @@ def _cache_path(key: str) -> str:
def get_cached(key: str, ttl: int | None = None):
"""Return cached data, or None if missing or older than ttl seconds."""
path = _cache_path(key)
if not os.path.exists(path):
try:
if not os.path.exists(path):
return None
if ttl is not None and time.time() - os.path.getmtime(path) > ttl:
return None
with open(path) as f:
return json.load(f)
except (OSError, json.JSONDecodeError):
return None
if ttl is not None and time.time() - os.path.getmtime(path) > ttl:
return None
with open(path) as f:
return json.load(f)
def set_cached(key: str, data) -> None:
os.makedirs(CACHE_DIR, exist_ok=True)
with open(_cache_path(key), 'w') as f:
path = _cache_path(key)
tmp_path = f"{path}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
with open(tmp_path, 'w') as f:
json.dump(data, f, indent=2)
os.replace(tmp_path, path)

View file

@ -23,7 +23,7 @@
<div class="date-nav">
<a href="{{ prev_results_url }}"
class="btn-nav">&larr; Prev</a>
<strong>{{ travel_date_display }}{% if return_date %} to {{ return_date }}{% endif %}</strong>
<strong>{{ travel_date_display }}{% if return_date_display %} to {{ return_date_display }}{% endif %}</strong>
<a href="{{ next_results_url }}"
class="btn-nav">Next &rarr;</a>
</div>
@ -87,8 +87,11 @@
const DEFAULT_MAX_CONN = {{ default_max_connection }};
let TRIP_FARES = {{ trip_fares_json | safe }};
let ADVANCE_FARES = {{ advance_fares_json | safe }};
let WALKON_API_URLS = {{ walkon_api_urls_json | safe }};
let ADVANCE_API_URLS = {{ advance_api_urls_json | safe }};
let ADVANCE_STREAM_URLS = {{ advance_stream_urls_json | safe }};
const TIMETABLE_REFRESH_URL = {{ timetable_refresh_url|tojson }};
const HAS_PROVISIONAL_TIMETABLE = {{ 'true' if provisional_timetable else 'false' }};
let cachedAdvanceFares = ADVANCE_FARES;
let currentNrClass = '{{ nr_class }}';
let currentEsClass = '{{ es_class }}';
@ -158,6 +161,26 @@
}
}
function mergeWalkonFares(sectionId, fares) {
for (var key in TRIP_FARES) {
var row = TRIP_FARES[key];
if (row.section !== sectionId || !row.advance_key || !fares[row.advance_key]) continue;
var fare = fares[row.advance_key];
row.walkon = {price: fare.price, ticket: fare.ticket || ''};
}
}
function mergeEurostarPrices(prices) {
for (var key in prices) {
for (var rowKey in TRIP_FARES) {
var row = TRIP_FARES[rowKey];
if (rowKey !== key && row.eurostar_key !== key) continue;
if (prices[key].es_standard) row.es_standard = prices[key].es_standard;
if (prices[key].es_plus) row.es_plus = prices[key].es_plus;
}
}
}
function sectionNeedsAdvance(sectionId) {
for (var key in TRIP_FARES) {
var row = TRIP_FARES[key];
@ -306,7 +329,8 @@
html += '</span>';
totalSpan.innerHTML = html;
} else {
totalSpan.innerHTML = '';
var missing = !nrFare ? 'No NR fare' : 'No Eurostar fare';
totalSpan.innerHTML = '<span class="text-xs text-muted">' + missing + '</span>';
}
}
});
@ -315,6 +339,30 @@
function initialiseResultsPage() {
if (currentNrClass === 'advance_std' || currentNrClass === 'advance_1st') loadMissingAdvanceFares();
updateDisplay();
startTimetableRefresh();
}
function startTimetableRefresh() {
if (!HAS_PROVISIONAL_TIMETABLE || !TIMETABLE_REFRESH_URL || !window.EventSource) return;
var source = new EventSource(TIMETABLE_REFRESH_URL);
source.onmessage = function(event) {
var msg = JSON.parse(event.data);
if (msg.type === 'reload') {
source.close();
window.location.reload();
} else if (msg.type === 'eurostar_prices') {
mergeEurostarPrices(msg.prices);
updateDisplay();
} else if (msg.type === 'walkon_fares') {
mergeWalkonFares(msg.section, msg.fares);
updateDisplay();
} else if (msg.type === 'done' || msg.type === 'error') {
source.close();
}
};
source.onerror = function() {
source.close();
};
}
if (document.readyState === 'loading') {
@ -330,6 +378,9 @@
{% if from_cache %}
&nbsp;&middot;&nbsp; <span class="text-muted text-sm">(cached)</span>
{% endif %}
{% if provisional_timetable %}
&nbsp;&middot;&nbsp; <span class="text-muted text-sm">checking exact timetable</span>
{% endif %}
</p>
{% if error %}
<div class="alert alert-error">

View file

@ -21,7 +21,7 @@
{% endif %}
</h2>
<p class="card-meta">
{{ travel_date_display }}{% if return_date %} to {{ return_date }}{% endif %}
{{ travel_date_display }}{% if return_date_display %} to {{ return_date_display }}{% endif %}
</p>
<div class="loading-panel" role="status" aria-live="polite">
<span class="spinner" aria-hidden="true"></span>
@ -37,6 +37,8 @@
<script>
(function() {
var attempts = 0;
function runScripts(root) {
root.querySelectorAll('script').forEach(function(oldScript) {
var script = document.createElement('script');
@ -49,27 +51,36 @@
});
}
fetch({{ full_results_url|tojson }}, {headers: {'X-Requested-With': 'fetch'}})
.then(function(response) {
if (!response.ok) throw new Error('Could not load results');
return response.text();
})
.then(function(html) {
var doc = new DOMParser().parseFromString(html, 'text/html');
document.title = doc.title;
var nextMain = doc.querySelector('main');
var currentMain = document.querySelector('main');
if (!nextMain || !currentMain) throw new Error('Results page was incomplete');
currentMain.innerHTML = nextMain.innerHTML;
runScripts(currentMain);
history.replaceState(null, '', window.location.href);
})
.catch(function() {
var panel = document.querySelector('.loading-panel');
if (panel) {
panel.innerHTML = '<div><strong>Could not load results</strong><p class="text-muted text-sm"><a href="{{ full_results_url }}">Try loading the full results page</a>.</p></div>';
}
});
function loadResults() {
attempts += 1;
fetch({{ full_results_url|tojson }}, {headers: {'X-Requested-With': 'fetch'}})
.then(function(response) {
if (!response.ok) throw new Error('Could not load results');
return response.text();
})
.then(function(html) {
var doc = new DOMParser().parseFromString(html, 'text/html');
document.title = doc.title;
var nextMain = doc.querySelector('main');
var currentMain = document.querySelector('main');
if (!nextMain || !currentMain) throw new Error('Results page was incomplete');
currentMain.innerHTML = nextMain.innerHTML;
runScripts(currentMain);
history.replaceState(null, '', window.location.href);
})
.catch(function() {
if (attempts < 3) {
window.setTimeout(loadResults, attempts * 2000);
return;
}
var panel = document.querySelector('.loading-panel');
if (panel) {
panel.innerHTML = '<div><strong>Could not load results</strong><p class="text-muted text-sm"><a href="{{ full_results_url }}">Try loading the full results page</a>.</p></div>';
}
});
}
loadResults();
})();
</script>

View file

@ -79,6 +79,12 @@ def test_search_redirects_return_with_return_date():
)
def test_nr_weekday_cache_key_includes_timetable_period():
key = app_module._nr_weekday_cache_key("to_paddington", "BRI", "2026-06-22")
assert key == "weekday_rtt_to_paddington_BRI_2026-05-17_2026-12-12_mon"
def test_results_shows_same_day_destination_switcher(monkeypatch):
_stub_data(monkeypatch)
client = _client()
@ -94,6 +100,140 @@ def test_results_shows_same_day_destination_switcher(monkeypatch):
assert 'ES 9014' in html
def test_results_can_render_from_weekday_timetable_cache(monkeypatch):
travel_date = "2026-06-22"
cache = {
app_module._nr_weekday_cache_key("to_paddington", "BRI", travel_date): [
{'depart_bristol': '07:00', 'arrive_paddington': '08:45', 'headcode': '1A23'},
],
app_module._eurostar_weekday_cache_key("outbound", travel_date, "Paris Gare du Nord"): [
{'depart_st_pancras': '10:01', 'arrive_destination': '13:34',
'destination': 'Paris Gare du Nord', 'train_number': 'ES 9014'},
],
}
monkeypatch.setattr(app_module, 'get_cached', lambda key, ttl=None: cache.get(key))
monkeypatch.setattr(app_module, 'set_cached', lambda key, data: None)
monkeypatch.setattr(
app_module.rtt_scraper,
'fetch',
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("should use weekday NR cache")),
)
monkeypatch.setattr(
app_module.eurostar_scraper,
'fetch',
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("should use weekday Eurostar cache")),
)
monkeypatch.setattr(
app_module.gwr_fares_scraper,
'fetch',
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("should stream prices later")),
)
client = _client()
resp = client.get('/results/BRI/paris/2026-06-22?min_connection=60&max_connection=120')
html = resp.get_data(as_text=True)
assert resp.status_code == 200
assert '07:00 &rarr; 08:45' in html
assert '10:01 &rarr; 13:34' in html
assert 'ES 9014' in html
assert 'checking exact timetable' in html
assert '/api/results_refresh/BRI/paris/2026-06-22' in html
def test_results_refresh_reloads_when_exact_timetable_differs(monkeypatch):
travel_date = "2026-06-22"
cache = {
app_module._nr_weekday_cache_key("to_paddington", "BRI", travel_date): [
{'depart_bristol': '07:00', 'arrive_paddington': '08:45', 'headcode': '1A23'},
],
app_module._eurostar_weekday_cache_key("outbound", travel_date, "Paris Gare du Nord"): [
{'depart_st_pancras': '10:01', 'arrive_destination': '13:34',
'destination': 'Paris Gare du Nord', 'train_number': 'ES 9014'},
],
}
monkeypatch.setattr(app_module, 'get_cached', lambda key, ttl=None: cache.get(key))
monkeypatch.setattr(app_module, 'set_cached', lambda key, data: cache.__setitem__(key, data))
monkeypatch.setattr(
app_module.rtt_scraper,
'fetch',
lambda travel_date, user_agent, station_crs='BRI': [
{'depart_bristol': '07:05', 'arrive_paddington': '08:50', 'headcode': '1A24'},
],
)
monkeypatch.setattr(
app_module.eurostar_scraper,
'fetch',
lambda destination, travel_date: [
{'depart_st_pancras': '10:01', 'arrive_destination': '13:34',
'destination': destination, 'train_number': 'ES 9014',
'price': 59, 'seats': 42, 'plus_price': 89, 'plus_seats': 5},
],
)
monkeypatch.setattr(
app_module.gwr_fares_scraper,
'fetch',
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("reload should stop before fare fetch")),
)
client = _client()
resp = client.get('/api/results_refresh/BRI/paris/2026-06-22')
body = resp.get_data(as_text=True)
assert resp.status_code == 200
assert '"type": "reload"' in body
assert cache[app_module._nr_exact_cache_key("to_paddington", "BRI", travel_date)][0]['depart_bristol'] == '07:05'
assert cache[app_module._nr_weekday_cache_key("to_paddington", "BRI", travel_date)][0]['depart_bristol'] == '07:05'
def test_results_refresh_streams_prices_when_timetable_matches(monkeypatch):
travel_date = "2026-06-22"
nr_timetable = [
{'depart_bristol': '07:00', 'arrive_paddington': '08:45', 'headcode': '1A23'},
]
es_timetable = [
{'depart_st_pancras': '10:01', 'arrive_destination': '13:34',
'destination': 'Paris Gare du Nord', 'train_number': 'ES 9014'},
]
cache = {
app_module._nr_weekday_cache_key("to_paddington", "BRI", travel_date): nr_timetable,
app_module._eurostar_weekday_cache_key("outbound", travel_date, "Paris Gare du Nord"): es_timetable,
}
monkeypatch.setattr(app_module, 'get_cached', lambda key, ttl=None: cache.get(key))
monkeypatch.setattr(app_module, 'set_cached', lambda key, data: cache.__setitem__(key, data))
monkeypatch.setattr(
app_module.rtt_scraper,
'fetch',
lambda travel_date, user_agent, station_crs='BRI': nr_timetable,
)
monkeypatch.setattr(
app_module.eurostar_scraper,
'fetch',
lambda destination, travel_date: [
{**es_timetable[0], 'price': 59, 'seats': 42, 'plus_price': 89, 'plus_seats': 5},
],
)
monkeypatch.setattr(
app_module.gwr_fares_scraper,
'fetch',
lambda station_crs, travel_date: {
'07:00': {'ticket': 'Anytime Day Single', 'price': 138.70, 'code': 'SDS'},
},
)
client = _client()
resp = client.get('/api/results_refresh/BRI/paris/2026-06-22')
body = resp.get_data(as_text=True)
assert resp.status_code == 200
assert '"type": "reload"' not in body
assert '"type": "eurostar_prices"' in body
assert '"main:10:01"' in body
assert '"price": 59' in body
assert '"type": "walkon_fares"' in body
assert '"price": 138.7' in body
def test_results_progressive_shell_loads_without_scraping(monkeypatch):
def fail_fetch(*args, **kwargs):
raise AssertionError("progressive shell should not fetch data")
@ -111,6 +251,23 @@ def test_results_progressive_shell_loads_without_scraping(monkeypatch):
assert 'render=full' in html
def test_return_progressive_shell_formats_return_date(monkeypatch):
def fail_fetch(*args, **kwargs):
raise AssertionError("progressive shell should not fetch data")
monkeypatch.setattr(app_module.rtt_scraper, 'fetch', fail_fetch)
monkeypatch.setattr(app_module.eurostar_scraper, 'fetch_return', fail_fetch)
monkeypatch.setattr(app_module.gwr_fares_scraper, 'fetch', fail_fetch)
client = _client()
resp = client.get('/results/BRI/paris/2026-04-10/return/2026-04-17?progressive=1')
html = resp.get_data(as_text=True)
assert resp.status_code == 200
assert 'Friday 10 April 2026 to Friday 17 April 2026' in html
assert 'to 2026-04-17' not in html
def test_results_title_and_social_meta_include_destination(monkeypatch):
_stub_data(monkeypatch)
client = _client()
@ -221,6 +378,42 @@ def test_results_shows_eurostar_price_and_total(monkeypatch):
assert 'data-es-std="59"' in html
def test_results_uses_unique_row_keys_for_same_eurostar(monkeypatch):
monkeypatch.setattr(app_module, 'get_cached', lambda key, ttl=None: None)
monkeypatch.setattr(app_module, 'set_cached', lambda key, data: None)
monkeypatch.setattr(app_module.gwr_fares_scraper, 'fetch', lambda s, d: {
'07:00': {'ticket': 'Anytime Day Single', 'price': 138.70, 'code': 'SDS'},
'07:30': {'ticket': 'Anytime Day Single', 'price': 138.70, 'code': 'SDS'},
})
monkeypatch.setattr(
app_module.rtt_scraper,
'fetch',
lambda travel_date, user_agent, station_crs='BRI': [
{'depart_bristol': '07:00', 'arrive_paddington': '08:30', 'headcode': '1A01'},
{'depart_bristol': '07:30', 'arrive_paddington': '09:00', 'headcode': '1A02'},
],
)
monkeypatch.setattr(
app_module.eurostar_scraper,
'fetch',
lambda destination, travel_date: [
{'depart_st_pancras': '10:01', 'arrive_destination': '13:34',
'destination': destination, 'train_number': 'ES 9014',
'price': 59, 'seats': 42, 'plus_price': 89, 'plus_seats': 5},
],
)
client = _client()
resp = client.get('/results/BRI/paris/2026-04-10?min_connection=60&max_connection=120')
html = resp.get_data(as_text=True)
assert resp.status_code == 200
assert 'data-row-key="main:07:00:10:01"' in html
assert 'data-row-key="main:07:30:10:01"' in html
assert '"main:07:00:10:01"' in html
assert '"main:07:30:10:01"' in html
def test_results_shows_unreachable_service_when_no_trips(monkeypatch):
# Only one Eurostar at 09:30; GWR arrives 08:45 with min=60 → unreachable.
# No trips at all, so the unreachable service is shown as "Too early".
@ -423,8 +616,9 @@ def test_results_return_renders_outbound_and_inbound_tables(monkeypatch):
assert resp.status_code == 200
assert 'Outbound: Bristol Temple Meads &rarr; Paris Gare du Nord' in html
assert 'Return: Paris Gare du Nord &rarr; Bristol Temple Meads' in html
assert '/results/BRI/paris/2026-04-09/return/2026-04-17' in html
assert '/results/BRI/paris/2026-04-11/return/2026-04-17' in html
assert 'Friday 10 April 2026 to Friday 17 April 2026' in html
assert '/results/BRI/paris/2026-04-09/return/2026-04-16' in html
assert '/results/BRI/paris/2026-04-11/return/2026-04-18' in html
assert "/results/BRI/paris/2026-04-10/return/2026-04-17" in html
assert 'journey_type=return' not in html
assert 'return_date=2026-04-17' not in html

View file

@ -40,3 +40,10 @@ def test_get_cached_expired_returns_none(tmp_cache):
old = time.time() - 25 * 3600 # 25 hours ago
os.utime(path, (old, old))
assert get_cached('k', ttl=24 * 3600) is None
def test_get_cached_invalid_json_returns_none(tmp_cache):
path = tmp_cache / 'broken.json'
path.write_text('{"not": "finished"')
assert get_cached('broken') is None