343 lines
12 KiB
Python
343 lines
12 KiB
Python
"""On-demand flight searches with shared disk caching and refresh throttling."""
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import tempfile
|
|
import typing
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import flask
|
|
|
|
from agenda import airport_lookup, flight_search_cache, google_flights
|
|
from agenda.types import StrDict
|
|
|
|
LONDON_AIRPORTS = ("LHR", "LGW", "STN", "LTN", "LCY", "SEN")
|
|
CACHE_VERSION = 4
|
|
logger = logging.getLogger(__name__)
|
|
REFRESH_INTERVAL = timedelta(hours=6)
|
|
ERROR_INTERVAL = timedelta(minutes=15)
|
|
|
|
|
|
def cache_path(
|
|
data_dir: str, start: date, end: date, destination: str, short_haul: bool
|
|
) -> Path:
|
|
"""Cache by all search inputs, independently of conference names."""
|
|
key = json.dumps(
|
|
[
|
|
CACHE_VERSION,
|
|
start.isoformat(),
|
|
end.isoformat(),
|
|
destination,
|
|
short_haul,
|
|
"GBP",
|
|
"en-GB",
|
|
"GB",
|
|
]
|
|
)
|
|
return (
|
|
Path(data_dir)
|
|
/ "conference-flights"
|
|
/ (hashlib.sha256(key.encode()).hexdigest() + ".json")
|
|
)
|
|
|
|
|
|
def read_cache(path: Path) -> StrDict | None:
|
|
"""Read cached results; a missing or damaged cache is a cache miss."""
|
|
try:
|
|
value = json.loads(path.read_text())
|
|
if not isinstance(value, dict) or not isinstance(value.get("updated_at"), str):
|
|
return None
|
|
for field in ("updated_at", "attempted_at"):
|
|
if field in value and datetime.fromisoformat(value[field]).tzinfo is None:
|
|
return None
|
|
if not isinstance(value.get("searches"), list):
|
|
return None
|
|
return typing.cast(StrDict, value)
|
|
except (OSError, ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def write_cache(path: Path, value: StrDict) -> None:
|
|
"""Replace JSON atomically so concurrent readers never see partial results."""
|
|
with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as output:
|
|
temporary = output.name
|
|
json.dump(value, output)
|
|
try:
|
|
os.replace(temporary, path)
|
|
finally:
|
|
if os.path.exists(temporary):
|
|
os.unlink(temporary)
|
|
|
|
|
|
def airport_matches(query: str) -> list[StrDict]:
|
|
"""Look up locally, preferring airport names in the app's personal data."""
|
|
personal_data = (
|
|
str(flask.current_app.config["PERSONAL_DATA"])
|
|
if flask.has_app_context()
|
|
else None
|
|
)
|
|
return airport_lookup.matches(query, personal_data)
|
|
|
|
|
|
def resolve_airport(query: str) -> tuple[str, str]:
|
|
"""Resolve a code or the best name match, and expose its name for confirmation."""
|
|
codes = airport_lookup.code_list(query)
|
|
if len(codes) > 1:
|
|
for code in codes:
|
|
if not valid_airport(code):
|
|
raise ValueError(f"Unknown IATA airport code {code!r}.")
|
|
matches = airport_matches(",".join(codes))
|
|
if not matches:
|
|
raise ValueError("Unknown airport in destination group")
|
|
return str(matches[0]["code"]), str(matches[0]["name"])
|
|
code = query.strip().upper()
|
|
if valid_airport(code):
|
|
matches = airport_matches(code)
|
|
return code, str(matches[0]["name"]) if matches else code
|
|
if len(code) == 3 and code.isascii() and code.isalpha():
|
|
raise ValueError(
|
|
f"Unknown IATA airport code {code!r}. Enter a valid code or a longer city/airport name."
|
|
)
|
|
matches = airport_matches(query)
|
|
if not matches:
|
|
raise ValueError(
|
|
f"No airport found for {query!r}. Try a city, airport name or IATA code."
|
|
)
|
|
return str(matches[0]["code"]), str(matches[0]["name"])
|
|
|
|
|
|
def error_detail(exc: Exception) -> str:
|
|
"""Describe retry wrappers and underlying causes without losing the useful error."""
|
|
messages: list[str] = []
|
|
current: BaseException | None = exc
|
|
seen: set[int] = set()
|
|
while current is not None and id(current) not in seen and len(messages) < 3:
|
|
seen.add(id(current))
|
|
last_attempt = getattr(current, "last_attempt", None)
|
|
if last_attempt is not None:
|
|
underlying = last_attempt.exception()
|
|
if underlying is not None and id(underlying) not in seen:
|
|
current = underlying
|
|
continue
|
|
raw_message = str(current)
|
|
if "Browser logs:" in raw_message:
|
|
# Playwright repeats its long command before the useful stderr.
|
|
diagnostic = list(dict.fromkeys(re.findall(r"\[err\] (.+)", raw_message)))
|
|
if diagnostic:
|
|
raw_message = (
|
|
raw_message.split("Browser logs:", 1)[0]
|
|
+ " "
|
|
+ " ".join(diagnostic)
|
|
)
|
|
message = " ".join(raw_message.split())[:1000]
|
|
messages.append(
|
|
f"{type(current).__name__}: {message or 'No error message provided'}"
|
|
)
|
|
current = current.__cause__ or current.__context__
|
|
return " — ".join(messages)
|
|
|
|
|
|
def date_unavailable_message(exc: BaseException) -> str | None:
|
|
"""Use Google's date-range rejection as the banner through route wrappers."""
|
|
current: BaseException | None = exc
|
|
seen: set[int] = set()
|
|
while current is not None and id(current) not in seen:
|
|
seen.add(id(current))
|
|
if isinstance(current, google_flights.FlightDateUnavailableError):
|
|
return str(current)
|
|
current = current.__cause__ or current.__context__
|
|
return None
|
|
|
|
|
|
def flight_rank(result: StrDict) -> tuple[int, int]:
|
|
"""Prefer fewer stops, then BA-operated legs, preserving Google's ranking."""
|
|
ba_legs = [
|
|
leg.get("operating_airline_code", leg.get("airline_code")) == "BA"
|
|
for leg in result["legs"]
|
|
]
|
|
ba_preference = 0 if ba_legs and all(ba_legs) else 1 if any(ba_legs) else 2
|
|
return int(result["stops"]), ba_preference
|
|
|
|
|
|
def search_day(origin: str, destination: str, day: date, direct: bool) -> list[StrDict]:
|
|
"""Reuse a route/date cache before making any paced provider requests."""
|
|
directory = flight_search_cache.search_directory.get()
|
|
if directory is None:
|
|
raise RuntimeError(
|
|
"Flight searches must run inside a cached conference lookup."
|
|
)
|
|
return flight_search_cache.cached_day(
|
|
directory,
|
|
origin,
|
|
destination,
|
|
day.isoformat(),
|
|
direct,
|
|
lambda: fetch_day(origin, destination, day, direct, directory),
|
|
)
|
|
|
|
|
|
def fetch_day(
|
|
origin: str, destination: str, day: date, direct: bool, directory: Path
|
|
) -> list[StrDict]:
|
|
"""Read the browser search page, preferring direct and one-stop flights."""
|
|
max_stops = 0 if direct else 1
|
|
rows = google_flights.search(origin, destination, day, max_stops)
|
|
rows = [row for row in rows if row["stops"] <= max_stops]
|
|
if not rows and not direct:
|
|
rows = google_flights.search(origin, destination, day, 2)
|
|
rows = [row for row in rows if row["stops"] <= 2]
|
|
rows.sort(key=flight_rank)
|
|
return rows[:5]
|
|
|
|
|
|
def valid_airport(code: str) -> bool:
|
|
"""Accept known IATA codes and London's metropolitan code."""
|
|
return code == "LON" or code in airport_lookup.airports()
|
|
|
|
|
|
def search_origin(
|
|
origin: str, destination: str, start: date, end: date, flexible: bool
|
|
) -> StrDict:
|
|
"""Expand each direction from the closest date, stopping when flights are found."""
|
|
days = range(1, 5) if flexible else range(1, 3)
|
|
outbound: list[StrDict] = []
|
|
inbound: list[StrDict] = []
|
|
outbound_dates: list[str] = []
|
|
inbound_dates: list[str] = []
|
|
for offset in days:
|
|
departure = start - timedelta(days=offset if flexible else offset + 1)
|
|
if not outbound and departure >= date.today():
|
|
outbound_dates.append(departure.isoformat())
|
|
# Only stop expanding when flights arrive before the opening date.
|
|
outbound = [
|
|
row
|
|
for row in search_day(origin, destination, departure, origin == "BRS")
|
|
if date.fromisoformat(row["arrival"][:10]) < start
|
|
]
|
|
if not inbound and (flexible or offset == 1):
|
|
return_day = end + timedelta(days=offset)
|
|
inbound_dates.append(return_day.isoformat())
|
|
inbound = search_day(destination, origin, return_day, origin == "BRS")
|
|
if outbound and inbound:
|
|
break
|
|
return {
|
|
"origin": origin,
|
|
"outbound": outbound,
|
|
"inbound": inbound,
|
|
"outbound_dates": outbound_dates,
|
|
"inbound_dates": inbound_dates,
|
|
}
|
|
|
|
|
|
def lookup(
|
|
path: Path, start: date, end: date, destination: str, short_haul: bool
|
|
) -> StrDict:
|
|
"""Serialize searches across workers, reuse recent results, and cache errors."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
# A shared lock limits all conference requests, not just identical searches.
|
|
with (path.parent / "search.lock").open(
|
|
"a+"
|
|
) as lock, flight_search_cache.use_directory(
|
|
path.parent
|
|
), google_flights.use_browser(
|
|
path.parent
|
|
):
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise ValueError(
|
|
"Another flight lookup is running. Please try again shortly."
|
|
) from None
|
|
cached = read_cache(path)
|
|
now = datetime.now(timezone.utc)
|
|
legacy_empty_or_failed = (
|
|
cached
|
|
and cached.get("transport") != flight_search_cache.TRANSPORT
|
|
and (
|
|
cached.get("error")
|
|
or not any(
|
|
search.get("outbound") or search.get("inbound")
|
|
for search in cached.get("searches", [])
|
|
)
|
|
)
|
|
)
|
|
retry_old_parse_failure = (
|
|
cached
|
|
and cached.get("error")
|
|
and "Unrecognized Google Flights result format"
|
|
in cached.get("error_details", "")
|
|
and cached.get("parser_version") != google_flights.PARSER_VERSION
|
|
)
|
|
needs_earlier_departure = (
|
|
cached
|
|
and not cached.get("error")
|
|
and any(
|
|
search.get("origin") == "LON"
|
|
and not search.get("outbound")
|
|
and "outbound_dates" not in search
|
|
for search in cached.get("searches", [])
|
|
)
|
|
)
|
|
if (
|
|
cached
|
|
and not needs_earlier_departure
|
|
and not retry_old_parse_failure
|
|
and not legacy_empty_or_failed
|
|
and (not cached.get("error") or cached.get("error_details"))
|
|
):
|
|
timestamp = str(cached.get("attempted_at", cached["updated_at"]))
|
|
interval = ERROR_INTERVAL if cached.get("error") else REFRESH_INTERVAL
|
|
if now - datetime.fromisoformat(timestamp) < interval:
|
|
return cached
|
|
try:
|
|
searches = []
|
|
if short_haul:
|
|
bristol = search_origin("BRS", destination, start, end, flexible=True)
|
|
searches.append(bristol)
|
|
if not bristol["outbound"] or not bristol["inbound"]:
|
|
searches.append(
|
|
search_origin("LON", destination, start, end, flexible=False)
|
|
)
|
|
else:
|
|
searches.append(
|
|
search_origin("LON", destination, start, end, flexible=False)
|
|
)
|
|
value: StrDict = {
|
|
"updated_at": now.isoformat(),
|
|
"destination": destination,
|
|
"searches": searches,
|
|
"transport": flight_search_cache.TRANSPORT,
|
|
"parser_version": google_flights.PARSER_VERSION,
|
|
}
|
|
except Exception as exc:
|
|
logger.exception(
|
|
"Conference flight lookup failed for %s (%s to %s)",
|
|
destination,
|
|
start,
|
|
end,
|
|
)
|
|
# Preserve previously successful data and its original timestamp.
|
|
value = dict(
|
|
cached
|
|
or {
|
|
"updated_at": now.isoformat(),
|
|
"destination": destination,
|
|
"searches": [],
|
|
}
|
|
)
|
|
value.update(
|
|
error=flight_search_cache.cooldown_message(path.parent)
|
|
or date_unavailable_message(exc)
|
|
or "Flight lookup failed. Please try again in 15 minutes.",
|
|
error_details=error_detail(exc),
|
|
attempted_at=now.isoformat(),
|
|
transport=flight_search_cache.TRANSPORT,
|
|
parser_version=google_flights.PARSER_VERSION,
|
|
)
|
|
write_cache(path, value)
|
|
return value
|