Stream advance fares and selectable ticket classes

scraper/gwr_fares.py

@@ -85,7 +85,7 @@ def _run_pages(station_crs: str, travel_date: str, first_class: bool = False):
                 body["standardclass"] = False
             resp = client.post(_API_URL, json=body)
             resp.raise_for_status()
-            data = resp.json().get("data", {})
+            data = resp.json().get("data") or {}
             conversation_token = data.get("conversationToken")
             for journey in data.get("outwardOpenPureReturnFare", []):
                 dep_iso = journey.get("departureTime", "")
@@ -99,6 +99,39 @@ def _run_pages(station_crs: str, travel_date: str, first_class: bool = False):
             later = True
 
 
+def _run_pages_batched(station_crs: str, travel_date: str, first_class: bool = False):
+    """
+    Like _run_pages but yields one list of (dep_time, fares_list) per API page call,
+    allowing callers to stream results a page at a time.
+    """
+    seen: set[str] = set()
+    with httpx.Client(headers=_headers(), timeout=30) as client:
+        conversation_token = None
+        later = False
+        for _ in range(_MAX_PAGES):
+            body = _request_body(station_crs, travel_date, conversation_token, later)
+            if first_class:
+                body["firstclass"] = True
+                body["standardclass"] = False
+            resp = client.post(_API_URL, json=body)
+            resp.raise_for_status()
+            data = resp.json().get("data") or {}
+            conversation_token = data.get("conversationToken")
+            batch = []
+            for journey in data.get("outwardOpenPureReturnFare", []):
+                dep_iso = journey.get("departureTime", "")
+                dep_time = dep_iso[11:16]
+                if not dep_time or dep_time in seen:
+                    continue
+                seen.add(dep_time)
+                batch.append((dep_time, journey.get("journeyFareDetails", [])))
+            if batch:
+                yield batch
+            if not data.get("showLaterOutward", False):
+                break
+            later = True
+
+
 def fetch(station_crs: str, travel_date: str) -> dict[str, dict]:
     """
     Fetch GWR walk-on single fares from station_crs to London Paddington on travel_date.
@@ -192,3 +225,69 @@ def fetch_advance(station_crs: str, travel_date: str) -> dict[str, dict]:
         }
         for t in all_times
     }
+
+
+def fetch_advance_streaming(station_crs: str, travel_date: str):
+    """
+    Generator yielding partial advance fare dicts one GWR API page at a time.
+
+    Each yield is {dep_time: {'advance_std': dict|None, 'advance_1st': dict|None}}.
+    Two passes are made (standard class then first class); each page of results is
+    yielded immediately so callers can stream prices to clients as they arrive.
+    """
+    # Pass 1: standard class advance fares
+    for batch in _run_pages_batched(station_crs, travel_date, first_class=False):
+        page: dict[str, dict] = {}
+        for dep_time, fares in batch:
+            cheapest = None
+            for fare in fares:
+                code = fare.get("ticketTypeCode")
+                if code in _WALKON_CODES:
+                    continue
+                if not fare.get("isStandardClass"):
+                    continue
+                price_pence = fare.get("fare", 0)
+                if cheapest is None or price_pence < cheapest["price_pence"]:
+                    cheapest = {
+                        "ticket": fare.get("ticketType", ""),
+                        "price": price_pence / 100,
+                        "price_pence": price_pence,
+                        "code": code,
+                    }
+            if cheapest:
+                page[dep_time] = {
+                    "advance_std": {
+                        "ticket": cheapest["ticket"],
+                        "price": cheapest["price"],
+                        "code": cheapest["code"],
+                    },
+                    "advance_1st": None,
+                }
+        if page:
+            yield page
+
+    # Pass 2: first class advance fares
+    for batch in _run_pages_batched(station_crs, travel_date, first_class=True):
+        page = {}
+        for dep_time, fares in batch:
+            cheapest = None
+            for fare in fares:
+                price_pence = fare.get("fare", 0)
+                if cheapest is None or price_pence < cheapest["price_pence"]:
+                    cheapest = {
+                        "ticket": fare.get("ticketType", ""),
+                        "price": price_pence / 100,
+                        "price_pence": price_pence,
+                        "code": fare.get("ticketTypeCode"),
+                    }
+            if cheapest:
+                page[dep_time] = {
+                    "advance_std": None,
+                    "advance_1st": {
+                        "ticket": cheapest["ticket"],
+                        "price": cheapest["price"],
+                        "code": cheapest["code"],
+                    },
+                }
+        if page:
+            yield page