#!/usr/bin/python3 """Combined update script for various data sources.""" import asyncio import os import sys import typing from datetime import date, datetime from time import time import deepdiff import flask import requests import agenda.bristol_waste import agenda.fx import agenda.geomob import agenda.gwr import agenda.mail import agenda.thespacedevs import agenda.types import agenda.uk_holiday from agenda.types import StrDict from web_view import app async def update_bank_holidays(config: flask.config.Config) -> None: """Update cached copy of UK Bank holidays.""" t0 = time() events = await agenda.uk_holiday.get_holiday_list(config["DATA_DIR"]) time_taken = time() - t0 if not sys.stdin.isatty(): return print(len(events), "bank holidays in list") print(f"took {time_taken:.1f} seconds") async def update_bristol_bins(config: flask.config.Config) -> None: """Update waste schedule from Bristol City Council.""" t0 = time() events = await agenda.bristol_waste.get( date.today(), config["DATA_DIR"], config["BRISTOL_UPRN"], cache="refresh", ) time_taken = time() - t0 if not sys.stdin.isatty(): return for event in events: print(event) print(f"took {time_taken:.1f} seconds") def update_gwr_advance_ticket_date(config: flask.config.Config) -> None: """Update GWR advance ticket date cache.""" filename = os.path.join(config["DATA_DIR"], "advance-tickets.html") existing_html = open(filename).read() existing_dates = agenda.gwr.extract_dates(existing_html) assert existing_dates assert list(existing_dates.keys()) == ["Weekdays", "Saturdays", "Sundays"] new_html = requests.get(agenda.gwr.url).text new_dates = agenda.gwr.extract_dates(new_html) if not new_dates: subject = "Error parsing GWR advance ticket booking dates" body = new_html agenda.mail.send_mail(config, subject, body) return assert new_dates assert list(new_dates.keys()) == ["Weekdays", "Saturdays", "Sundays"] if existing_dates == new_dates: if sys.stdin.isatty(): print(filename) print(agenda.gwr.url) print("dates haven't changed:", existing_dates) return open(filename, "w").write(new_html) subject = ( "New GWR advance ticket booking date: " + f'{new_dates["Weekdays"].strftime("%d %b %Y")} (Weekdays)' ) body = f""" {"\n".join(f'{key}: {when.strftime("%d %b %Y")}' for key, when in new_dates.items())} {agenda.gwr.url} Agenda: https://edwardbetts.com/agenda/ """ if sys.stdin.isatty(): print(filename) print(agenda.gwr.url) print() print("dates have changed") print("old:", existing_dates) print("new:", new_dates) print() print(subject) print(body) agenda.mail.send_mail(config, subject, body) def format_launch_changes(differences: StrDict) -> str: """Convert deepdiff output to human-readable format.""" changes = [] # Handle value changes if "values_changed" in differences: for path, change in differences["values_changed"].items(): field = path.replace("root['", "").replace("']", "").replace("root.", "") old_val = change["old_value"] new_val = change["new_value"] # Format specific fields nicely if field == "net": try: old_dt = datetime.fromisoformat(old_val.replace("Z", "+00:00")) new_dt = datetime.fromisoformat(new_val.replace("Z", "+00:00")) changes.append( "Launch time changed from " + old_dt.strftime("%d %b %Y at %H:%M UTC") + " to " + new_dt.strftime("%d %b %Y at %H:%M UTC") ) except: changes.append(f"Launch time changed from {old_val} to {new_val}") elif field == "name": changes.append(f"Mission name changed from '{old_val}' to '{new_val}'") elif field == "probability": if old_val is None: changes.append(f"Launch probability set to {new_val}%") elif new_val is None: changes.append("Launch probability removed") else: changes.append( f"Launch probability changed from {old_val}% to {new_val}%" ) elif "status" in field: changes.append(f"Status changed from '{old_val}' to '{new_val}'") else: changes.append( f"{field.replace('_', ' ').title()} changed " + f"from '{old_val}' to '{new_val}'" ) # Handle additions if "dictionary_item_added" in differences: for path in differences["dictionary_item_added"]: field = path.replace("root['", "").replace("']", "").replace("root.", "") changes.append(f"New field added: {field.replace('_', ' ').title()}") # Handle removals if "dictionary_item_removed" in differences: for path in differences["dictionary_item_removed"]: field = path.replace("root['", "").replace("']", "").replace("root.", "") changes.append(f"Field removed: {field.replace('_', ' ').title()}") # Handle type changes if "type_changes" in differences: for path, change in differences["type_changes"].items(): field = path.replace("root['", "").replace("']", "").replace("root.", "") changes.append( f"{field.replace('_', ' ').title()} type changed " + f"from {change['old_type'].__name__} to {change['new_type'].__name__}" ) return ( "\n".join(f"β€’ {change}" for change in changes) if changes else "No specific changes detected" ) def report_space_launch_change( config: flask.config.Config, prev_launch: StrDict | None, cur_launch: StrDict | None ) -> None: """Send mail to announce change to space launch data.""" # Handle case where launch disappeared from upcoming list if prev_launch and not cur_launch: # Launch is no longer in upcoming list - could be completed, cancelled, or failed # Check if we can determine status from previous data prev_status_id = ( prev_launch.get("status", {}).get("id", 0) if isinstance(prev_launch.get("status"), dict) else 0 ) name = prev_launch["name"] launch_date = prev_launch.get("net", "Unknown") location = ( prev_launch.get("pad", {}).get("location", {}).get("name", "Unknown") if isinstance(prev_launch.get("pad"), dict) else "Unknown" ) # Since launch is no longer in upcoming list, it likely completed # We can't know the exact outcome, so provide helpful message subject = f"πŸš€ Space Launch Completed: {name}" # Format launch date nicely formatted_date = "Unknown" if launch_date and launch_date != "Unknown": try: dt = datetime.fromisoformat(launch_date.replace("Z", "+00:00")) formatted_date = dt.strftime("%d %b %Y at %H:%M UTC") except: formatted_date = launch_date body = f"""πŸš€ Space Launch Completed Mission: {name} Launch Date: {formatted_date} Location: {location} This launch is no longer appearing in the upcoming launches list, which typically means it has taken place. To check if the launch was successful or failed, visit: https://edwardbetts.com/agenda/launches View all launches: https://edwardbetts.com/agenda/launches """ agenda.mail.send_mail(config, subject, body) return # Handle regular status updates if cur_launch: name = cur_launch["name"] status = ( cur_launch.get("status", {}).get("name", "Unknown") if isinstance(cur_launch.get("status"), dict) else "Unknown" ) status_id = ( cur_launch.get("status", {}).get("id", 0) if isinstance(cur_launch.get("status"), dict) else 0 ) launch_date = cur_launch.get("net", "Unknown") location = ( cur_launch.get("pad", {}).get("location", {}).get("name", "Unknown") if isinstance(cur_launch.get("pad"), dict) else "Unknown" ) # Check for specific status changes that deserve special attention prev_status_id = 0 if prev_launch and isinstance(prev_launch.get("status"), dict): prev_status_id = prev_launch.get("status", {}).get("id", 0) # Customize subject based on status changes if status_id == 3: # Launch Successful subject = f"πŸŽ‰ Launch Successful: {name}" elif status_id == 4: # Launch Failure subject = f"πŸ’₯ Launch Failed: {name}" elif status_id == 7: # Partial Failure subject = f"⚠️ Launch Partial Failure: {name}" elif status_id == 6: # In Flight subject = f"πŸš€ Launch In Flight: {name}" elif status_id == 5: # On Hold subject = f"⏸️ Launch On Hold: {name}" else: subject = f"Space Launch Update: {name}" else: # This shouldn't happen with the new logic above, but keep as fallback assert prev_launch name = prev_launch["name"] status = "Unknown" launch_date = prev_launch.get("net", "Unknown") location = ( prev_launch.get("pad", {}).get("location", {}).get("name", "Unknown") if isinstance(prev_launch.get("pad"), dict) else "Unknown" ) subject = f"Space Launch Update: {name}" differences = deepdiff.DeepDiff(prev_launch, cur_launch) changes_text = format_launch_changes(differences) # Format launch date nicely formatted_date = "Unknown" if launch_date and launch_date != "Unknown": try: dt = datetime.fromisoformat(launch_date.replace("Z", "+00:00")) formatted_date = dt.strftime("%d %b %Y at %H:%M UTC") except: formatted_date = launch_date body = f"""πŸš€ Space Launch Update Mission: {name} Status: {status} Launch Date: {formatted_date} Location: {location} Changes: {changes_text} View all launches: https://edwardbetts.com/agenda/launches """ agenda.mail.send_mail(config, subject, body) def get_launch_by_slug(data: StrDict, slug: str) -> StrDict | None: """Find last update for space launch.""" return {item["slug"]: typing.cast(StrDict, item) for item in data["results"]}.get( slug ) def update_thespacedevs(config: flask.config.Config) -> None: """Update cache of space launch API.""" rocket_dir = os.path.join(config["DATA_DIR"], "thespacedevs") existing_data = agenda.thespacedevs.load_cached_launches(rocket_dir) assert existing_data prev_launches = { slug: get_launch_by_slug(existing_data, slug) for slug in config["FOLLOW_LAUNCHES"] } t0 = time() data = agenda.thespacedevs.next_launch_api_data(rocket_dir) if not data: return # thespacedevs API call failed cur_launches = { slug: get_launch_by_slug(data, slug) for slug in config["FOLLOW_LAUNCHES"] } for slug in config["FOLLOW_LAUNCHES"]: prev, cur = prev_launches[slug], cur_launches[slug] if prev is None and cur is None: continue if prev and cur and prev["last_updated"] == cur["last_updated"]: continue report_space_launch_change(config, prev, cur) time_taken = time() - t0 if not sys.stdin.isatty(): return rockets = [agenda.thespacedevs.summarize_launch(item) for item in data["results"]] print(len(rockets), "launches") print(f"took {time_taken:.1f} seconds") def update_gandi(config: flask.config.Config) -> None: """Retrieve list of domains from gandi.net.""" url = "https://api.gandi.net/v5/domain/domains" headers = {"authorization": "Bearer " + config["GANDI_TOKEN"]} filename = os.path.join(config["DATA_DIR"], "gandi_domains.json") r = requests.request("GET", url, headers=headers) items = r.json() assert isinstance(items, list) assert all(item["fqdn"] and item["dates"]["registry_ends_at"] for item in items) with open(filename, "w") as out: out.write(r.text) def main() -> None: """Update caches.""" now = datetime.now() hour = now.hour with app.app_context(): if hour % 3 == 0: asyncio.run(update_bank_holidays(app.config)) asyncio.run(update_bristol_bins(app.config)) update_gwr_advance_ticket_date(app.config) # TODO: debug why update gandi fails # update_gandi(app.config) agenda.geomob.update(app.config) agenda.fx.get_rates(app.config) update_thespacedevs(app.config) if __name__ == "__main__": main()