Compare commits

..

No commits in common. "6d1b01485a7a340b3d055641707000f43a5a5ed7" and "3c3939c5253167f3e030aa823daea2f6d9c8b8aa" have entirely different histories.

5 changed files with 14 additions and 54 deletions

View file

@ -17,10 +17,12 @@ ttl_hours = 12
BristolSchedule = list[dict[str, typing.Any]] BristolSchedule = list[dict[str, typing.Any]]
async def get(start_date: date, data_dir: str, uprn: str, cache: str) -> list[Event]: async def get(
start_date: date, data_dir: str, uprn: str, refresh: bool = False
) -> list[Event]:
"""Get waste collection schedule from Bristol City Council.""" """Get waste collection schedule from Bristol City Council."""
by_date: defaultdict[date, list[str]] = defaultdict(list) by_date: defaultdict[date, list[str]] = defaultdict(list)
for item in await get_data(data_dir, uprn, cache): for item in await get_data(data_dir, uprn, refresh):
service = get_service(item) service = get_service(item)
for d in collections(item): for d in collections(item):
if d < start_date and service not in by_date[d]: if d < start_date and service not in by_date[d]:
@ -32,7 +34,7 @@ async def get(start_date: date, data_dir: str, uprn: str, cache: str) -> list[Ev
] ]
async def get_data(data_dir: str, uprn: str, cache: str) -> BristolSchedule: async def get_data(data_dir: str, uprn: str, refresh: bool = False) -> BristolSchedule:
"""Get Bristol Waste schedule, with cache.""" """Get Bristol Waste schedule, with cache."""
now = datetime.now() now = datetime.now()
waste_dir = os.path.join(data_dir, "waste") waste_dir = os.path.join(data_dir, "waste")
@ -50,11 +52,7 @@ async def get_data(data_dir: str, uprn: str, cache: str) -> BristolSchedule:
json_data = json.load(open(os.path.join(waste_dir, recent_filename))) json_data = json.load(open(os.path.join(waste_dir, recent_filename)))
return typing.cast(BristolSchedule, json_data["data"]) return typing.cast(BristolSchedule, json_data["data"])
if ( if not refresh and existing and delta < timedelta(hours=ttl_hours):
cache != "refresh"
and existing
and (cache == "force" or delta < timedelta(hours=ttl_hours))
):
return get_from_recent() return get_from_recent()
try: try:

View file

@ -62,21 +62,20 @@ def timezone_transition(
async def n_somerset_waste_collection_events( async def n_somerset_waste_collection_events(
data_dir: str, postcode: str, uprn: str, force_cache: bool = False data_dir: str, postcode: str, uprn: str
) -> list[Event]: ) -> list[Event]:
"""Waste colllection events.""" """Waste colllection events."""
html = await n_somerset_waste.get_html(data_dir, postcode, uprn, force_cache) html = await n_somerset_waste.get_html(data_dir, postcode, uprn)
root = lxml.html.fromstring(html) root = lxml.html.fromstring(html)
events = n_somerset_waste.parse(root) events = n_somerset_waste.parse(root)
return events return events
async def bristol_waste_collection_events( async def bristol_waste_collection_events(
data_dir: str, start_date: date, uprn: str, force_cache: bool = False data_dir: str, start_date: date, uprn: str
) -> list[Event]: ) -> list[Event]:
"""Waste colllection events.""" """Waste colllection events."""
cache = "force" if force_cache else "recent" return await bristol_waste.get(start_date, data_dir, uprn)
return await bristol_waste.get(start_date, data_dir, uprn, cache)
def find_events_during_stay( def find_events_during_stay(
@ -194,7 +193,6 @@ async def get_data(now: datetime, config: flask.config.Config) -> AgendaData:
data_dir, data_dir,
config["BACKWELL_POSTCODE"], config["BACKWELL_POSTCODE"],
config["BACKWELL_UPRN"], config["BACKWELL_UPRN"],
offline_mode,
), ),
time_function( time_function(
"bristol_bins", "bristol_bins",
@ -202,7 +200,6 @@ async def get_data(now: datetime, config: flask.config.Config) -> AgendaData:
data_dir, data_dir,
today, today,
config["BRISTOL_UPRN"], config["BRISTOL_UPRN"],
offline_mode,
), ),
) )
rockets = thespacedevs.read_cached_launches(rocket_dir) rockets = thespacedevs.read_cached_launches(rocket_dir)

View file

@ -15,9 +15,7 @@ from .utils import make_waste_dir
ttl_hours = 12 ttl_hours = 12
async def get_html( async def get_html(data_dir: str, postcode: str, uprn: str) -> str:
data_dir: str, postcode: str, uprn: str, force_cache: bool = False
) -> str:
"""Get waste schedule.""" """Get waste schedule."""
now = datetime.now() now = datetime.now()
waste_dir = os.path.join(data_dir, "waste") waste_dir = os.path.join(data_dir, "waste")
@ -31,7 +29,7 @@ async def get_html(
recent = datetime.strptime(recent_filename, "%Y-%m-%d_%H:%M.html") recent = datetime.strptime(recent_filename, "%Y-%m-%d_%H:%M.html")
delta = now - recent delta = now - recent
if existing and (force_cache or delta < timedelta(hours=ttl_hours)): if existing and delta < timedelta(hours=ttl_hours):
return open(os.path.join(waste_dir, recent_filename)).read() return open(os.path.join(waste_dir, recent_filename)).read()
now_str = now.strftime("%Y-%m-%d_%H:%M") now_str = now.strftime("%Y-%m-%d_%H:%M")

View file

@ -40,10 +40,7 @@ async def update_bristol_bins(config: flask.config.Config) -> None:
"""Update waste schedule from Bristol City Council.""" """Update waste schedule from Bristol City Council."""
t0 = time() t0 = time()
events = await agenda.bristol_waste.get( events = await agenda.bristol_waste.get(
date.today(), date.today(), config["DATA_DIR"], config["BRISTOL_UPRN"], refresh=True
config["DATA_DIR"],
config["BRISTOL_UPRN"],
cache="refresh",
) )
time_taken = time() - t0 time_taken = time() - t0
if not sys.stdin.isatty(): if not sys.stdin.isatty():

View file

@ -2,7 +2,6 @@
"""Load YAML data to ensure validity.""" """Load YAML data to ensure validity."""
import os import os
import sys
import typing import typing
from datetime import date, timedelta from datetime import date, timedelta
@ -19,18 +18,6 @@ import agenda.types
config = __import__("config.default", fromlist=[""]) config = __import__("config.default", fromlist=[""])
data_dir = config.PERSONAL_DATA data_dir = config.PERSONAL_DATA
currencies = set(config.CURRENCIES + ["GBP"])
def check_currency(item: agenda.types.StrDict) -> None:
"""Throw error if currency is not in config."""
currency = item.get("currency")
if not currency or currency in currencies:
return None
pprint(item)
print(f"currency {currency!r} not in {currencies!r}")
sys.exit(-1)
def check_trips() -> None: def check_trips() -> None:
"""Check trips.""" """Check trips."""
@ -47,10 +34,6 @@ def check_flights(airlines: set[str]) -> None:
bookings = agenda.travel.parse_yaml("flights", data_dir) bookings = agenda.travel.parse_yaml("flights", data_dir)
for booking in bookings: for booking in bookings:
assert all(flight["airline"] in airlines for flight in booking["flights"]) assert all(flight["airline"] in airlines for flight in booking["flights"])
for booking in bookings:
check_currency(booking)
print(len(bookings), "flights") print(len(bookings), "flights")
@ -62,18 +45,7 @@ def check_trains() -> None:
def check_conferences() -> None: def check_conferences() -> None:
"""Check conferences.""" """Check conferences."""
filepath = os.path.join(data_dir, "conferences.yaml") conferences = agenda.conference.get_list(os.path.join(data_dir, "conferences.yaml"))
conferences = [
agenda.conference.Conference(**conf)
for conf in yaml.safe_load(open(filepath, "r"))
]
for conf in conferences:
if not conf.currency or conf.currency in currencies:
continue
pprint(conf)
print(f"currency {conf.currency!r} not in {currencies!r}")
sys.exit(-1)
print(len(conferences), "conferences") print(len(conferences), "conferences")
@ -110,8 +82,6 @@ def check_accommodation() -> None:
pprint(stay) pprint(stay)
raise raise
check_currency(stay)
print(len(accommodation_list), "stays") print(len(accommodation_list), "stays")