From bc31db61436df8de0d4d5aad0d75f9db0839eb0f Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Sun, 5 Nov 2023 21:56:18 +0000 Subject: [PATCH] Make Bristol bin collection code async Closes: #54 --- agenda/__init__.py | 8 ++++-- agenda/waste_schedule.py | 60 ++++++++++++++++++++-------------------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/agenda/__init__.py b/agenda/__init__.py index 86bb2fb..4028a4d 100644 --- a/agenda/__init__.py +++ b/agenda/__init__.py @@ -301,11 +301,11 @@ async def waste_collection_events() -> list[Event]: return events -def bristol_waste_collection_events(start_date: date) -> list[Event]: +async def bristol_waste_collection_events(start_date: date) -> list[Event]: """Waste colllection events.""" uprn = "358335" - return waste_schedule.get_bristol_gov_uk(start_date, data_dir, uprn) + return await waste_schedule.get_bristol_gov_uk(start_date, data_dir, uprn) async def get_data(now: datetime) -> typing.Mapping[str, str | object]: @@ -325,12 +325,14 @@ async def get_data(now: datetime) -> typing.Mapping[str, str | object]: bank_holiday, rockets, backwell_bins, + bristol_bins, ) = await asyncio.gather( fx.get_gbpusd(config), gwr.advance_ticket_date(data_dir), get_next_bank_holiday(last_year, next_year), thespacedevs.get_launches(rocket_dir, limit=40), waste_collection_events(), + bristol_waste_collection_events(today), ) reply = { @@ -385,7 +387,7 @@ async def get_data(now: datetime) -> typing.Mapping[str, str | object]: events += get_accommodation(os.path.join(my_data, "accommodation.yaml")) events += travel.all_events(today, config["data"]["personal-data"]) events += conference.get_list(os.path.join(my_data, "conferences.yaml")) - events += backwell_bins + bristol_waste_collection_events(today) + events += backwell_bins + bristol_bins next_up_series = Event( date=date(2026, 6, 1), diff --git a/agenda/waste_schedule.py b/agenda/waste_schedule.py index 62c965c..5e63423 100644 --- a/agenda/waste_schedule.py +++ b/agenda/waste_schedule.py @@ -6,9 +6,8 @@ import typing from collections import defaultdict from datetime import date, datetime, timedelta -import lxml.html -import requests import httpx +import lxml.html from .types import Event @@ -91,7 +90,7 @@ def parse(root: lxml.html.HtmlElement) -> list[Event]: BristolSchedule = list[dict[str, typing.Any]] -def get_bristol_data(data_dir: str, uprn: str) -> BristolSchedule: +async def get_bristol_data(data_dir: str, uprn: str) -> BristolSchedule: """Get Bristol Waste schedule, with cache.""" now = datetime.now() waste_dir = os.path.join(data_dir, "waste") @@ -112,7 +111,7 @@ def get_bristol_data(data_dir: str, uprn: str) -> BristolSchedule: now_str = now.strftime("%Y-%m-%d_%H:%M") filename = f"{waste_dir}/{now_str}_{uprn}.json" - r = get_bristol_gov_uk_data(uprn) + r = await get_bristol_gov_uk_data(uprn) with open(filename, "wb") as out: out.write(r.content) @@ -120,7 +119,7 @@ def get_bristol_data(data_dir: str, uprn: str) -> BristolSchedule: return typing.cast(BristolSchedule, r.json()["data"]) -def get_bristol_gov_uk_data(uprn: str) -> requests.Response: +async def get_bristol_gov_uk_data(uprn: str) -> httpx.Response: """Get JSON from Bristol City Council.""" UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" HEADERS = { @@ -139,39 +138,40 @@ def get_bristol_gov_uk_data(uprn: str) -> requests.Response: } _uprn = str(uprn).zfill(12) - s = requests.Session() - # Initialise form - payload = {"servicetypeid": "7dce896c-b3ba-ea11-a812-000d3a7f1cdc"} - response = s.get( - "https://bristolcouncil.powerappsportals.com/completedynamicformunauth/", - headers=HEADERS, - params=payload, - ) + async with httpx.AsyncClient() as client: + # Initialise form + payload = {"servicetypeid": "7dce896c-b3ba-ea11-a812-000d3a7f1cdc"} + response = await client.get( + "https://bristolcouncil.powerappsportals.com/completedynamicformunauth/", + headers=HEADERS, + params=payload, + ) - host = "bcprdapidyna002.azure-api.net" + host = "bcprdapidyna002.azure-api.net" - # Set the search criteria - payload = {"Uprn": "UPRN" + _uprn} - response = s.post( - f"https://{host}/bcprdfundyna001-llpg/DetailedLLPG", - headers=HEADERS, - json=payload, - ) + # Set the search criteria + payload = {"Uprn": "UPRN" + _uprn} + response = await client.post( + f"https://{host}/bcprdfundyna001-llpg/DetailedLLPG", + headers=HEADERS, + json=payload, + ) + + # Retrieve the schedule + payload = {"uprn": _uprn} + response = await client.post( + f"https://{host}/bcprdfundyna001-alloy/NextCollectionDates", + headers=HEADERS, + json=payload, + ) - # Retrieve the schedule - payload = {"uprn": _uprn} - response = s.post( - f"https://{host}/bcprdfundyna001-alloy/NextCollectionDates", - headers=HEADERS, - json=payload, - ) return response -def get_bristol_gov_uk(start_date: date, data_dir: str, uprn: str) -> list[Event]: +async def get_bristol_gov_uk(start_date: date, data_dir: str, uprn: str) -> list[Event]: """Get waste collection schedule from Bristol City Council.""" - data = get_bristol_data(data_dir, uprn) + data = await get_bristol_data(data_dir, uprn) by_date: defaultdict[date, list[str]] = defaultdict(list)