From 2ea79c12d9fe3a350a97d2cefef22c2e75c74fd5 Mon Sep 17 00:00:00 2001
From: Edward Betts <edward@4angle.com>
Date: Mon, 6 Nov 2023 08:40:22 +0000
Subject: [PATCH] Bug fix waste collection HTTP requests
---
agenda/waste_schedule.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/agenda/waste_schedule.py b/agenda/waste_schedule.py
index 5e63423..e9b9994 100644
--- a/agenda/waste_schedule.py
+++ b/agenda/waste_schedule.py
@@ -2,6 +2,7 @@
import json
import os
+import re
import typing
from collections import defaultdict
from datetime import date, datetime, timedelta
@@ -41,9 +42,12 @@ async def get_html(data_dir: str, postcode: str, uprn: str) -> str:
now_str = now.strftime("%Y-%m-%d_%H:%M")
filename = f"{waste_dir}/{now_str}.html"
+ forms_base_url = "https://forms.n-somerset.gov.uk"
+ # url2 = "https://forms.n-somerset.gov.uk/Waste/CollectionSchedule/ViewSchedule"
+ url = "https://forms.n-somerset.gov.uk/Waste/CollectionSchedule"
async with httpx.AsyncClient() as client:
r = await client.post(
- "https://forms.n-somerset.gov.uk/Waste/CollectionSchedule",
+ url,
data={
"PreviousHouse": "",
"PreviousPostcode": "-",
@@ -51,6 +55,11 @@ async def get_html(data_dir: str, postcode: str, uprn: str) -> str:
"SelectedUprn": uprn,
},
)
+ form_post_html = r.text
+ pattern = r'<h2>Object moved to <a href="([^"]*)">here<\/a>\.<\/h2>'
+ m = re.search(pattern, form_post_html)
+ if m:
+ r = await client.get(forms_base_url + m.group(1))
html = r.text
open(filename, "w").write(html)
return html
@@ -139,7 +148,7 @@ async def get_bristol_gov_uk_data(uprn: str) -> httpx.Response:
_uprn = str(uprn).zfill(12)
- async with httpx.AsyncClient() as client:
+ async with httpx.AsyncClient(timeout=10) as client:
# Initialise form
payload = {"servicetypeid": "7dce896c-b3ba-ea11-a812-000d3a7f1cdc"}
response = await client.get(