Bug fix waste collection HTTP requests
This commit is contained in:
parent
125eb245e9
commit
2ea79c12d9
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import typing
|
import typing
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from datetime import date, datetime, timedelta
|
from datetime import date, datetime, timedelta
|
||||||
|
@ -41,9 +42,12 @@ async def get_html(data_dir: str, postcode: str, uprn: str) -> str:
|
||||||
now_str = now.strftime("%Y-%m-%d_%H:%M")
|
now_str = now.strftime("%Y-%m-%d_%H:%M")
|
||||||
filename = f"{waste_dir}/{now_str}.html"
|
filename = f"{waste_dir}/{now_str}.html"
|
||||||
|
|
||||||
|
forms_base_url = "https://forms.n-somerset.gov.uk"
|
||||||
|
# url2 = "https://forms.n-somerset.gov.uk/Waste/CollectionSchedule/ViewSchedule"
|
||||||
|
url = "https://forms.n-somerset.gov.uk/Waste/CollectionSchedule"
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
r = await client.post(
|
r = await client.post(
|
||||||
"https://forms.n-somerset.gov.uk/Waste/CollectionSchedule",
|
url,
|
||||||
data={
|
data={
|
||||||
"PreviousHouse": "",
|
"PreviousHouse": "",
|
||||||
"PreviousPostcode": "-",
|
"PreviousPostcode": "-",
|
||||||
|
@ -51,6 +55,11 @@ async def get_html(data_dir: str, postcode: str, uprn: str) -> str:
|
||||||
"SelectedUprn": uprn,
|
"SelectedUprn": uprn,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
form_post_html = r.text
|
||||||
|
pattern = r'<h2>Object moved to <a href="([^"]*)">here<\/a>\.<\/h2>'
|
||||||
|
m = re.search(pattern, form_post_html)
|
||||||
|
if m:
|
||||||
|
r = await client.get(forms_base_url + m.group(1))
|
||||||
html = r.text
|
html = r.text
|
||||||
open(filename, "w").write(html)
|
open(filename, "w").write(html)
|
||||||
return html
|
return html
|
||||||
|
@ -139,7 +148,7 @@ async def get_bristol_gov_uk_data(uprn: str) -> httpx.Response:
|
||||||
|
|
||||||
_uprn = str(uprn).zfill(12)
|
_uprn = str(uprn).zfill(12)
|
||||||
|
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient(timeout=10) as client:
|
||||||
# Initialise form
|
# Initialise form
|
||||||
payload = {"servicetypeid": "7dce896c-b3ba-ea11-a812-000d3a7f1cdc"}
|
payload = {"servicetypeid": "7dce896c-b3ba-ea11-a812-000d3a7f1cdc"}
|
||||||
response = await client.get(
|
response = await client.get(
|
||||||
|
|
Loading…
Reference in a new issue