edward/eurotunnel-scrape
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Initial commit.

Browse source
This commit is contained in:
Edward Betts 2023-08-08 16:17:50 +01:00
commit cd7c1f1e50
3 changed files with 294 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

137
check.py Executable file
View file

@ -0,0 +1,137 @@
#!/usr/bin/python3
import os
from datetime import datetime
from playwright.sync_api import Playwright, Request, expect, sync_playwright
from rich.pretty import pprint
skip_domains = {
"cdn.linkedin.oribi.io",
"cdn.evgnet.com",
"zdassets.com",
"doubleclick.net",
"linkedin.com",
"facebook",
"adservice.google.com",
"google",
"bing",
"eurotunnel.report-uri.com",
}
skip_content_type = {"image", "text/css", "application/x-woff", "text/javascript"}
data_loc = os.path.expanduser("~/lib/data/eurotunnel")
outbound_label = "Select your outbound tickets from Folkestone to Calais"
return_label = "Select your Return tickets from Calais to Folkestone"
choose_your_tickets = "https://www.eurotunnel.com/book/ChooseYourTickets/(0)"
def data_filename(page_type: str, ext: str = "html") -> str:
"""Filename to use for saving data."""
now_str = datetime.utcnow().strftime("%Y-%m-%d_%H%M%S")
return os.path.join(data_loc, now_str + f"_{page_type}.{ext}")
class HandleResponse:
got_outbound: bool = False
def requestfinished(self, request: Request) -> None:
"""Show details of finished request."""
if any(d in request.url for d in skip_domains):
return
if "tunnel" not in request.url:
return
response = request.response()
reply_ct = response.headers.get("content-type")
if reply_ct and any(reply_ct.startswith(ct) for ct in skip_content_type):
return
url = request.url
show = {
"url": url,
"method": request.method,
"request_headers": request.headers,
"response_headers": response.headers,
}
if request.method == "POST":
show["post_data"] = request.post_data
pprint(show)
if not request.method == "GET" or not url.startswith(choose_your_tickets):
return
if url == choose_your_tickets + "?d=o":
filename = data_filename("outbound")
with open(filename, "wb") as out:
out.write(response.body())
if url == choose_your_tickets + "?d=r":
filename = data_filename("return")
with open(filename, "wb") as out:
out.write(response.body())
def run(playwright: Playwright) -> None:
"""Launch browser and search for options."""
browser = playwright.chromium.launch(headless=False)
context = browser.new_context()
page = context.new_page()
hr = HandleResponse()
page.on("requestfinished", hr.requestfinished)
page.goto("https://www.eurotunnel.com/uk/")
page.get_by_role("button", name="Accept All Cookies").click()
page.get_by_role("textbox", name="Outbound").click()
page.get_by_role("row", name="August 2023").get_by_role("cell").nth(1).click()
page.get_by_role("cell", name="29").click()
page.get_by_role("textbox", name="Return").click()
page.get_by_role("row", name="September 2023").get_by_role("cell").nth(2).click()
page.get_by_role("cell", name="6", exact=True).click()
page.get_by_role("button", name="Search and book").click()
page.get_by_label("Enter your vehicle registration").fill("KE69HRR")
page.get_by_role("button", name="Find").click()
page.get_by_text("No", exact=True).first.click()
page.get_by_text("No", exact=True).nth(1).click()
page.get_by_text("Conventional (Diesel/Petrol)").click()
page.get_by_role("combobox", name="Country of residence").select_option("GB")
page.get_by_role("button", name="Continue").click()
expect(page.get_by_text(outbound_label)).to_be_visible()
filename = data_filename("outbound_prices")
with open(filename, "w") as out:
out.write(page.content())
page.locator("#slots div").filter(has_text="12:00To").locator("span").click()
page.locator("div:nth-child(5) > .times > div:nth-child(2) > .radio-button").click()
page.locator(
"div:nth-child(17) > .d-flex > div > .body > .radio-button"
).first.click()
page.get_by_role("button", name="Confirm and choose return ticket").click()
expect(page.get_by_text(return_label)).to_be_visible()
filename = data_filename("return_prices")
with open(filename, "w") as out:
out.write(page.content())
page.close()
context.close()
browser.close()
with sync_playwright() as playwright:
run(playwright)

61
templates/index.html Normal file
View file

@ -0,0 +1,61 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title></title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-4bw+/aepP/YC94hEpVNVgiZdgIC5+VKNBQNGCHeKRQN+PtmoHDEXuppvnDJzQIu9" crossorigin="anonymous">
</head>
{% macro time(t) %}{{ t[:2] + ":" + t[2:] }}{% endmacro %}
<body>
<div class="container mt-3">
<p>All Flexi Long Stay tickets are £269</p>
<p>Pay an extra £40 to make a price return refundable.</p>
<div class="row">
<div class="col">
<p>Updated: {{ out_ts.strftime("%a, %-d %b %Y at %H:%M") }}</p>
<table class="table table-sm w-auto">
<tr>
<th>Depart</th>
<th>Arrive</th>
<th class="text-end">Price</th>
</tr>
{% for t in out %}
<tr>
<td class="text-end">{{ time(t.dep) }}</td>
<td class="text-end">{{ time(t.arr) }}</td>
<td class="text-end">£{{ t.price }}</td>
</tr>
{% endfor %}
</table>
</div>
<div class="col">
<p>Updated: {{ back_ts.strftime("%a, %-d %b %Y at %H:%M") }}</p>
<table class="table table-sm w-auto">
<tr>
<th>Depart</th>
<th>Arrive</th>
<th class="text-end">Price</th>
</tr>
{% for t in back %}
<tr>
<td class="text-end">{{ time(t.dep) }}</td>
<td class="text-end">{{ time(t.arr) }}</td>
<td class="text-end">£{{ t.price }}</td>
</tr>
{% endfor %}
</table>
</div>
</div>
</div>
</body>
</html>

96
web_view.py Executable file
View file

@ -0,0 +1,96 @@
#!/usr/bin/python3
import os
from dataclasses import dataclass
from datetime import UTC, datetime
from decimal import Decimal
import flask
import lxml.html
import pytz
app = flask.Flask(__name__)
app.debug = True
data_loc = os.path.expanduser("~/lib/data/eurotunnel")
def get_filename(direction: str) -> tuple[datetime, str]:
"""Most recent list of outbound prices."""
end = f"_{direction}.html"
most_recent = max([f for f in os.listdir(data_loc) if f.endswith(end)])
filename = os.path.join(data_loc, most_recent)
timestamp = most_recent.removesuffix(end)
dt_utc = datetime.strptime(timestamp, "%Y-%m-%d_%H%M%S").replace(tzinfo=UTC)
dt = dt_utc.astimezone(pytz.timezone("Europe/London"))
return (dt, filename)
@dataclass
class Train:
"""Eurostar train."""
dep: str
arr: str
price: Decimal | None = None
def get_tickets(filename: str) -> list[Train]:
"""Get trains and prices."""
tree = lxml.html.parse(filename)
root = tree.getroot()
trains = []
by_time = {}
for mission in root.findall(".//div[@data-mission]"):
dep_text = mission.findtext(".//b")
assert dep_text
dep = dep_text.replace(":", "")
arr_text = mission.findtext(".//small")
assert arr_text
arr = arr_text.removeprefix("Arrive ").replace(":", "")
item = Train(dep=dep, arr=arr)
trains.append(item)
by_time[dep] = item
for ticket in root.xpath("//div[contains(@class, 'ticket')]"):
classes = ticket.get("class").split(" ")
if "ticket" not in classes:
continue
ticket_class = classes[-1]
if ticket_class != "standard":
continue
mission_time = ticket.getparent().getparent().get("data-mission-time")
onclick = ticket.get("onclick").split(",")
price = Decimal(onclick[2][1:-1])
by_time[mission_time].price = price
return trains
@app.route("/")
def index() -> str:
"""Index."""
out_ts, out_filename = get_filename("outbound")
out = get_tickets(out_filename)
out = [t for t in out if t.dep > "0800" and "0700" < t.arr < "2200"]
back_ts, back_filename = get_filename("return")
back = get_tickets(back_filename)
back = [t for t in back if t.dep > "1100" and "0700" < t.arr < "2200"]
return flask.render_template(
"index.html",
out_ts=out_ts,
out=out,
back_ts=back_ts,
back=back,
)
if __name__ == "__main__":
app.run(host="0.0.0.0")