edward/eurotunnel-scrape
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Simplify checker.

Browse source
This commit is contained in:
Edward Betts 2023-08-08 18:57:19 +01:00
parent cd7c1f1e50
commit 036c0cfbd8
1 changed files with 3 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
check.py
View file

@ -4,22 +4,6 @@ import os
from datetime import datetime from datetime import datetime
from playwright.sync_api import Playwright, Request, expect, sync_playwright from playwright.sync_api import Playwright, Request, expect, sync_playwright
from rich.pretty import pprint
skip_domains = {
"cdn.linkedin.oribi.io",
"cdn.evgnet.com",
"zdassets.com",
"doubleclick.net",
"linkedin.com",
"facebook",
"adservice.google.com",
"google",
"bing",
"eurotunnel.report-uri.com",
}
skip_content_type = {"image", "text/css", "application/x-woff", "text/javascript"}
data_loc = os.path.expanduser("~/lib/data/eurotunnel") data_loc = os.path.expanduser("~/lib/data/eurotunnel")
@ -41,32 +25,13 @@ class HandleResponse:
def requestfinished(self, request: Request) -> None: def requestfinished(self, request: Request) -> None:
"""Show details of finished request.""" """Show details of finished request."""
if any(d in request.url for d in skip_domains):
return
if "tunnel" not in request.url:
return
response = request.response()
reply_ct = response.headers.get("content-type")
if reply_ct and any(reply_ct.startswith(ct) for ct in skip_content_type):
return
url = request.url url = request.url
show = {
"url": url,
"method": request.method,
"request_headers": request.headers,
"response_headers": response.headers,
}
if request.method == "POST":
show["post_data"] = request.post_data
pprint(show)
if not request.method == "GET" or not url.startswith(choose_your_tickets): if not request.method == "GET" or not url.startswith(choose_your_tickets):
return return
response = request.response()
if url == choose_your_tickets + "?d=o": if url == choose_your_tickets + "?d=o":
filename = data_filename("outbound") filename = data_filename("outbound")
@ -82,7 +47,7 @@ class HandleResponse:
def run(playwright: Playwright) -> None: def run(playwright: Playwright) -> None:
"""Launch browser and search for options.""" """Launch browser and search for options."""
browser = playwright.chromium.launch(headless=False) browser = playwright.chromium.launch(headless=True)
context = browser.new_context() context = browser.new_context()
page = context.new_page() page = context.new_page()
@ -108,11 +73,6 @@ def run(playwright: Playwright) -> None:
expect(page.get_by_text(outbound_label)).to_be_visible() expect(page.get_by_text(outbound_label)).to_be_visible()
filename = data_filename("outbound_prices")
with open(filename, "w") as out:
out.write(page.content())
page.locator("#slots div").filter(has_text="12:00To").locator("span").click() page.locator("#slots div").filter(has_text="12:00To").locator("span").click()
page.locator("div:nth-child(5) > .times > div:nth-child(2) > .radio-button").click() page.locator("div:nth-child(5) > .times > div:nth-child(2) > .radio-button").click()
page.locator( page.locator(
@ -122,11 +82,6 @@ def run(playwright: Playwright) -> None:
expect(page.get_by_text(return_label)).to_be_visible() expect(page.get_by_text(return_label)).to_be_visible()
filename = data_filename("return_prices")
with open(filename, "w") as out:
out.write(page.content())
page.close() page.close()
context.close() context.close()