From d4308685f7ea2660d890a3d7ff222ae38bbc91e2 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Fri, 3 Apr 2026 15:32:38 +0100 Subject: [PATCH] Fix Playwright timeout on slow servers wait_until="networkidle" requires zero network activity for 500ms, which times out on ad-heavy pages when running on a server. Switch to domcontentloaded and wait explicitly for div.item-container instead. Co-Authored-By: Claude Sonnet 4.6 --- crawl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crawl.py b/crawl.py index e397a9f..daf36d5 100755 --- a/crawl.py +++ b/crawl.py @@ -74,7 +74,8 @@ def get_product_list(n: str, page: Optional[int] = None) -> str: "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" ) pg = context.new_page() - pg.goto(url, wait_until="networkidle", timeout=60000) + pg.goto(url, wait_until="domcontentloaded", timeout=60000) + pg.wait_for_selector("div.item-container", timeout=60000) if "areyouahuman" in pg.url: logger.info("bot detection triggered, simulating mouse movement...")