diff --git a/crawl.py b/crawl.py index e397a9f..daf36d5 100755 --- a/crawl.py +++ b/crawl.py @@ -74,7 +74,8 @@ def get_product_list(n: str, page: Optional[int] = None) -> str: "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" ) pg = context.new_page() - pg.goto(url, wait_until="networkidle", timeout=60000) + pg.goto(url, wait_until="domcontentloaded", timeout=60000) + pg.wait_for_selector("div.item-container", timeout=60000) if "areyouahuman" in pg.url: logger.info("bot detection triggered, simulating mouse movement...")