User cloudscraper

check.py

@@ -135,125 +135,35 @@ def normalize_url(url: str) -> str:
     return urlunparse(parsed_url._replace(netloc=normalized_netloc))
 
 
-def url_to_filename(url: str) -> str:
-    """
-    Convert a URL to a valid filename by replacing invalid characters with underscores.
+def check_conference(
+    name: str, src_url: str, year: int
+) -> tuple[bool, str, str | None]:
+    """Check if conference is live."""
+    url = src_url.format(year=year)
+    past_url = src_url.format(year=year - 1)
+    no_dot = {"bsideskbh.dk", "pif.camp"}
+    session = s if all(hostname not in url for hostname in no_dot) else s_no_dot
+    try:
+        r = session.get(url)
+    except requests.exceptions.ConnectionError:
+        return (False, "connection refused", None)
 
-    Args:
-        url (str): The URL to be converted.
+    not_here = find_not_here_message(r.text)
+    if (
+        len(r.text) < 2048
+        and 'http-equiv="refresh"' in r.text
+        and str(year) not in r.text
+    ):
+        return (False, "redirect to URL without year", r.url)
 
-    Returns:
-        str: A valid filename.
-    """
-    # Replace invalid characters with underscores
-    return re.sub(r'[\/:*?"<>|]', "_", url)
+    if normalize_url(r.url) == normalize_url(past_url):
+        return (False, "redirect to previous year", r.url)
 
-
-def parse_opengraph_tags(html: str) -> dict[str, str]:
-    """
-    Locate Open Graph meta tags, and return them as a dictionary.
-
-    Keys will match the name following 'og:', e.g. 'title', 'image', 'url', 'site_name'.
-    """
-    pattern = re.compile(
-        r'<meta\s+property="og:([^"]+)"\s+content="([^"]+)"\s*/?>', flags=re.IGNORECASE
-    )
-    matches = pattern.findall(html)
-
-    og_tags = {}
-    for prop, content in matches:
-        og_tags[prop] = content
-
-    return og_tags
-
-
-class Conference:
-    """Conference."""
-
-    name: str
-    src_url: str
-    year: int
-    response: None | requests.models.Response
-
-    def __init__(self, name: str, src_url: str, year: int):
-        """Init."""
-        self.name = name
-        self.src_url = src_url
-        self.year = year
-        self.response = None
-
-    @property
-    def url(self) -> str:
-        """Conference URL."""
-        return self.src_url.format(year=self.year)
-
-    @property
-    def past_url(self) -> str:
-        """URL for previous year."""
-        return self.src_url.format(year=self.year - 1)
-
-    def check(self) -> tuple[bool, str, str | None]:
-        """Check if conference is live."""
-        no_dot = {"bsideskbh.dk", "pif.camp"}
-        session = (
-            s if all(hostname not in self.url for hostname in no_dot) else s_no_dot
-        )
-        try:
-            r = session.get(self.url)
-        except requests.exceptions.ConnectionError:
-            return (False, "connection refused", None)
-
-        self.response = r
-
-        not_here = find_not_here_message(r.text)
-        if (
-            len(r.text) < 2048
-            and 'http-equiv="refresh"' in r.text
-            and str(self.year) not in r.text
-        ):
-            return (False, "redirect to URL without year", r.url)
-
-        if normalize_url(r.url) == normalize_url(self.past_url):
-            return (False, "redirect to previous year", r.url)
-
-        if not_here:
-            return (False, not_here, r.url)
-
-        return (True, get_title(r.text), r.url)
-
-    def og_tags(self) -> dict[str, str]:
-        """Open Graph tags."""
-        assert self.response
-        return parse_opengraph_tags(self.response.text)
-
-    def check_web_site(self) -> bool:
-        """Check if an individual web site is live."""
-        assert "{year}" in self.src_url
-        live, msg, redirect_to_url = self.check()
-        if not live:
-            return False
-
-        og = "".join(f"\n{key}: {value}" for key, value in self.og_tags().items())
-
-        if og:
-            og = "\n\nOpen Graph\n\n" + og
-
-        if redirect_to_url and normalize_url(redirect_to_url) == normalize_url(
-            self.url
-        ):
-            body = f"{self.name}\n{self.url}\n"
-        else:
-            body = f"{self.name}\n{self.url} redirects to {redirect_to_url}\n"
-
-        body += "Web page title: {msg}{og}" ""
-        send_mail(f"Conference site live: {self.name} - {self.year}", body)
-
-        return True
+    return (False, not_here, r.url) if not_here else (True, get_title(r.text), r.url)
 
 
 def send_mail(subject: str, body: str) -> None:
     """Send an e-mail."""
-    return
     mail_from_address = config["mail"]["from_address"]
     mail_from_name = config["mail"]["from_name"]
     mail_to_address = config["mail"]["to_address"]
@@ -275,10 +185,21 @@ def send_mail(subject: str, body: str) -> None:
     s.quit()
 
 
-def check(name: str, src_url: str, year: int) -> bool:
-    """Check to see if conference site is live."""
-    conf = Conference(name, src_url, year)
-    return conf.check_web_site()
+def check_conference_web_site(name: str, src_url: str, year: int) -> bool:
+    """Check if an individual web site is live."""
+    assert "{year}" in src_url
+    live, msg, redirect_to_url = check_conference(name, src_url, year)
+    url = src_url.format(year=year)
+    if live:
+        if redirect_to_url and normalize_url(redirect_to_url) == normalize_url(url):
+            body = f"{name}\n{url}\nWeb page title: {msg}"
+        else:
+            body = f"""{name}
+{url} redirects to {redirect_to_url}
+Web page title: {msg}"""
+        send_mail(f"Conference site live: {name} - {year}", body)
+
+    return live
 
 
 def find_new_conference_web_sites(
@@ -294,7 +215,8 @@ def find_new_conference_web_sites(
         new += [
             {"conference": name, "year": year, "live": today}
             for year in (this_year, this_year + 1)
-            if (name, year) not in live_set and check(name, src_url, year)
+            if (name, year) not in live_set
+            and check_conference_web_site(name, src_url, year)
         ]
     return new