check.py

@@ -79,7 +79,6 @@ not_here_list = [
     "There is currently no text in this page.",
     "This page does not exist yet",
     "404 Not Found",
-    "HTTP 404",
     "500 Internal Server Error",
     "500: Internal Server Error",
     "Test Page for the Apache HTTP Server",
@@ -120,10 +119,6 @@ not_here_list = [
     "503 self-signed certificate",
     "504 Gateway Timeout",
     "<h2>Pages</h2>",
-    "The requested page could not be found",
-    "403 Forbidden",
-    "You don't have permission to access this resource.",
-    "Bad Request (400)",
 ]
 
 
@@ -139,37 +134,18 @@ def get_title(page_html: str) -> str:
 
 
 def normalize_url(url: str) -> str:
-    """Return a URL suitable for comparisons.
-
-    Host names are case-insensitive and may end in a DNS root dot. A trailing
-    slash on the path is also not a meaningful redirect for this application.
     """
-    parsed_url = urlparse(remove_hostname_dot(url))
-    normalized_path = parsed_url.path.rstrip("/")
-    return urlunparse(
-        parsed_url._replace(
-            netloc=parsed_url.netloc.lower(),
-            path=normalized_path,
-        )
-    )
+    Normalize the URL by parsing and reconstructing to ensure uniformity.
 
-
-def remove_hostname_dot(url: str) -> str:
-    """Remove the absolute-DNS trailing dot from a URL's hostname."""
+    This handles cases like differing schemes, casing in the domain
+    and trailing slashes.
+    """
+    # Parse the URL into components
     parsed_url = urlparse(url)
-    userinfo, separator, host_and_port = parsed_url.netloc.rpartition("@")
-    prefix = f"{userinfo}{separator}" if separator else ""
-
-    # A bracketed IPv6 address cannot have an absolute-DNS trailing dot.
-    if host_and_port.startswith("["):
-        return url
-
-    hostname, port_separator, port = host_and_port.partition(":")
-    if not hostname.endswith("."):
-        return url
-
-    netloc = f"{prefix}{hostname.rstrip('.')}{port_separator}{port}"
-    return urlunparse(parsed_url._replace(netloc=netloc))
+    # Normalize the domain to lowercase and remove any trailing dot
+    normalized_netloc = parsed_url.netloc.lower().rstrip(".")
+    # Reconstruct the URL with normalized components
+    return urlunparse(parsed_url._replace(netloc=normalized_netloc))
 
 
 def url_to_filename(url: str) -> str:
@@ -239,13 +215,12 @@ class Conference:
             return (False, "connection refused", None)
 
         self.response = r
-        response_url = remove_hostname_dot(r.url)
 
         if r.url.endswith("404.html") or r.url.endswith("404.htm"):
-            return (False, "URL ends with 404.html/404.htm", response_url)
+            return (False, "URL ends with 404.html/404.htm", r.url)
 
         if not r.text:
-            return (False, "empty response", response_url)
+            return (False, "empty response", r.url)
 
         not_here = find_not_here_message(r.text)
         if (
@@ -253,15 +228,15 @@ class Conference:
             and 'http-equiv="refresh"' in r.text
             and str(self.year) not in r.text
         ):
-            return (False, "redirect to URL without year", response_url)
+            return (False, "redirect to URL without year", r.url)
 
-        if normalize_url(response_url) == normalize_url(self.past_url):
-            return (False, "redirect to previous year", response_url)
+        if normalize_url(r.url) == normalize_url(self.past_url):
+            return (False, "redirect to previous year", r.url)
 
         if not_here:
-            return (False, not_here, response_url)
+            return (False, not_here, r.url)
 
-        return (True, get_title(r.text), response_url)
+        return (True, get_title(r.text), r.url)
 
     def og_tags(self) -> dict[str, str]:
         """Open Graph tags."""
@@ -273,8 +248,6 @@ class Conference:
         if IS_TTY:
             print(f"Checking {self.name} {self.year}: {self.url}")
         live, msg, redirect_to_url = self.check()
-        if redirect_to_url:
-            redirect_to_url = remove_hostname_dot(redirect_to_url)
         if not live:
             if IS_TTY:
                 print(f"  Not live: {msg}")

test_check.py

@@ -1,53 +0,0 @@
-"""Tests for conference website checks."""
-
-import unittest
-from unittest.mock import patch
-
-from check import Conference, normalize_url, remove_hostname_dot
-
-
-class UrlTests(unittest.TestCase):
-    """URL cleanup and comparison tests."""
-
-    def test_remove_hostname_dot(self) -> None:
-        self.assertEqual(
-            remove_hostname_dot("https://fosdem.org./2027/"),
-            "https://fosdem.org/2027/",
-        )
-
-    def test_remove_hostname_dot_before_port(self) -> None:
-        self.assertEqual(
-            remove_hostname_dot("https://fosdem.org.:8443/2027/"),
-            "https://fosdem.org:8443/2027/",
-        )
-
-    def test_normalize_ignores_hostname_dot_and_trailing_slash(self) -> None:
-        self.assertEqual(
-            normalize_url("https://fosdem.org/2027"),
-            normalize_url("https://FOSDEM.org./2027/"),
-        )
-
-
-class NotificationTests(unittest.TestCase):
-    """Email notification tests."""
-
-    @patch("check.send_mail")
-    @patch.object(
-        Conference,
-        "check",
-        return_value=(True, "FOSDEM 2027", "https://fosdem.org./2027/"),
-    )
-    def test_slash_only_redirect_is_not_reported(self, _check, send_mail) -> None:
-        conference = Conference("FOSDEM", "https://fosdem.org/{year}", 2027)
-
-        self.assertTrue(conference.check_web_site())
-
-        send_mail.assert_called_once_with(
-            "Conference site live: FOSDEM - 2027",
-            "FOSDEM\nhttps://fosdem.org/2027\nWeb page title: FOSDEM 2027",
-        )
-        self.assertIsNone(conference.redirect_to_url)
-
-
-if __name__ == "__main__":
-    unittest.main()