More strings for not_here_list.

check.py

@@ -79,6 +79,7 @@ not_here_list = [
     "There is currently no text in this page.",
     "This page does not exist yet",
     "404 Not Found",
+    "HTTP 404",
     "500 Internal Server Error",
     "500: Internal Server Error",
     "Test Page for the Apache HTTP Server",
@@ -119,6 +120,10 @@ not_here_list = [
     "503 self-signed certificate",
     "504 Gateway Timeout",
     "<h2>Pages</h2>",
+    "The requested page could not be found",
+    "403 Forbidden",
+    "You don't have permission to access this resource.",
+    "Bad Request (400)",
 ]
 
 
@@ -134,18 +139,37 @@ def get_title(page_html: str) -> str:
 
 
 def normalize_url(url: str) -> str:
-    """
+    """Return a URL suitable for comparisons.
-    Normalize the URL by parsing and reconstructing to ensure uniformity.
 
-    This handles cases like differing schemes, casing in the domain
+    Host names are case-insensitive and may end in a DNS root dot. A trailing
-    and trailing slashes.
+    slash on the path is also not a meaningful redirect for this application.
     """
-    # Parse the URL into components
+    parsed_url = urlparse(remove_hostname_dot(url))
+    normalized_path = parsed_url.path.rstrip("/")
+    return urlunparse(
+        parsed_url._replace(
+            netloc=parsed_url.netloc.lower(),
+            path=normalized_path,
+        )
+    )
+
+
+def remove_hostname_dot(url: str) -> str:
+    """Remove the absolute-DNS trailing dot from a URL's hostname."""
     parsed_url = urlparse(url)
-    # Normalize the domain to lowercase and remove any trailing dot
+    userinfo, separator, host_and_port = parsed_url.netloc.rpartition("@")
-    normalized_netloc = parsed_url.netloc.lower().rstrip(".")
+    prefix = f"{userinfo}{separator}" if separator else ""
-    # Reconstruct the URL with normalized components
+
-    return urlunparse(parsed_url._replace(netloc=normalized_netloc))
+    # A bracketed IPv6 address cannot have an absolute-DNS trailing dot.
+    if host_and_port.startswith("["):
+        return url
+
+    hostname, port_separator, port = host_and_port.partition(":")
+    if not hostname.endswith("."):
+        return url
+
+    netloc = f"{prefix}{hostname.rstrip('.')}{port_separator}{port}"
+    return urlunparse(parsed_url._replace(netloc=netloc))
 
 
 def url_to_filename(url: str) -> str:
@@ -215,12 +239,13 @@ class Conference:
             return (False, "connection refused", None)
 
         self.response = r
+        response_url = remove_hostname_dot(r.url)
 
         if r.url.endswith("404.html") or r.url.endswith("404.htm"):
-            return (False, "URL ends with 404.html/404.htm", r.url)
+            return (False, "URL ends with 404.html/404.htm", response_url)
 
         if not r.text:
-            return (False, "empty response", r.url)
+            return (False, "empty response", response_url)
 
         not_here = find_not_here_message(r.text)
         if (
@@ -228,15 +253,15 @@ class Conference:
             and 'http-equiv="refresh"' in r.text
             and str(self.year) not in r.text
         ):
-            return (False, "redirect to URL without year", r.url)
+            return (False, "redirect to URL without year", response_url)
 
-        if normalize_url(r.url) == normalize_url(self.past_url):
+        if normalize_url(response_url) == normalize_url(self.past_url):
-            return (False, "redirect to previous year", r.url)
+            return (False, "redirect to previous year", response_url)
 
         if not_here:
-            return (False, not_here, r.url)
+            return (False, not_here, response_url)
 
-        return (True, get_title(r.text), r.url)
+        return (True, get_title(r.text), response_url)
 
     def og_tags(self) -> dict[str, str]:
         """Open Graph tags."""
@@ -248,6 +273,8 @@ class Conference:
         if IS_TTY:
             print(f"Checking {self.name} {self.year}: {self.url}")
         live, msg, redirect_to_url = self.check()
+        if redirect_to_url:
+            redirect_to_url = remove_hostname_dot(redirect_to_url)
         if not live:
             if IS_TTY:
                 print(f"  Not live: {msg}")

test_check.py

@@ -0,0 +1,53 @@
+"""Tests for conference website checks."""
+
+import unittest
+from unittest.mock import patch
+
+from check import Conference, normalize_url, remove_hostname_dot
+
+
+class UrlTests(unittest.TestCase):
+    """URL cleanup and comparison tests."""
+
+    def test_remove_hostname_dot(self) -> None:
+        self.assertEqual(
+            remove_hostname_dot("https://fosdem.org./2027/"),
+            "https://fosdem.org/2027/",
+        )
+
+    def test_remove_hostname_dot_before_port(self) -> None:
+        self.assertEqual(
+            remove_hostname_dot("https://fosdem.org.:8443/2027/"),
+            "https://fosdem.org:8443/2027/",
+        )
+
+    def test_normalize_ignores_hostname_dot_and_trailing_slash(self) -> None:
+        self.assertEqual(
+            normalize_url("https://fosdem.org/2027"),
+            normalize_url("https://FOSDEM.org./2027/"),
+        )
+
+
+class NotificationTests(unittest.TestCase):
+    """Email notification tests."""
+
+    @patch("check.send_mail")
+    @patch.object(
+        Conference,
+        "check",
+        return_value=(True, "FOSDEM 2027", "https://fosdem.org./2027/"),
+    )
+    def test_slash_only_redirect_is_not_reported(self, _check, send_mail) -> None:
+        conference = Conference("FOSDEM", "https://fosdem.org/{year}", 2027)
+
+        self.assertTrue(conference.check_web_site())
+
+        send_mail.assert_called_once_with(
+            "Conference site live: FOSDEM - 2027",
+            "FOSDEM\nhttps://fosdem.org/2027\nWeb page title: FOSDEM 2027",
+        )
+        self.assertIsNone(conference.redirect_to_url)
+
+
+if __name__ == "__main__":
+    unittest.main()