Normalize conference redirect URLs

check.py

@@ -134,18 +134,37 @@ def get_title(page_html: str) -> str:
 
 
 def normalize_url(url: str) -> str:
-    """
-    Normalize the URL by parsing and reconstructing to ensure uniformity.
+    """Return a URL suitable for comparisons.
 
-    This handles cases like differing schemes, casing in the domain
-    and trailing slashes.
+    Host names are case-insensitive and may end in a DNS root dot. A trailing
+    slash on the path is also not a meaningful redirect for this application.
     """
-    # Parse the URL into components
+    parsed_url = urlparse(remove_hostname_dot(url))
+    normalized_path = parsed_url.path.rstrip("/")
+    return urlunparse(
+        parsed_url._replace(
+            netloc=parsed_url.netloc.lower(),
+            path=normalized_path,
+        )
+    )
+
+
+def remove_hostname_dot(url: str) -> str:
+    """Remove the absolute-DNS trailing dot from a URL's hostname."""
     parsed_url = urlparse(url)
-    # Normalize the domain to lowercase and remove any trailing dot
-    normalized_netloc = parsed_url.netloc.lower().rstrip(".")
-    # Reconstruct the URL with normalized components
-    return urlunparse(parsed_url._replace(netloc=normalized_netloc))
+    userinfo, separator, host_and_port = parsed_url.netloc.rpartition("@")
+    prefix = f"{userinfo}{separator}" if separator else ""
+
+    # A bracketed IPv6 address cannot have an absolute-DNS trailing dot.
+    if host_and_port.startswith("["):
+        return url
+
+    hostname, port_separator, port = host_and_port.partition(":")
+    if not hostname.endswith("."):
+        return url
+
+    netloc = f"{prefix}{hostname.rstrip('.')}{port_separator}{port}"
+    return urlunparse(parsed_url._replace(netloc=netloc))
 
 
 def url_to_filename(url: str) -> str:
@@ -215,12 +234,13 @@ class Conference:
             return (False, "connection refused", None)
 
         self.response = r
+        response_url = remove_hostname_dot(r.url)
 
         if r.url.endswith("404.html") or r.url.endswith("404.htm"):
-            return (False, "URL ends with 404.html/404.htm", r.url)
+            return (False, "URL ends with 404.html/404.htm", response_url)
 
         if not r.text:
-            return (False, "empty response", r.url)
+            return (False, "empty response", response_url)
 
         not_here = find_not_here_message(r.text)
         if (
@@ -228,15 +248,15 @@ class Conference:
             and 'http-equiv="refresh"' in r.text
             and str(self.year) not in r.text
         ):
-            return (False, "redirect to URL without year", r.url)
+            return (False, "redirect to URL without year", response_url)
 
-        if normalize_url(r.url) == normalize_url(self.past_url):
-            return (False, "redirect to previous year", r.url)
+        if normalize_url(response_url) == normalize_url(self.past_url):
+            return (False, "redirect to previous year", response_url)
 
         if not_here:
-            return (False, not_here, r.url)
+            return (False, not_here, response_url)
 
-        return (True, get_title(r.text), r.url)
+        return (True, get_title(r.text), response_url)
 
     def og_tags(self) -> dict[str, str]:
         """Open Graph tags."""
@@ -248,6 +268,8 @@ class Conference:
         if IS_TTY:
             print(f"Checking {self.name} {self.year}: {self.url}")
         live, msg, redirect_to_url = self.check()
+        if redirect_to_url:
+            redirect_to_url = remove_hostname_dot(redirect_to_url)
         if not live:
             if IS_TTY:
                 print(f"  Not live: {msg}")