diff --git a/check.py b/check.py
index 62349ec..33ba545 100755
--- a/check.py
+++ b/check.py
@@ -79,6 +79,7 @@ not_here_list = [
"There is currently no text in this page.",
"This page does not exist yet",
"404 Not Found",
+ "HTTP 404",
"500 Internal Server Error",
"500: Internal Server Error",
"Test Page for the Apache HTTP Server",
@@ -119,6 +120,10 @@ not_here_list = [
"503 self-signed certificate",
"504 Gateway Timeout",
"
Pages
",
+ "The requested page could not be found",
+ "403 Forbidden",
+ "You don't have permission to access this resource.",
+ "Bad Request (400)",
]
@@ -134,18 +139,37 @@ def get_title(page_html: str) -> str:
def normalize_url(url: str) -> str:
- """
- Normalize the URL by parsing and reconstructing to ensure uniformity.
+ """Return a URL suitable for comparisons.
- This handles cases like differing schemes, casing in the domain
- and trailing slashes.
+ Host names are case-insensitive and may end in a DNS root dot. A trailing
+ slash on the path is also not a meaningful redirect for this application.
"""
- # Parse the URL into components
+ parsed_url = urlparse(remove_hostname_dot(url))
+ normalized_path = parsed_url.path.rstrip("/")
+ return urlunparse(
+ parsed_url._replace(
+ netloc=parsed_url.netloc.lower(),
+ path=normalized_path,
+ )
+ )
+
+
+def remove_hostname_dot(url: str) -> str:
+ """Remove the absolute-DNS trailing dot from a URL's hostname."""
parsed_url = urlparse(url)
- # Normalize the domain to lowercase and remove any trailing dot
- normalized_netloc = parsed_url.netloc.lower().rstrip(".")
- # Reconstruct the URL with normalized components
- return urlunparse(parsed_url._replace(netloc=normalized_netloc))
+ userinfo, separator, host_and_port = parsed_url.netloc.rpartition("@")
+ prefix = f"{userinfo}{separator}" if separator else ""
+
+ # A bracketed IPv6 address cannot have an absolute-DNS trailing dot.
+ if host_and_port.startswith("["):
+ return url
+
+ hostname, port_separator, port = host_and_port.partition(":")
+ if not hostname.endswith("."):
+ return url
+
+ netloc = f"{prefix}{hostname.rstrip('.')}{port_separator}{port}"
+ return urlunparse(parsed_url._replace(netloc=netloc))
def url_to_filename(url: str) -> str:
@@ -215,12 +239,13 @@ class Conference:
return (False, "connection refused", None)
self.response = r
+ response_url = remove_hostname_dot(r.url)
if r.url.endswith("404.html") or r.url.endswith("404.htm"):
- return (False, "URL ends with 404.html/404.htm", r.url)
+ return (False, "URL ends with 404.html/404.htm", response_url)
if not r.text:
- return (False, "empty response", r.url)
+ return (False, "empty response", response_url)
not_here = find_not_here_message(r.text)
if (
@@ -228,15 +253,15 @@ class Conference:
and 'http-equiv="refresh"' in r.text
and str(self.year) not in r.text
):
- return (False, "redirect to URL without year", r.url)
+ return (False, "redirect to URL without year", response_url)
- if normalize_url(r.url) == normalize_url(self.past_url):
- return (False, "redirect to previous year", r.url)
+ if normalize_url(response_url) == normalize_url(self.past_url):
+ return (False, "redirect to previous year", response_url)
if not_here:
- return (False, not_here, r.url)
+ return (False, not_here, response_url)
- return (True, get_title(r.text), r.url)
+ return (True, get_title(r.text), response_url)
def og_tags(self) -> dict[str, str]:
"""Open Graph tags."""
@@ -248,6 +273,8 @@ class Conference:
if IS_TTY:
print(f"Checking {self.name} {self.year}: {self.url}")
live, msg, redirect_to_url = self.check()
+ if redirect_to_url:
+ redirect_to_url = remove_hostname_dot(redirect_to_url)
if not live:
if IS_TTY:
print(f" Not live: {msg}")
diff --git a/test_check.py b/test_check.py
new file mode 100644
index 0000000..68d00ad
--- /dev/null
+++ b/test_check.py
@@ -0,0 +1,53 @@
+"""Tests for conference website checks."""
+
+import unittest
+from unittest.mock import patch
+
+from check import Conference, normalize_url, remove_hostname_dot
+
+
+class UrlTests(unittest.TestCase):
+ """URL cleanup and comparison tests."""
+
+ def test_remove_hostname_dot(self) -> None:
+ self.assertEqual(
+ remove_hostname_dot("https://fosdem.org./2027/"),
+ "https://fosdem.org/2027/",
+ )
+
+ def test_remove_hostname_dot_before_port(self) -> None:
+ self.assertEqual(
+ remove_hostname_dot("https://fosdem.org.:8443/2027/"),
+ "https://fosdem.org:8443/2027/",
+ )
+
+ def test_normalize_ignores_hostname_dot_and_trailing_slash(self) -> None:
+ self.assertEqual(
+ normalize_url("https://fosdem.org/2027"),
+ normalize_url("https://FOSDEM.org./2027/"),
+ )
+
+
+class NotificationTests(unittest.TestCase):
+ """Email notification tests."""
+
+ @patch("check.send_mail")
+ @patch.object(
+ Conference,
+ "check",
+ return_value=(True, "FOSDEM 2027", "https://fosdem.org./2027/"),
+ )
+ def test_slash_only_redirect_is_not_reported(self, _check, send_mail) -> None:
+ conference = Conference("FOSDEM", "https://fosdem.org/{year}", 2027)
+
+ self.assertTrue(conference.check_web_site())
+
+ send_mail.assert_called_once_with(
+ "Conference site live: FOSDEM - 2027",
+ "FOSDEM\nhttps://fosdem.org/2027\nWeb page title: FOSDEM 2027",
+ )
+ self.assertIsNone(conference.redirect_to_url)
+
+
+if __name__ == "__main__":
+ unittest.main()