diff --git a/check.py b/check.py
index 33ba545..62349ec 100755
--- a/check.py
+++ b/check.py
@@ -79,7 +79,6 @@ not_here_list = [
"There is currently no text in this page.",
"This page does not exist yet",
"404 Not Found",
- "HTTP 404",
"500 Internal Server Error",
"500: Internal Server Error",
"Test Page for the Apache HTTP Server",
@@ -120,10 +119,6 @@ not_here_list = [
"503 self-signed certificate",
"504 Gateway Timeout",
"
Pages
",
- "The requested page could not be found",
- "403 Forbidden",
- "You don't have permission to access this resource.",
- "Bad Request (400)",
]
@@ -139,37 +134,18 @@ def get_title(page_html: str) -> str:
def normalize_url(url: str) -> str:
- """Return a URL suitable for comparisons.
-
- Host names are case-insensitive and may end in a DNS root dot. A trailing
- slash on the path is also not a meaningful redirect for this application.
"""
- parsed_url = urlparse(remove_hostname_dot(url))
- normalized_path = parsed_url.path.rstrip("/")
- return urlunparse(
- parsed_url._replace(
- netloc=parsed_url.netloc.lower(),
- path=normalized_path,
- )
- )
+ Normalize the URL by parsing and reconstructing to ensure uniformity.
-
-def remove_hostname_dot(url: str) -> str:
- """Remove the absolute-DNS trailing dot from a URL's hostname."""
+ This handles cases like differing schemes, casing in the domain
+ and trailing slashes.
+ """
+ # Parse the URL into components
parsed_url = urlparse(url)
- userinfo, separator, host_and_port = parsed_url.netloc.rpartition("@")
- prefix = f"{userinfo}{separator}" if separator else ""
-
- # A bracketed IPv6 address cannot have an absolute-DNS trailing dot.
- if host_and_port.startswith("["):
- return url
-
- hostname, port_separator, port = host_and_port.partition(":")
- if not hostname.endswith("."):
- return url
-
- netloc = f"{prefix}{hostname.rstrip('.')}{port_separator}{port}"
- return urlunparse(parsed_url._replace(netloc=netloc))
+ # Normalize the domain to lowercase and remove any trailing dot
+ normalized_netloc = parsed_url.netloc.lower().rstrip(".")
+ # Reconstruct the URL with normalized components
+ return urlunparse(parsed_url._replace(netloc=normalized_netloc))
def url_to_filename(url: str) -> str:
@@ -239,13 +215,12 @@ class Conference:
return (False, "connection refused", None)
self.response = r
- response_url = remove_hostname_dot(r.url)
if r.url.endswith("404.html") or r.url.endswith("404.htm"):
- return (False, "URL ends with 404.html/404.htm", response_url)
+ return (False, "URL ends with 404.html/404.htm", r.url)
if not r.text:
- return (False, "empty response", response_url)
+ return (False, "empty response", r.url)
not_here = find_not_here_message(r.text)
if (
@@ -253,15 +228,15 @@ class Conference:
and 'http-equiv="refresh"' in r.text
and str(self.year) not in r.text
):
- return (False, "redirect to URL without year", response_url)
+ return (False, "redirect to URL without year", r.url)
- if normalize_url(response_url) == normalize_url(self.past_url):
- return (False, "redirect to previous year", response_url)
+ if normalize_url(r.url) == normalize_url(self.past_url):
+ return (False, "redirect to previous year", r.url)
if not_here:
- return (False, not_here, response_url)
+ return (False, not_here, r.url)
- return (True, get_title(r.text), response_url)
+ return (True, get_title(r.text), r.url)
def og_tags(self) -> dict[str, str]:
"""Open Graph tags."""
@@ -273,8 +248,6 @@ class Conference:
if IS_TTY:
print(f"Checking {self.name} {self.year}: {self.url}")
live, msg, redirect_to_url = self.check()
- if redirect_to_url:
- redirect_to_url = remove_hostname_dot(redirect_to_url)
if not live:
if IS_TTY:
print(f" Not live: {msg}")
diff --git a/test_check.py b/test_check.py
deleted file mode 100644
index 68d00ad..0000000
--- a/test_check.py
+++ /dev/null
@@ -1,53 +0,0 @@
-"""Tests for conference website checks."""
-
-import unittest
-from unittest.mock import patch
-
-from check import Conference, normalize_url, remove_hostname_dot
-
-
-class UrlTests(unittest.TestCase):
- """URL cleanup and comparison tests."""
-
- def test_remove_hostname_dot(self) -> None:
- self.assertEqual(
- remove_hostname_dot("https://fosdem.org./2027/"),
- "https://fosdem.org/2027/",
- )
-
- def test_remove_hostname_dot_before_port(self) -> None:
- self.assertEqual(
- remove_hostname_dot("https://fosdem.org.:8443/2027/"),
- "https://fosdem.org:8443/2027/",
- )
-
- def test_normalize_ignores_hostname_dot_and_trailing_slash(self) -> None:
- self.assertEqual(
- normalize_url("https://fosdem.org/2027"),
- normalize_url("https://FOSDEM.org./2027/"),
- )
-
-
-class NotificationTests(unittest.TestCase):
- """Email notification tests."""
-
- @patch("check.send_mail")
- @patch.object(
- Conference,
- "check",
- return_value=(True, "FOSDEM 2027", "https://fosdem.org./2027/"),
- )
- def test_slash_only_redirect_is_not_reported(self, _check, send_mail) -> None:
- conference = Conference("FOSDEM", "https://fosdem.org/{year}", 2027)
-
- self.assertTrue(conference.check_web_site())
-
- send_mail.assert_called_once_with(
- "Conference site live: FOSDEM - 2027",
- "FOSDEM\nhttps://fosdem.org/2027\nWeb page title: FOSDEM 2027",
- )
- self.assertIsNone(conference.redirect_to_url)
-
-
-if __name__ == "__main__":
- unittest.main()