diff --git a/check.py b/check.py index 33ba545..62349ec 100755 --- a/check.py +++ b/check.py @@ -79,7 +79,6 @@ not_here_list = [ "There is currently no text in this page.", "This page does not exist yet", "404 Not Found", - "HTTP 404", "500 Internal Server Error", "500: Internal Server Error", "Test Page for the Apache HTTP Server", @@ -120,10 +119,6 @@ not_here_list = [ "503 self-signed certificate", "504 Gateway Timeout", "

Pages

", - "The requested page could not be found", - "403 Forbidden", - "You don't have permission to access this resource.", - "Bad Request (400)", ] @@ -139,37 +134,18 @@ def get_title(page_html: str) -> str: def normalize_url(url: str) -> str: - """Return a URL suitable for comparisons. - - Host names are case-insensitive and may end in a DNS root dot. A trailing - slash on the path is also not a meaningful redirect for this application. """ - parsed_url = urlparse(remove_hostname_dot(url)) - normalized_path = parsed_url.path.rstrip("/") - return urlunparse( - parsed_url._replace( - netloc=parsed_url.netloc.lower(), - path=normalized_path, - ) - ) + Normalize the URL by parsing and reconstructing to ensure uniformity. - -def remove_hostname_dot(url: str) -> str: - """Remove the absolute-DNS trailing dot from a URL's hostname.""" + This handles cases like differing schemes, casing in the domain + and trailing slashes. + """ + # Parse the URL into components parsed_url = urlparse(url) - userinfo, separator, host_and_port = parsed_url.netloc.rpartition("@") - prefix = f"{userinfo}{separator}" if separator else "" - - # A bracketed IPv6 address cannot have an absolute-DNS trailing dot. - if host_and_port.startswith("["): - return url - - hostname, port_separator, port = host_and_port.partition(":") - if not hostname.endswith("."): - return url - - netloc = f"{prefix}{hostname.rstrip('.')}{port_separator}{port}" - return urlunparse(parsed_url._replace(netloc=netloc)) + # Normalize the domain to lowercase and remove any trailing dot + normalized_netloc = parsed_url.netloc.lower().rstrip(".") + # Reconstruct the URL with normalized components + return urlunparse(parsed_url._replace(netloc=normalized_netloc)) def url_to_filename(url: str) -> str: @@ -239,13 +215,12 @@ class Conference: return (False, "connection refused", None) self.response = r - response_url = remove_hostname_dot(r.url) if r.url.endswith("404.html") or r.url.endswith("404.htm"): - return (False, "URL ends with 404.html/404.htm", response_url) + return (False, "URL ends with 404.html/404.htm", r.url) if not r.text: - return (False, "empty response", response_url) + return (False, "empty response", r.url) not_here = find_not_here_message(r.text) if ( @@ -253,15 +228,15 @@ class Conference: and 'http-equiv="refresh"' in r.text and str(self.year) not in r.text ): - return (False, "redirect to URL without year", response_url) + return (False, "redirect to URL without year", r.url) - if normalize_url(response_url) == normalize_url(self.past_url): - return (False, "redirect to previous year", response_url) + if normalize_url(r.url) == normalize_url(self.past_url): + return (False, "redirect to previous year", r.url) if not_here: - return (False, not_here, response_url) + return (False, not_here, r.url) - return (True, get_title(r.text), response_url) + return (True, get_title(r.text), r.url) def og_tags(self) -> dict[str, str]: """Open Graph tags.""" @@ -273,8 +248,6 @@ class Conference: if IS_TTY: print(f"Checking {self.name} {self.year}: {self.url}") live, msg, redirect_to_url = self.check() - if redirect_to_url: - redirect_to_url = remove_hostname_dot(redirect_to_url) if not live: if IS_TTY: print(f" Not live: {msg}") diff --git a/test_check.py b/test_check.py deleted file mode 100644 index 68d00ad..0000000 --- a/test_check.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Tests for conference website checks.""" - -import unittest -from unittest.mock import patch - -from check import Conference, normalize_url, remove_hostname_dot - - -class UrlTests(unittest.TestCase): - """URL cleanup and comparison tests.""" - - def test_remove_hostname_dot(self) -> None: - self.assertEqual( - remove_hostname_dot("https://fosdem.org./2027/"), - "https://fosdem.org/2027/", - ) - - def test_remove_hostname_dot_before_port(self) -> None: - self.assertEqual( - remove_hostname_dot("https://fosdem.org.:8443/2027/"), - "https://fosdem.org:8443/2027/", - ) - - def test_normalize_ignores_hostname_dot_and_trailing_slash(self) -> None: - self.assertEqual( - normalize_url("https://fosdem.org/2027"), - normalize_url("https://FOSDEM.org./2027/"), - ) - - -class NotificationTests(unittest.TestCase): - """Email notification tests.""" - - @patch("check.send_mail") - @patch.object( - Conference, - "check", - return_value=(True, "FOSDEM 2027", "https://fosdem.org./2027/"), - ) - def test_slash_only_redirect_is_not_reported(self, _check, send_mail) -> None: - conference = Conference("FOSDEM", "https://fosdem.org/{year}", 2027) - - self.assertTrue(conference.check_web_site()) - - send_mail.assert_called_once_with( - "Conference site live: FOSDEM - 2027", - "FOSDEM\nhttps://fosdem.org/2027\nWeb page title: FOSDEM 2027", - ) - self.assertIsNone(conference.redirect_to_url) - - -if __name__ == "__main__": - unittest.main()