diff --git a/agenda/add_new_conference.py b/agenda/add_new_conference.py index 3b041b4..e74455c 100644 --- a/agenda/add_new_conference.py +++ b/agenda/add_new_conference.py @@ -10,7 +10,7 @@ from datetime import date, datetime, time, timezone from urllib.parse import parse_qs, urlparse import html2text -import lxml.html +import lxml.html # type: ignore[import-untyped] import openai import pycountry import requests @@ -75,12 +75,7 @@ Common optional fields: `ca`, `gb`, `us`. Do not output country names. - `venue`, `address`, `latitude`, `longitude`, `url`, `cfp_url`, `cfp_end`, `hashtag`, `description`. -- For an address written in a non-Latin script, output a conventional - Latin-script rendering only when it can be derived confidently. Do not invent - or loosely translate an address. If no reliable rendering is available, omit - `address`; coordinates are preferable to an unreadable or inaccurate address. - `free`, `price`, `currency`, `hackathon`, `online`, `attendees`. -- When `free: true`, omit `price` and `currency` because they are redundant. - Do not include `going`, `registered`, `accommodation_booked`, `transport_booked`, or `trip` unless the source explicitly says they apply to my attendance. @@ -238,7 +233,6 @@ def fetch_webpage(url: str) -> lxml.html.HtmlElement: def webpage_to_text(root: lxml.html.HtmlElement) -> str: """Convert parsed HTML into readable text content.""" - metadata = extract_event_metadata(root) root_copy = lxml.html.fromstring(lxml.html.tostring(root)) for script_or_style in root_copy.xpath("//script|//style"): @@ -247,47 +241,9 @@ def webpage_to_text(root: lxml.html.HtmlElement) -> str: text_maker = html2text.HTML2Text() text_maker.ignore_links = True text_maker.ignore_images = True - page_text = text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode")) - if not metadata: - return page_text - - metadata_text = "\n".join(f"- {key}: {value}" for key, value in metadata.items()) - return f"Structured event metadata:\n{metadata_text}\n\n{page_text}" - - -def extract_event_metadata(root: lxml.html.HtmlElement) -> dict[str, str]: - """Extract Schema.org Event microdata, including visually hidden values.""" - event_nodes = root.xpath( - '//*[@itemscope and contains(@itemtype, "schema.org/Event")]' + return typing.cast( + str, text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode")) ) - if not event_nodes: - return {} - - metadata: dict[str, str] = {} - for element in event_nodes[0].xpath(".//*[@itemprop]"): - key = str(element.get("itemprop", "")).strip() - value = str(element.get("content", "")).strip() - if key and value and key not in metadata: - metadata[key] = value - return metadata - - -def apply_event_metadata_dates( - conf: dict[str, typing.Any], root: lxml.html.HtmlElement -) -> None: - """Fill missing generated dates from Schema.org Event microdata.""" - dates = conf.get("dates") - if isinstance(dates, dict) and dates.get("start") is not None: - return - if conf.get("start") is not None: - return - - metadata = extract_event_metadata(root) - start = parse_yaml_date_value(metadata.get("startDate")) - if start is None: - return - end = parse_yaml_date_value(metadata.get("endDate")) or start - conf["dates"] = {"status": "exact", "start": start, "end": end} def parse_osm_url(url: str) -> tuple[float, float] | None: @@ -529,27 +485,6 @@ def validate_country(conf: dict[str, typing.Any]) -> None: conf["country"] = match.alpha_2.lower() -def validate_series( - conf: dict[str, typing.Any], series: dict[str, ConferenceSeries] -) -> None: - """Ensure a generated series ID exists in conference_series.yaml.""" - series_id = conf.get("series") - if series_id is None: - return - if not isinstance(series_id, str) or series_id not in series: - raise ValueError( - f"Generated conference uses unknown series {series_id!r}. " - "Add it to conference_series.yaml first or remove the series field." - ) - - -def normalize_free_event_fields(conf: dict[str, typing.Any]) -> None: - """Remove redundant pricing fields from free conferences.""" - if conf.get("free") is True: - conf.pop("price", None) - conf.pop("currency", None) - - def parse_yaml_datetime(value: typing.Any) -> datetime | None: """Convert YAML date/datetime values to a datetime.""" if isinstance(value, datetime): @@ -731,7 +666,10 @@ def load_conferences(yaml_path: str) -> list[dict[str, typing.Any]]: def load_conference_series_for_path(yaml_path: str) -> dict[str, ConferenceSeries]: """Load conference series next to the target conferences YAML file.""" - return load_series(data_dir_from_conferences_path(yaml_path)) + return typing.cast( + dict[str, ConferenceSeries], + load_series(data_dir_from_conferences_path(yaml_path)), + ) def dump_conferences(yaml_path: str, conferences: list[dict[str, typing.Any]]) -> None: @@ -771,9 +709,6 @@ def add_new_conference(url: str, yaml_path: str) -> bool: assert isinstance(new_conf, dict) validate_country(new_conf) - validate_series(new_conf, series) - normalize_free_event_fields(new_conf) - apply_event_metadata_dates(new_conf, soup) normalize_dates_field(new_conf) normalise_end_field(new_conf, source_text) normalize_dates_field(new_conf) diff --git a/tests/test_add_new_conference.py b/tests/test_add_new_conference.py index 8038724..02d19df 100644 --- a/tests/test_add_new_conference.py +++ b/tests/test_add_new_conference.py @@ -3,12 +3,11 @@ from datetime import date, datetime import typing -import lxml.html +import lxml.html # type: ignore[import-untyped] import pytest import yaml from agenda import add_new_conference -from agenda.conference import ConferenceSeries def test_parse_osm_url_mlat_mlon() -> None: @@ -175,41 +174,6 @@ def test_validate_generated_conference_reports_missing_dates() -> None: add_new_conference.validate_generated_conference(conf) -def test_webpage_to_text_includes_hidden_event_metadata() -> None: - """Schema.org event metadata should be included even when visually hidden.""" - root = lxml.html.fromstring(""" - -
- - -
- - """) - - text = add_new_conference.webpage_to_text(root) - - assert "- name: State of the Map Asia 2026 OSAKA" in text - assert "- startDate: 2026-09-06T18:30" in text - - -def test_apply_event_metadata_dates_fills_missing_generated_dates() -> None: - """Structured page dates should recover incomplete model output.""" - root = lxml.html.fromstring(""" -
- -
- """) - conf: dict[str, typing.Any] = {"name": "State of the Map Asia 2026"} - - add_new_conference.apply_event_metadata_dates(conf, root) - - assert conf["dates"] == { - "status": "exact", - "start": datetime(2026, 9, 6, 18, 30), - "end": datetime(2026, 9, 6, 18, 30), - } - - def test_build_prompt_includes_nested_dates_and_series() -> None: """The prompt should describe nested dates and known series IDs.""" prompt = add_new_conference.build_prompt( @@ -230,9 +194,6 @@ def test_build_prompt_includes_nested_dates_and_series() -> None: assert "dates.status" in prompt assert "- pycascades: PyCascades" in prompt assert "March 2027" in prompt - assert "For an address written in a non-Latin script" in prompt - assert "If no reliable rendering is available, omit" in prompt - assert "When `free: true`, omit `price` and `currency`" in prompt def test_validate_country_normalises_name() -> None: @@ -244,53 +205,6 @@ def test_validate_country_normalises_name() -> None: assert conf["country"] == "gb" -def test_validate_series_rejects_unknown_generated_id() -> None: - """Generated series IDs must exist in conference_series.yaml.""" - conf: dict[str, typing.Any] = {"series": "state-of-the-map-asia"} - - with pytest.raises(ValueError, match="unknown series 'state-of-the-map-asia'"): - add_new_conference.validate_series( - conf, - {"state-of-the-map": {"name": "State of the Map"}}, - ) - - -def test_validate_series_accepts_known_id_or_missing_field() -> None: - """Known series IDs and conferences without a series should pass.""" - series: dict[str, ConferenceSeries] = { - "state-of-the-map": {"name": "State of the Map"} - } - - add_new_conference.validate_series({"series": "state-of-the-map"}, series) - add_new_conference.validate_series({}, series) - - -def test_normalize_free_event_fields_removes_price_and_currency() -> None: - """Free events should not retain redundant pricing fields.""" - conf: dict[str, typing.Any] = { - "free": True, - "price": 0, - "currency": "JPY", - } - - add_new_conference.normalize_free_event_fields(conf) - - assert conf == {"free": True} - - -def test_normalize_free_event_fields_keeps_paid_event_pricing() -> None: - """Pricing fields should remain for events that are not marked free.""" - conf: dict[str, typing.Any] = { - "free": False, - "price": 25, - "currency": "GBP", - } - - add_new_conference.normalize_free_event_fields(conf) - - assert conf == {"free": False, "price": 25, "currency": "GBP"} - - def test_normalise_end_field_defaults_single_day_date() -> None: """Non-Geomob conferences should default end to the start date.""" conf: dict[str, typing.Any] = { @@ -419,12 +333,6 @@ def test_add_new_conference_reuses_generic_url_for_new_year( ) -> None: """Generic URLs with digits in the domain should not be skipped early.""" yaml_path = tmp_path / "conferences.yaml" - (tmp_path / "conference_series.yaml").write_text( - yaml.dump( - {"foss4g-north-america": {"name": "FOSS4G North America"}}, - sort_keys=False, - ) - ) yaml_path.write_text( yaml.dump( [