agenda/add_new_conference.py

@@ -10,7 +10,7 @@ from datetime import date, datetime, time, timezone
 from urllib.parse import parse_qs, urlparse
 
 import html2text
-import lxml.html
+import lxml.html  # type: ignore[import-untyped]
 import openai
 import pycountry
 import requests
@@ -75,12 +75,7 @@ Common optional fields:
   `ca`, `gb`, `us`. Do not output country names.
 - `venue`, `address`, `latitude`, `longitude`, `url`, `cfp_url`, `cfp_end`,
   `hashtag`, `description`.
-- For an address written in a non-Latin script, output a conventional
-  Latin-script rendering only when it can be derived confidently. Do not invent
-  or loosely translate an address. If no reliable rendering is available, omit
-  `address`; coordinates are preferable to an unreadable or inaccurate address.
 - `free`, `price`, `currency`, `hackathon`, `online`, `attendees`.
-- When `free: true`, omit `price` and `currency` because they are redundant.
 - Do not include `going`, `registered`, `accommodation_booked`,
   `transport_booked`, or `trip` unless the source explicitly says they apply to
   my attendance.
@@ -238,7 +233,6 @@ def fetch_webpage(url: str) -> lxml.html.HtmlElement:
 
 def webpage_to_text(root: lxml.html.HtmlElement) -> str:
     """Convert parsed HTML into readable text content."""
-    metadata = extract_event_metadata(root)
     root_copy = lxml.html.fromstring(lxml.html.tostring(root))
 
     for script_or_style in root_copy.xpath("//script|//style"):
@@ -247,47 +241,9 @@ def webpage_to_text(root: lxml.html.HtmlElement) -> str:
     text_maker = html2text.HTML2Text()
     text_maker.ignore_links = True
     text_maker.ignore_images = True
-    page_text = text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
+    return typing.cast(
-    if not metadata:
+        str, text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
-        return page_text
-
-    metadata_text = "\n".join(f"- {key}: {value}" for key, value in metadata.items())
-    return f"Structured event metadata:\n{metadata_text}\n\n{page_text}"
-
-
-def extract_event_metadata(root: lxml.html.HtmlElement) -> dict[str, str]:
-    """Extract Schema.org Event microdata, including visually hidden values."""
-    event_nodes = root.xpath(
-        '//*[@itemscope and contains(@itemtype, "schema.org/Event")]'
     )
-    if not event_nodes:
-        return {}
-
-    metadata: dict[str, str] = {}
-    for element in event_nodes[0].xpath(".//*[@itemprop]"):
-        key = str(element.get("itemprop", "")).strip()
-        value = str(element.get("content", "")).strip()
-        if key and value and key not in metadata:
-            metadata[key] = value
-    return metadata
-
-
-def apply_event_metadata_dates(
-    conf: dict[str, typing.Any], root: lxml.html.HtmlElement
-) -> None:
-    """Fill missing generated dates from Schema.org Event microdata."""
-    dates = conf.get("dates")
-    if isinstance(dates, dict) and dates.get("start") is not None:
-        return
-    if conf.get("start") is not None:
-        return
-
-    metadata = extract_event_metadata(root)
-    start = parse_yaml_date_value(metadata.get("startDate"))
-    if start is None:
-        return
-    end = parse_yaml_date_value(metadata.get("endDate")) or start
-    conf["dates"] = {"status": "exact", "start": start, "end": end}
 
 
 def parse_osm_url(url: str) -> tuple[float, float] | None:
@@ -529,27 +485,6 @@ def validate_country(conf: dict[str, typing.Any]) -> None:
     conf["country"] = match.alpha_2.lower()
 
 
-def validate_series(
-    conf: dict[str, typing.Any], series: dict[str, ConferenceSeries]
-) -> None:
-    """Ensure a generated series ID exists in conference_series.yaml."""
-    series_id = conf.get("series")
-    if series_id is None:
-        return
-    if not isinstance(series_id, str) or series_id not in series:
-        raise ValueError(
-            f"Generated conference uses unknown series {series_id!r}. "
-            "Add it to conference_series.yaml first or remove the series field."
-        )
-
-
-def normalize_free_event_fields(conf: dict[str, typing.Any]) -> None:
-    """Remove redundant pricing fields from free conferences."""
-    if conf.get("free") is True:
-        conf.pop("price", None)
-        conf.pop("currency", None)
-
-
 def parse_yaml_datetime(value: typing.Any) -> datetime | None:
     """Convert YAML date/datetime values to a datetime."""
     if isinstance(value, datetime):
@@ -731,7 +666,10 @@ def load_conferences(yaml_path: str) -> list[dict[str, typing.Any]]:
 
 def load_conference_series_for_path(yaml_path: str) -> dict[str, ConferenceSeries]:
     """Load conference series next to the target conferences YAML file."""
-    return load_series(data_dir_from_conferences_path(yaml_path))
+    return typing.cast(
+        dict[str, ConferenceSeries],
+        load_series(data_dir_from_conferences_path(yaml_path)),
+    )
 
 
 def dump_conferences(yaml_path: str, conferences: list[dict[str, typing.Any]]) -> None:
@@ -771,9 +709,6 @@ def add_new_conference(url: str, yaml_path: str) -> bool:
     assert isinstance(new_conf, dict)
 
     validate_country(new_conf)
-    validate_series(new_conf, series)
-    normalize_free_event_fields(new_conf)
-    apply_event_metadata_dates(new_conf, soup)
     normalize_dates_field(new_conf)
     normalise_end_field(new_conf, source_text)
     normalize_dates_field(new_conf)

tests/test_add_new_conference.py

@@ -3,12 +3,11 @@
 from datetime import date, datetime
 import typing
 
-import lxml.html
+import lxml.html  # type: ignore[import-untyped]
 import pytest
 import yaml
 
 from agenda import add_new_conference
-from agenda.conference import ConferenceSeries
 
 
 def test_parse_osm_url_mlat_mlon() -> None:
@@ -175,41 +174,6 @@ def test_validate_generated_conference_reports_missing_dates() -> None:
         add_new_conference.validate_generated_conference(conf)
 
 
-def test_webpage_to_text_includes_hidden_event_metadata() -> None:
-    """Schema.org event metadata should be included even when visually hidden."""
-    root = lxml.html.fromstring("""
-        <html><body>
-          <div style="display:none" itemscope itemtype="http://schema.org/Event">
-            <meta itemprop="name" content="State of the Map Asia 2026 OSAKA">
-            <meta itemprop="startDate" content="2026-09-06T18:30">
-          </div>
-        </body></html>
-        """)
-
-    text = add_new_conference.webpage_to_text(root)
-
-    assert "- name: State of the Map Asia 2026 OSAKA" in text
-    assert "- startDate: 2026-09-06T18:30" in text
-
-
-def test_apply_event_metadata_dates_fills_missing_generated_dates() -> None:
-    """Structured page dates should recover incomplete model output."""
-    root = lxml.html.fromstring("""
-        <div itemscope itemtype="http://schema.org/Event">
-          <meta itemprop="startDate" content="2026-09-06T18:30">
-        </div>
-        """)
-    conf: dict[str, typing.Any] = {"name": "State of the Map Asia 2026"}
-
-    add_new_conference.apply_event_metadata_dates(conf, root)
-
-    assert conf["dates"] == {
-        "status": "exact",
-        "start": datetime(2026, 9, 6, 18, 30),
-        "end": datetime(2026, 9, 6, 18, 30),
-    }
-
-
 def test_build_prompt_includes_nested_dates_and_series() -> None:
     """The prompt should describe nested dates and known series IDs."""
     prompt = add_new_conference.build_prompt(
@@ -230,9 +194,6 @@ def test_build_prompt_includes_nested_dates_and_series() -> None:
     assert "dates.status" in prompt
     assert "- pycascades: PyCascades" in prompt
     assert "March 2027" in prompt
-    assert "For an address written in a non-Latin script" in prompt
-    assert "If no reliable rendering is available, omit" in prompt
-    assert "When `free: true`, omit `price` and `currency`" in prompt
 
 
 def test_validate_country_normalises_name() -> None:
@@ -244,53 +205,6 @@ def test_validate_country_normalises_name() -> None:
     assert conf["country"] == "gb"
 
 
-def test_validate_series_rejects_unknown_generated_id() -> None:
-    """Generated series IDs must exist in conference_series.yaml."""
-    conf: dict[str, typing.Any] = {"series": "state-of-the-map-asia"}
-
-    with pytest.raises(ValueError, match="unknown series 'state-of-the-map-asia'"):
-        add_new_conference.validate_series(
-            conf,
-            {"state-of-the-map": {"name": "State of the Map"}},
-        )
-
-
-def test_validate_series_accepts_known_id_or_missing_field() -> None:
-    """Known series IDs and conferences without a series should pass."""
-    series: dict[str, ConferenceSeries] = {
-        "state-of-the-map": {"name": "State of the Map"}
-    }
-
-    add_new_conference.validate_series({"series": "state-of-the-map"}, series)
-    add_new_conference.validate_series({}, series)
-
-
-def test_normalize_free_event_fields_removes_price_and_currency() -> None:
-    """Free events should not retain redundant pricing fields."""
-    conf: dict[str, typing.Any] = {
-        "free": True,
-        "price": 0,
-        "currency": "JPY",
-    }
-
-    add_new_conference.normalize_free_event_fields(conf)
-
-    assert conf == {"free": True}
-
-
-def test_normalize_free_event_fields_keeps_paid_event_pricing() -> None:
-    """Pricing fields should remain for events that are not marked free."""
-    conf: dict[str, typing.Any] = {
-        "free": False,
-        "price": 25,
-        "currency": "GBP",
-    }
-
-    add_new_conference.normalize_free_event_fields(conf)
-
-    assert conf == {"free": False, "price": 25, "currency": "GBP"}
-
-
 def test_normalise_end_field_defaults_single_day_date() -> None:
     """Non-Geomob conferences should default end to the start date."""
     conf: dict[str, typing.Any] = {
@@ -419,12 +333,6 @@ def test_add_new_conference_reuses_generic_url_for_new_year(
 ) -> None:
     """Generic URLs with digits in the domain should not be skipped early."""
     yaml_path = tmp_path / "conferences.yaml"
-    (tmp_path / "conference_series.yaml").write_text(
-        yaml.dump(
-            {"foss4g-north-america": {"name": "FOSS4G North America"}},
-            sort_keys=False,
-        )
-    )
     yaml_path.write_text(
         yaml.dump(
             [