Validate generated conference fields

agenda/add_new_conference.py

@@ -10,7 +10,7 @@ from datetime import date, datetime, time, timezone
 from urllib.parse import parse_qs, urlparse
 
 import html2text
-import lxml.html  # type: ignore[import-untyped]
+import lxml.html
 import openai
 import pycountry
 import requests
@@ -75,7 +75,12 @@ Common optional fields:
   `ca`, `gb`, `us`. Do not output country names.
 - `venue`, `address`, `latitude`, `longitude`, `url`, `cfp_url`, `cfp_end`,
   `hashtag`, `description`.
+- For an address written in a non-Latin script, output a conventional
+  Latin-script rendering only when it can be derived confidently. Do not invent
+  or loosely translate an address. If no reliable rendering is available, omit
+  `address`; coordinates are preferable to an unreadable or inaccurate address.
 - `free`, `price`, `currency`, `hackathon`, `online`, `attendees`.
+- When `free: true`, omit `price` and `currency` because they are redundant.
 - Do not include `going`, `registered`, `accommodation_booked`,
   `transport_booked`, or `trip` unless the source explicitly says they apply to
   my attendance.
@@ -233,6 +238,7 @@ def fetch_webpage(url: str) -> lxml.html.HtmlElement:
 
 def webpage_to_text(root: lxml.html.HtmlElement) -> str:
     """Convert parsed HTML into readable text content."""
+    metadata = extract_event_metadata(root)
     root_copy = lxml.html.fromstring(lxml.html.tostring(root))
 
     for script_or_style in root_copy.xpath("//script|//style"):
@@ -241,9 +247,47 @@ def webpage_to_text(root: lxml.html.HtmlElement) -> str:
     text_maker = html2text.HTML2Text()
     text_maker.ignore_links = True
     text_maker.ignore_images = True
-    return typing.cast(
-        str, text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
+    page_text = text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
+    if not metadata:
+        return page_text
+
+    metadata_text = "\n".join(f"- {key}: {value}" for key, value in metadata.items())
+    return f"Structured event metadata:\n{metadata_text}\n\n{page_text}"
+
+
+def extract_event_metadata(root: lxml.html.HtmlElement) -> dict[str, str]:
+    """Extract Schema.org Event microdata, including visually hidden values."""
+    event_nodes = root.xpath(
+        '//*[@itemscope and contains(@itemtype, "schema.org/Event")]'
     )
+    if not event_nodes:
+        return {}
+
+    metadata: dict[str, str] = {}
+    for element in event_nodes[0].xpath(".//*[@itemprop]"):
+        key = str(element.get("itemprop", "")).strip()
+        value = str(element.get("content", "")).strip()
+        if key and value and key not in metadata:
+            metadata[key] = value
+    return metadata
+
+
+def apply_event_metadata_dates(
+    conf: dict[str, typing.Any], root: lxml.html.HtmlElement
+) -> None:
+    """Fill missing generated dates from Schema.org Event microdata."""
+    dates = conf.get("dates")
+    if isinstance(dates, dict) and dates.get("start") is not None:
+        return
+    if conf.get("start") is not None:
+        return
+
+    metadata = extract_event_metadata(root)
+    start = parse_yaml_date_value(metadata.get("startDate"))
+    if start is None:
+        return
+    end = parse_yaml_date_value(metadata.get("endDate")) or start
+    conf["dates"] = {"status": "exact", "start": start, "end": end}
 
 
 def parse_osm_url(url: str) -> tuple[float, float] | None:
@@ -485,6 +529,27 @@ def validate_country(conf: dict[str, typing.Any]) -> None:
     conf["country"] = match.alpha_2.lower()
 
 
+def validate_series(
+    conf: dict[str, typing.Any], series: dict[str, ConferenceSeries]
+) -> None:
+    """Ensure a generated series ID exists in conference_series.yaml."""
+    series_id = conf.get("series")
+    if series_id is None:
+        return
+    if not isinstance(series_id, str) or series_id not in series:
+        raise ValueError(
+            f"Generated conference uses unknown series {series_id!r}. "
+            "Add it to conference_series.yaml first or remove the series field."
+        )
+
+
+def normalize_free_event_fields(conf: dict[str, typing.Any]) -> None:
+    """Remove redundant pricing fields from free conferences."""
+    if conf.get("free") is True:
+        conf.pop("price", None)
+        conf.pop("currency", None)
+
+
 def parse_yaml_datetime(value: typing.Any) -> datetime | None:
     """Convert YAML date/datetime values to a datetime."""
     if isinstance(value, datetime):
@@ -666,10 +731,7 @@ def load_conferences(yaml_path: str) -> list[dict[str, typing.Any]]:
 
 def load_conference_series_for_path(yaml_path: str) -> dict[str, ConferenceSeries]:
     """Load conference series next to the target conferences YAML file."""
-    return typing.cast(
-        dict[str, ConferenceSeries],
-        load_series(data_dir_from_conferences_path(yaml_path)),
-    )
+    return load_series(data_dir_from_conferences_path(yaml_path))
 
 
 def dump_conferences(yaml_path: str, conferences: list[dict[str, typing.Any]]) -> None:
@@ -709,6 +771,9 @@ def add_new_conference(url: str, yaml_path: str) -> bool:
     assert isinstance(new_conf, dict)
 
     validate_country(new_conf)
+    validate_series(new_conf, series)
+    normalize_free_event_fields(new_conf)
+    apply_event_metadata_dates(new_conf, soup)
     normalize_dates_field(new_conf)
     normalise_end_field(new_conf, source_text)
     normalize_dates_field(new_conf)

tests/test_add_new_conference.py

@@ -3,11 +3,12 @@
 from datetime import date, datetime
 import typing
 
-import lxml.html  # type: ignore[import-untyped]
+import lxml.html
 import pytest
 import yaml
 
 from agenda import add_new_conference
+from agenda.conference import ConferenceSeries
 
 
 def test_parse_osm_url_mlat_mlon() -> None:
@@ -174,6 +175,41 @@ def test_validate_generated_conference_reports_missing_dates() -> None:
         add_new_conference.validate_generated_conference(conf)
 
 
+def test_webpage_to_text_includes_hidden_event_metadata() -> None:
+    """Schema.org event metadata should be included even when visually hidden."""
+    root = lxml.html.fromstring("""
+        <html><body>
+          <div style="display:none" itemscope itemtype="http://schema.org/Event">
+            <meta itemprop="name" content="State of the Map Asia 2026 OSAKA">
+            <meta itemprop="startDate" content="2026-09-06T18:30">
+          </div>
+        </body></html>
+        """)
+
+    text = add_new_conference.webpage_to_text(root)
+
+    assert "- name: State of the Map Asia 2026 OSAKA" in text
+    assert "- startDate: 2026-09-06T18:30" in text
+
+
+def test_apply_event_metadata_dates_fills_missing_generated_dates() -> None:
+    """Structured page dates should recover incomplete model output."""
+    root = lxml.html.fromstring("""
+        <div itemscope itemtype="http://schema.org/Event">
+          <meta itemprop="startDate" content="2026-09-06T18:30">
+        </div>
+        """)
+    conf: dict[str, typing.Any] = {"name": "State of the Map Asia 2026"}
+
+    add_new_conference.apply_event_metadata_dates(conf, root)
+
+    assert conf["dates"] == {
+        "status": "exact",
+        "start": datetime(2026, 9, 6, 18, 30),
+        "end": datetime(2026, 9, 6, 18, 30),
+    }
+
+
 def test_build_prompt_includes_nested_dates_and_series() -> None:
     """The prompt should describe nested dates and known series IDs."""
     prompt = add_new_conference.build_prompt(
@@ -194,6 +230,9 @@ def test_build_prompt_includes_nested_dates_and_series() -> None:
     assert "dates.status" in prompt
     assert "- pycascades: PyCascades" in prompt
     assert "March 2027" in prompt
+    assert "For an address written in a non-Latin script" in prompt
+    assert "If no reliable rendering is available, omit" in prompt
+    assert "When `free: true`, omit `price` and `currency`" in prompt
 
 
 def test_validate_country_normalises_name() -> None:
@@ -205,6 +244,53 @@ def test_validate_country_normalises_name() -> None:
     assert conf["country"] == "gb"
 
 
+def test_validate_series_rejects_unknown_generated_id() -> None:
+    """Generated series IDs must exist in conference_series.yaml."""
+    conf: dict[str, typing.Any] = {"series": "state-of-the-map-asia"}
+
+    with pytest.raises(ValueError, match="unknown series 'state-of-the-map-asia'"):
+        add_new_conference.validate_series(
+            conf,
+            {"state-of-the-map": {"name": "State of the Map"}},
+        )
+
+
+def test_validate_series_accepts_known_id_or_missing_field() -> None:
+    """Known series IDs and conferences without a series should pass."""
+    series: dict[str, ConferenceSeries] = {
+        "state-of-the-map": {"name": "State of the Map"}
+    }
+
+    add_new_conference.validate_series({"series": "state-of-the-map"}, series)
+    add_new_conference.validate_series({}, series)
+
+
+def test_normalize_free_event_fields_removes_price_and_currency() -> None:
+    """Free events should not retain redundant pricing fields."""
+    conf: dict[str, typing.Any] = {
+        "free": True,
+        "price": 0,
+        "currency": "JPY",
+    }
+
+    add_new_conference.normalize_free_event_fields(conf)
+
+    assert conf == {"free": True}
+
+
+def test_normalize_free_event_fields_keeps_paid_event_pricing() -> None:
+    """Pricing fields should remain for events that are not marked free."""
+    conf: dict[str, typing.Any] = {
+        "free": False,
+        "price": 25,
+        "currency": "GBP",
+    }
+
+    add_new_conference.normalize_free_event_fields(conf)
+
+    assert conf == {"free": False, "price": 25, "currency": "GBP"}
+
+
 def test_normalise_end_field_defaults_single_day_date() -> None:
     """Non-Geomob conferences should default end to the start date."""
     conf: dict[str, typing.Any] = {
@@ -333,6 +419,12 @@ def test_add_new_conference_reuses_generic_url_for_new_year(
 ) -> None:
     """Generic URLs with digits in the domain should not be skipped early."""
     yaml_path = tmp_path / "conferences.yaml"
+    (tmp_path / "conference_series.yaml").write_text(
+        yaml.dump(
+            {"foss4g-north-america": {"name": "FOSS4G North America"}},
+            sort_keys=False,
+        )
+    )
     yaml_path.write_text(
         yaml.dump(
             [