Handle structured conference metadata

agenda/add_new_conference.py

@@ -10,7 +10,7 @@ from datetime import date, datetime, time, timezone
 from urllib.parse import parse_qs, urlparse
 
 import html2text
-import lxml.html  # type: ignore[import-untyped]
+import lxml.html
 import openai
 import pycountry
 import requests
@@ -75,6 +75,10 @@ Common optional fields:
   `ca`, `gb`, `us`. Do not output country names.
 - `venue`, `address`, `latitude`, `longitude`, `url`, `cfp_url`, `cfp_end`,
   `hashtag`, `description`.
+- For an address written in a non-Latin script, output a conventional
+  Latin-script rendering only when it can be derived confidently. Do not invent
+  or loosely translate an address. If no reliable rendering is available, omit
+  `address`; coordinates are preferable to an unreadable or inaccurate address.
 - `free`, `price`, `currency`, `hackathon`, `online`, `attendees`.
 - Do not include `going`, `registered`, `accommodation_booked`,
   `transport_booked`, or `trip` unless the source explicitly says they apply to
@@ -233,6 +237,7 @@ def fetch_webpage(url: str) -> lxml.html.HtmlElement:
 
 def webpage_to_text(root: lxml.html.HtmlElement) -> str:
     """Convert parsed HTML into readable text content."""
+    metadata = extract_event_metadata(root)
     root_copy = lxml.html.fromstring(lxml.html.tostring(root))
 
     for script_or_style in root_copy.xpath("//script|//style"):
@@ -241,9 +246,47 @@ def webpage_to_text(root: lxml.html.HtmlElement) -> str:
     text_maker = html2text.HTML2Text()
     text_maker.ignore_links = True
     text_maker.ignore_images = True
-    return typing.cast(
-        str, text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
+    page_text = text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
+    if not metadata:
+        return page_text
+
+    metadata_text = "\n".join(f"- {key}: {value}" for key, value in metadata.items())
+    return f"Structured event metadata:\n{metadata_text}\n\n{page_text}"
+
+
+def extract_event_metadata(root: lxml.html.HtmlElement) -> dict[str, str]:
+    """Extract Schema.org Event microdata, including visually hidden values."""
+    event_nodes = root.xpath(
+        '//*[@itemscope and contains(@itemtype, "schema.org/Event")]'
     )
+    if not event_nodes:
+        return {}
+
+    metadata: dict[str, str] = {}
+    for element in event_nodes[0].xpath(".//*[@itemprop]"):
+        key = str(element.get("itemprop", "")).strip()
+        value = str(element.get("content", "")).strip()
+        if key and value and key not in metadata:
+            metadata[key] = value
+    return metadata
+
+
+def apply_event_metadata_dates(
+    conf: dict[str, typing.Any], root: lxml.html.HtmlElement
+) -> None:
+    """Fill missing generated dates from Schema.org Event microdata."""
+    dates = conf.get("dates")
+    if isinstance(dates, dict) and dates.get("start") is not None:
+        return
+    if conf.get("start") is not None:
+        return
+
+    metadata = extract_event_metadata(root)
+    start = parse_yaml_date_value(metadata.get("startDate"))
+    if start is None:
+        return
+    end = parse_yaml_date_value(metadata.get("endDate")) or start
+    conf["dates"] = {"status": "exact", "start": start, "end": end}
 
 
 def parse_osm_url(url: str) -> tuple[float, float] | None:
@@ -666,10 +709,7 @@ def load_conferences(yaml_path: str) -> list[dict[str, typing.Any]]:
 
 def load_conference_series_for_path(yaml_path: str) -> dict[str, ConferenceSeries]:
     """Load conference series next to the target conferences YAML file."""
-    return typing.cast(
-        dict[str, ConferenceSeries],
-        load_series(data_dir_from_conferences_path(yaml_path)),
-    )
+    return load_series(data_dir_from_conferences_path(yaml_path))
 
 
 def dump_conferences(yaml_path: str, conferences: list[dict[str, typing.Any]]) -> None:
@@ -709,6 +749,7 @@ def add_new_conference(url: str, yaml_path: str) -> bool:
     assert isinstance(new_conf, dict)
 
     validate_country(new_conf)
+    apply_event_metadata_dates(new_conf, soup)
     normalize_dates_field(new_conf)
     normalise_end_field(new_conf, source_text)
     normalize_dates_field(new_conf)

tests/test_add_new_conference.py

@@ -174,6 +174,41 @@ def test_validate_generated_conference_reports_missing_dates() -> None:
         add_new_conference.validate_generated_conference(conf)
 
 
+def test_webpage_to_text_includes_hidden_event_metadata() -> None:
+    """Schema.org event metadata should be included even when visually hidden."""
+    root = lxml.html.fromstring("""
+        <html><body>
+          <div style="display:none" itemscope itemtype="http://schema.org/Event">
+            <meta itemprop="name" content="State of the Map Asia 2026 OSAKA">
+            <meta itemprop="startDate" content="2026-09-06T18:30">
+          </div>
+        </body></html>
+        """)
+
+    text = add_new_conference.webpage_to_text(root)
+
+    assert "- name: State of the Map Asia 2026 OSAKA" in text
+    assert "- startDate: 2026-09-06T18:30" in text
+
+
+def test_apply_event_metadata_dates_fills_missing_generated_dates() -> None:
+    """Structured page dates should recover incomplete model output."""
+    root = lxml.html.fromstring("""
+        <div itemscope itemtype="http://schema.org/Event">
+          <meta itemprop="startDate" content="2026-09-06T18:30">
+        </div>
+        """)
+    conf: dict[str, typing.Any] = {"name": "State of the Map Asia 2026"}
+
+    add_new_conference.apply_event_metadata_dates(conf, root)
+
+    assert conf["dates"] == {
+        "status": "exact",
+        "start": datetime(2026, 9, 6, 18, 30),
+        "end": datetime(2026, 9, 6, 18, 30),
+    }
+
+
 def test_build_prompt_includes_nested_dates_and_series() -> None:
     """The prompt should describe nested dates and known series IDs."""
     prompt = add_new_conference.build_prompt(
@@ -194,6 +229,8 @@ def test_build_prompt_includes_nested_dates_and_series() -> None:
     assert "dates.status" in prompt
     assert "- pycascades: PyCascades" in prompt
     assert "March 2027" in prompt
+    assert "For an address written in a non-Latin script" in prompt
+    assert "If no reliable rendering is available, omit" in prompt
 
 
 def test_validate_country_normalises_name() -> None: