diff --git a/agenda/add_new_conference.py b/agenda/add_new_conference.py
index e74455c..d5ce104 100644
--- a/agenda/add_new_conference.py
+++ b/agenda/add_new_conference.py
@@ -10,7 +10,7 @@ from datetime import date, datetime, time, timezone
from urllib.parse import parse_qs, urlparse
import html2text
-import lxml.html # type: ignore[import-untyped]
+import lxml.html
import openai
import pycountry
import requests
@@ -75,6 +75,10 @@ Common optional fields:
`ca`, `gb`, `us`. Do not output country names.
- `venue`, `address`, `latitude`, `longitude`, `url`, `cfp_url`, `cfp_end`,
`hashtag`, `description`.
+- For an address written in a non-Latin script, output a conventional
+ Latin-script rendering only when it can be derived confidently. Do not invent
+ or loosely translate an address. If no reliable rendering is available, omit
+ `address`; coordinates are preferable to an unreadable or inaccurate address.
- `free`, `price`, `currency`, `hackathon`, `online`, `attendees`.
- Do not include `going`, `registered`, `accommodation_booked`,
`transport_booked`, or `trip` unless the source explicitly says they apply to
@@ -233,6 +237,7 @@ def fetch_webpage(url: str) -> lxml.html.HtmlElement:
def webpage_to_text(root: lxml.html.HtmlElement) -> str:
"""Convert parsed HTML into readable text content."""
+ metadata = extract_event_metadata(root)
root_copy = lxml.html.fromstring(lxml.html.tostring(root))
for script_or_style in root_copy.xpath("//script|//style"):
@@ -241,9 +246,47 @@ def webpage_to_text(root: lxml.html.HtmlElement) -> str:
text_maker = html2text.HTML2Text()
text_maker.ignore_links = True
text_maker.ignore_images = True
- return typing.cast(
- str, text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
+ page_text = text_maker.handle(lxml.html.tostring(root_copy, encoding="unicode"))
+ if not metadata:
+ return page_text
+
+ metadata_text = "\n".join(f"- {key}: {value}" for key, value in metadata.items())
+ return f"Structured event metadata:\n{metadata_text}\n\n{page_text}"
+
+
+def extract_event_metadata(root: lxml.html.HtmlElement) -> dict[str, str]:
+ """Extract Schema.org Event microdata, including visually hidden values."""
+ event_nodes = root.xpath(
+ '//*[@itemscope and contains(@itemtype, "schema.org/Event")]'
)
+ if not event_nodes:
+ return {}
+
+ metadata: dict[str, str] = {}
+ for element in event_nodes[0].xpath(".//*[@itemprop]"):
+ key = str(element.get("itemprop", "")).strip()
+ value = str(element.get("content", "")).strip()
+ if key and value and key not in metadata:
+ metadata[key] = value
+ return metadata
+
+
+def apply_event_metadata_dates(
+ conf: dict[str, typing.Any], root: lxml.html.HtmlElement
+) -> None:
+ """Fill missing generated dates from Schema.org Event microdata."""
+ dates = conf.get("dates")
+ if isinstance(dates, dict) and dates.get("start") is not None:
+ return
+ if conf.get("start") is not None:
+ return
+
+ metadata = extract_event_metadata(root)
+ start = parse_yaml_date_value(metadata.get("startDate"))
+ if start is None:
+ return
+ end = parse_yaml_date_value(metadata.get("endDate")) or start
+ conf["dates"] = {"status": "exact", "start": start, "end": end}
def parse_osm_url(url: str) -> tuple[float, float] | None:
@@ -666,10 +709,7 @@ def load_conferences(yaml_path: str) -> list[dict[str, typing.Any]]:
def load_conference_series_for_path(yaml_path: str) -> dict[str, ConferenceSeries]:
"""Load conference series next to the target conferences YAML file."""
- return typing.cast(
- dict[str, ConferenceSeries],
- load_series(data_dir_from_conferences_path(yaml_path)),
- )
+ return load_series(data_dir_from_conferences_path(yaml_path))
def dump_conferences(yaml_path: str, conferences: list[dict[str, typing.Any]]) -> None:
@@ -709,6 +749,7 @@ def add_new_conference(url: str, yaml_path: str) -> bool:
assert isinstance(new_conf, dict)
validate_country(new_conf)
+ apply_event_metadata_dates(new_conf, soup)
normalize_dates_field(new_conf)
normalise_end_field(new_conf, source_text)
normalize_dates_field(new_conf)
diff --git a/tests/test_add_new_conference.py b/tests/test_add_new_conference.py
index 02d19df..ebe96ce 100644
--- a/tests/test_add_new_conference.py
+++ b/tests/test_add_new_conference.py
@@ -174,6 +174,41 @@ def test_validate_generated_conference_reports_missing_dates() -> None:
add_new_conference.validate_generated_conference(conf)
+def test_webpage_to_text_includes_hidden_event_metadata() -> None:
+ """Schema.org event metadata should be included even when visually hidden."""
+ root = lxml.html.fromstring("""
+
+
+
+
+
+
+ """)
+
+ text = add_new_conference.webpage_to_text(root)
+
+ assert "- name: State of the Map Asia 2026 OSAKA" in text
+ assert "- startDate: 2026-09-06T18:30" in text
+
+
+def test_apply_event_metadata_dates_fills_missing_generated_dates() -> None:
+ """Structured page dates should recover incomplete model output."""
+ root = lxml.html.fromstring("""
+
+
+
+ """)
+ conf: dict[str, typing.Any] = {"name": "State of the Map Asia 2026"}
+
+ add_new_conference.apply_event_metadata_dates(conf, root)
+
+ assert conf["dates"] == {
+ "status": "exact",
+ "start": datetime(2026, 9, 6, 18, 30),
+ "end": datetime(2026, 9, 6, 18, 30),
+ }
+
+
def test_build_prompt_includes_nested_dates_and_series() -> None:
"""The prompt should describe nested dates and known series IDs."""
prompt = add_new_conference.build_prompt(
@@ -194,6 +229,8 @@ def test_build_prompt_includes_nested_dates_and_series() -> None:
assert "dates.status" in prompt
assert "- pycascades: PyCascades" in prompt
assert "March 2027" in prompt
+ assert "For an address written in a non-Latin script" in prompt
+ assert "If no reliable rendering is available, omit" in prompt
def test_validate_country_normalises_name() -> None: