Code for importing data from pretalx
This commit is contained in:
parent
1001c51bc1
commit
e48acab387
5 changed files with 392 additions and 5 deletions
|
|
@ -1,12 +1,17 @@
|
|||
"""Flask views."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import typing
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import flask
|
||||
import requests
|
||||
from sqlalchemy import func, or_, update
|
||||
from werkzeug.wrappers import Response
|
||||
|
||||
from confarchive import database, model, wikidata, query, utils
|
||||
from confarchive import database, model, query, utils, wikidata
|
||||
|
||||
app = flask.Flask(__name__)
|
||||
app.debug = True
|
||||
|
|
@ -14,6 +19,8 @@ app.debug = True
|
|||
app.config.from_object("config.default")
|
||||
database.init_app(app)
|
||||
|
||||
user_agent = "conference-archive/0.1 (contact: edward@4angle.com)"
|
||||
|
||||
|
||||
@app.route("/person/<int:person_id>", methods=["GET", "POST"])
|
||||
def person(person_id: int) -> str | Response:
|
||||
|
|
@ -425,6 +432,251 @@ def github_wikidata() -> str:
|
|||
return flask.render_template("github.html", items=items)
|
||||
|
||||
|
||||
@app.route("/import")
|
||||
def import_start() -> str | Response:
|
||||
"""Begin import."""
|
||||
assert check_admin_mode()
|
||||
url = flask.request.args.get("url")
|
||||
if not url:
|
||||
return flask.render_template("import/start.html")
|
||||
|
||||
m = re.match("https?://([^/]+)/", url)
|
||||
assert m
|
||||
hostname = m.group(1)
|
||||
|
||||
return flask.redirect(flask.url_for("import_hostname", hostname=hostname))
|
||||
|
||||
|
||||
def pretalx_api(hostname: str, path: str) -> typing.Any:
|
||||
"""Call pretalx API and cache results."""
|
||||
import_dir = os.path.join(app.config["DATA_DIR"], "import")
|
||||
api_url = f"https://{hostname}/api/{path}"
|
||||
|
||||
cache_start = hostname + "_" + path.strip("/").replace("/", "_")
|
||||
existing = [
|
||||
f for f in os.listdir(import_dir) if f.startswith(cache_start + "_2023")
|
||||
]
|
||||
if existing:
|
||||
recent_filename = max(existing)
|
||||
recent = datetime.strptime(
|
||||
recent_filename, cache_start + "_%Y-%m-%d_%H:%M.json"
|
||||
)
|
||||
delta = flask.g.now - recent
|
||||
if not existing or delta > timedelta(hours=1):
|
||||
filename = os.path.join(import_dir, f"{cache_start}_{flask.g.now_str}.json")
|
||||
r = requests.get(api_url, params={"limit": 500})
|
||||
with open(filename, "w") as out:
|
||||
out.write(r.text)
|
||||
events = r.json()
|
||||
else:
|
||||
events = json.load(open(os.path.join(import_dir, recent_filename)))
|
||||
|
||||
return typing.cast(typing.Any, events)
|
||||
|
||||
|
||||
def set_now() -> None:
|
||||
"""Record current datetime in flask globals."""
|
||||
flask.g.now = datetime.utcnow()
|
||||
flask.g.now_str = flask.g.now.strftime("%Y-%m-%d_%H:%M")
|
||||
|
||||
|
||||
@app.route("/import/<hostname>")
|
||||
def import_hostname(hostname: str) -> str:
|
||||
"""Import from hostname."""
|
||||
assert check_admin_mode()
|
||||
set_now()
|
||||
events = pretalx_api(hostname, "events/")
|
||||
|
||||
slugs = [event["slug"] for event in events]
|
||||
titles = [prefer_en_label(event["name"]) for event in events]
|
||||
slug_lookup = {event["slug"]: event for event in events}
|
||||
name_lookup = {prefer_en_label(event["name"]): event for event in events}
|
||||
|
||||
print(slugs)
|
||||
|
||||
q = model.Conference.query.filter(
|
||||
or_(model.Conference.short_name.in_(slugs), model.Conference.title.in_(titles))
|
||||
)
|
||||
print(q.count())
|
||||
|
||||
for conf in q:
|
||||
slug = conf.short_name
|
||||
if slug in slug_lookup:
|
||||
slug_lookup[slug]["existing"] = conf
|
||||
if conf.title in name_lookup:
|
||||
name_lookup[conf.title]["existing"] = conf
|
||||
|
||||
return flask.render_template(
|
||||
"import/start.html",
|
||||
events=events,
|
||||
hostname=hostname,
|
||||
prefer_en_label=prefer_en_label,
|
||||
)
|
||||
|
||||
|
||||
def run_import(
|
||||
event: dict[str, typing.Any],
|
||||
speakers: list[dict[str, typing.Any]],
|
||||
talks: list[dict[str, typing.Any]],
|
||||
speaker_lookup: dict[str, typing.Any],
|
||||
) -> model.Conference:
|
||||
if True:
|
||||
conf = model.Conference(
|
||||
short_name=event["slug"],
|
||||
title=prefer_en_label(event["name"]),
|
||||
start=event["date_from"],
|
||||
end=event["date_to"],
|
||||
)
|
||||
database.session.add(conf)
|
||||
|
||||
code_to_speaker = {}
|
||||
for speaker in speakers:
|
||||
code, name, photo_url = speaker["code"], speaker["name"], speaker["avatar"]
|
||||
if code == "9NX3NE":
|
||||
continue
|
||||
|
||||
if speaker_lookup[code].count():
|
||||
person = speaker_lookup[code].one()
|
||||
else:
|
||||
person = model.Person(name=name)
|
||||
database.session.add(person)
|
||||
code_to_speaker[code] = person
|
||||
|
||||
cp = model.ConferencePerson(
|
||||
person=person,
|
||||
conference=conf,
|
||||
named_as=name,
|
||||
photo_url=photo_url or None,
|
||||
)
|
||||
database.session.add(cp)
|
||||
|
||||
for talk in talks:
|
||||
if not talk["speakers"]:
|
||||
continue
|
||||
|
||||
for s in talk["speakers"]:
|
||||
if s["code"] == "9NX3NE":
|
||||
s["code"] = "3BRWWP"
|
||||
|
||||
start_time = talk["slot"]["start"][11:16]
|
||||
assert re.match(r"\d\d:\d\d", start_time)
|
||||
event = model.Event(
|
||||
title=talk["title"],
|
||||
conference=conf,
|
||||
event_date=talk["slot"]["start"],
|
||||
duration=talk["duration"],
|
||||
room=prefer_en_label(talk["slot"]["room"]),
|
||||
track=prefer_en_label(talk["track"]),
|
||||
abstract=talk["abstract"],
|
||||
description=talk["description"],
|
||||
start=start_time,
|
||||
people=[code_to_speaker[s["code"]] for s in talk["speakers"]],
|
||||
)
|
||||
database.session.add(event)
|
||||
|
||||
database.session.commit()
|
||||
|
||||
if False:
|
||||
conf = model.Conference.query.filter_by(short_name="osfc2018").one()
|
||||
|
||||
q = model.ConferencePerson.query.filter(
|
||||
model.ConferencePerson.conference == conf,
|
||||
model.ConferencePerson.photo_url.isnot(None),
|
||||
)
|
||||
|
||||
photo_dir = "confarchive/static/conference_photo"
|
||||
web_session = requests.Session()
|
||||
web_session.headers.update({"User-Agent": user_agent})
|
||||
|
||||
for cp in q:
|
||||
if not cp.photo_url:
|
||||
continue
|
||||
print(cp.photo_url)
|
||||
r = web_session.get(cp.photo_url)
|
||||
content_type = r.headers.get("Content-Type")
|
||||
print(content_type)
|
||||
assert content_type in model.content_type_to_extension
|
||||
image_ext = model.content_type_to_extension[content_type]
|
||||
photo_filename = f"{conf.id}_{cp.person_id}.{image_ext}"
|
||||
print((cp.person_id, cp.named_as, content_type, image_ext, photo_filename))
|
||||
full_photo = os.path.join(photo_dir, photo_filename)
|
||||
|
||||
with open(full_photo, "wb") as out:
|
||||
out.write(r.content)
|
||||
cp.photo_url_content_type = content_type
|
||||
|
||||
database.session.commit()
|
||||
|
||||
return conf
|
||||
|
||||
|
||||
def prefer_en_label(labels: dict[str, str] | None) -> str | None:
|
||||
if labels is None:
|
||||
return None
|
||||
if "en" in labels:
|
||||
return labels["en"]
|
||||
if len(labels) == 1:
|
||||
return list(labels.values())[0]
|
||||
else:
|
||||
return " / ".join(f"{lang}: {label}" for lang, label in labels.items())
|
||||
|
||||
|
||||
def find_matching_name(name: str):
|
||||
"""People with a matching name."""
|
||||
name_parts = name.split()
|
||||
q1 = model.Person.query.filter(model.Person.name == name)
|
||||
if q1.count():
|
||||
return q1
|
||||
name_pattern = "%" + "%".join(name_parts) + "%"
|
||||
q2 = model.Person.query.filter(model.Person.name.ilike(name_pattern))
|
||||
if len(name_parts) == 1 and q2.count() > 1:
|
||||
return q1
|
||||
return q2
|
||||
|
||||
|
||||
@app.route("/import/<hostname>/<slug>", methods=["GET", "POST"])
|
||||
def import_event(hostname: str, slug: str) -> str | Response:
|
||||
"""Import event."""
|
||||
set_now()
|
||||
|
||||
event = pretalx_api(hostname, f"events/{slug}")
|
||||
rooms = pretalx_api(hostname, f"events/{slug}/rooms")
|
||||
speakers = pretalx_api(hostname, f"events/{slug}/speakers")
|
||||
talks = pretalx_api(hostname, f"events/{slug}/talks")
|
||||
|
||||
print(len(speakers["results"]), speakers["count"])
|
||||
print((speakers["next"], speakers["previous"]))
|
||||
assert len(speakers["results"]) == speakers["count"]
|
||||
assert len(talks["results"]) == talks["count"]
|
||||
|
||||
all_talk_speakers: set[str] = set()
|
||||
for talk in talks["results"]:
|
||||
all_talk_speakers.update(speaker["code"] for speaker in talk["speakers"])
|
||||
|
||||
person_candidates = {
|
||||
speaker["code"]: find_matching_name(speaker["name"])
|
||||
for speaker in speakers["results"]
|
||||
}
|
||||
|
||||
if flask.request.method == "GET":
|
||||
return flask.render_template(
|
||||
"import/event.html",
|
||||
hostname=hostname,
|
||||
slug=slug,
|
||||
event=event,
|
||||
rooms=rooms,
|
||||
speakers=[s for s in speakers["results"] if s["code"] in all_talk_speakers],
|
||||
talks=talks,
|
||||
person_candidates=person_candidates,
|
||||
plural=utils.plural,
|
||||
prefer_en_label=prefer_en_label,
|
||||
)
|
||||
|
||||
conf = run_import(event, speakers["results"], talks["results"], person_candidates)
|
||||
|
||||
return flask.redirect(flask.url_for("conference_page", short_name=conf.short_name))
|
||||
|
||||
|
||||
@app.route("/reports")
|
||||
def reports_page() -> str:
|
||||
"""Page showing statistics."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue