diff --git a/frontend/App.vue b/frontend/App.vue index 9e3ce71..08cd664 100644 --- a/frontend/App.vue +++ b/frontend/App.vue @@ -48,7 +48,7 @@ -
+
edits: {{ edits.length }} +
-
+
-
+

{{ edits.length }} edits to upload

+
- +
- +
+ +
+
+
+ +
+ + + + + + + + + +
@@ -166,6 +213,11 @@ remove tag: wikidata={{ edit.qid }} + uploading + saved + @@ -440,6 +492,9 @@ export default { edits: [], view_edits: false, changeset_comment: "Add wikidata tag", + changeset_id: undefined, + upload_state: undefined, + upload_progress: 0, }; }, computed: { @@ -472,6 +527,7 @@ export default { selected_items() { var ret = {}; for (const qid in this.items) { + if (this.items[qid] === undefined) continue; var item = this.items[qid]; if (!item.wikidata) continue; @@ -569,6 +625,82 @@ export default { } }, methods: { + close_edit_list() { + this.view_edits = false; + if (this.upload_state == 'done') { + this.edits = []; + this.upload_progress = 0; + this.upload_state = undefined; + } + }, + upload() { + console.log('upload triggered'); + this.upload_state = "init"; + var edit_list = []; + this.edits.forEach((edit) => { + var e = { + 'qid': edit.item.qid, + 'osm': edit.osm.identifier, + 'op': (edit.osm.selected ? 'add' : 'remove'), + }; + edit_list.push(e); + }); + var post_json = { + 'comment': this.changeset_comment, + 'edit_list': edit_list, + } + console.log('post new session'); + var edit_session_url = `${this.api_base_url}/api/1/edit`; + axios.post(edit_session_url, post_json).then((response) => { + var session_id = response.data.session_id; + var save_url = `${this.api_base_url}/api/1/save/${session_id}`; + console.log('new event source'); + const es = new EventSource(save_url); + es.onerror = function(event) { + console.log('event source:', es); + console.log('ready state:', es.readyState); + } + var app = this; + es.onmessage = function(event) { + const data = JSON.parse(event.data); + switch(data.type) { + case "auth-fail": + app.upload_state = "auth-fail"; + console.log("auth-fail"); + es.close(); + break; + case "changeset-error": + app.upload_state = "changeset-error"; + app.upload_error = data.error; + console.log("changeset-error", data.error); + es.close(); + break; + case "open": + app.upload_state = "uploading"; + app.changeset_id = data.id; + break; + case "progress": + var edit = app.edits[data.num]; + app.upload_progress = ((edit.num + 1) * 100) / app.edits.length; + console.log(app.upload_progress); + edit.osm.upload_state = "progress"; + break; + case "saved": + var edit = app.edits[data.num]; + edit.osm.upload_state = "saved"; + break; + case "closing": + app.upload_state = "closing"; + break; + case "done": + app.upload_state = "done"; + es.close(); + break; + } + console.log('upload state:', app.upload_state); + } + }); + }, edit_list_index(item, osm) { var index = -1; for (var i = 0; i < this.edits.length; i++) { diff --git a/matcher/mail.py b/matcher/mail.py new file mode 100644 index 0000000..a7a6136 --- /dev/null +++ b/matcher/mail.py @@ -0,0 +1,143 @@ +from flask import current_app, g, request, has_request_context +from email.mime.text import MIMEText +from email.utils import formatdate, make_msgid +from pprint import pformat +import smtplib +import traceback +import sys + + +def send_mail(subject, body, config=None): + try: + send_mail_main(subject, body, config=config) + except smtplib.SMTPDataError: + pass # ignore email errors + + +def send_mail_main(subject, body, config=None): + return + if config is None: + config = current_app.config + + mail_to = config["ADMIN_EMAIL"] + mail_from = config["MAIL_FROM"] + msg = MIMEText(body, "plain", "UTF-8") + + msg["Subject"] = subject + msg["To"] = mail_to + msg["From"] = mail_from + msg["Date"] = formatdate() + msg["Message-ID"] = make_msgid() + + s = smtplib.SMTP(config["SMTP_HOST"]) + s.sendmail(mail_from, [mail_to], msg.as_string()) + s.quit() + + +def get_username(): + if hasattr(g, "user"): + if g.user.is_authenticated: + user = g.user.username + else: + user = "not authenticated" + else: + user = "no user" + + return user + + +def get_area(place): + return f"{place.area_in_sq_km:,.2f} sq km" if place.area else "n/a" + + +def error_mail(subject, data, r, via_web=True): + body = f""" +remote URL: {r.url} +status code: {r.status_code} + +request data: +{data} + +status code: {r.status_code} +content-type: {r.headers["content-type"]} + +reply: +{r.text} +""" + + if has_request_context(): + body = f"site URL: {request.url}\nuser: {get_username()}\n" + body + + send_mail(subject, body) + + +def announce_change(change): + body = f""" +user: {change.user.username} +name: {change.place.display_name} +page: {change.place.candidates_url(_external=True)} +items: {change.update_count} +comment: {change.comment} + +https://www.openstreetmap.org/changeset/{change.id} + +""" + + send_mail(f"tags added: {change.place.name_for_changeset}", body) + + +def place_error(place, error_type, error_detail): + body = f""" +user: {get_username()} +name: {place.display_name} +page: {place.candidates_url(_external=True)} +area: {get_area(place)} +error: +{error_detail} +""" + + if error_detail is None: + error_detail = "[None]" + elif len(error_detail) > 100: + error_detail = "[long error message]" + + subject = f"{error_type}: {place.name} - {error_detail}" + send_mail(subject, body) + + +def open_changeset_error(place, changeset, r): + url = place.candidates_url(_external=True) + username = g.user.username + body = f""" +user: {username} +name: {place.display_name} +page: {url} + +message user: https://www.openstreetmap.org/message/new/{username} + +sent: + +{changeset} + +reply: + +{r.text} + +""" + + send_mail("error creating changeset:" + place.name, body) + + +def send_traceback(info, prefix="osm-wikidata"): + exception_name = sys.exc_info()[0].__name__ + subject = f"{prefix} error: {exception_name}" + body = f"user: {get_username()}\n" + info + "\n" + traceback.format_exc() + send_mail(subject, body) + + +def datavalue_missing(field, entity): + qid = entity["title"] + body = f"https://www.wikidata.org/wiki/{qid}\n\n{pformat(entity)}" + + subject = f"{qid}: datavalue missing in {field}" + send_mail(subject, body) diff --git a/web_view.py b/web_view.py index ad83a55..411ea1e 100755 --- a/web_view.py +++ b/web_view.py @@ -1,12 +1,13 @@ #!/usr/bin/python3 from flask import (Flask, render_template, request, jsonify, redirect, url_for, g, - flash, session) + flash, session, Response) from sqlalchemy import func, or_ from sqlalchemy.orm import selectinload -from matcher import nominatim, model, database, commons, wikidata, wikidata_api, osm_oauth +from matcher import (nominatim, model, database, commons, wikidata, wikidata_api, + osm_oauth, edit, mail) from collections import Counter -from time import time +from time import time, sleep from geoalchemy2 import Geography from requests_oauthlib import OAuth1Session import flask_login @@ -1282,5 +1283,129 @@ def oauth_callback(): return redirect(next_page) +def validate_edit_list(edits): + for e in edits: + assert model.Item.get_by_qid(e["qid"]) + assert e["op"] in {"add", "remove"} + osm_type, _, osm_id = e['osm'].partition('/') + osm_id = int(osm_id) + if osm_type == 'node': + assert model.Point.get(osm_id) + else: + src_id = osm_id if osm_type == "way" else -osm_id + assert (model.Line.query.get(src_id) + or model.Polygon.query.get(src_id)) + + +@app.route("/api/1/edit", methods=["POST"]) +def api_new_edit_session(): + user = flask_login.current_user + incoming = request.json + + validate_edit_list(incoming["edit_list"]) + es = model.EditSession(user=user, + edit_list=incoming['edit_list'], + comment=incoming['comment']) + database.session.add(es) + database.session.commit() + + session_id = es.id + + response = jsonify(success=True, session_id=session_id) + response.headers["Access-Control-Allow-Origin"] = "*" + return response + +@app.route("/api/1/edit/", methods=["POST"]) +def api_edit_session(session_id): + es = model.EditSession.query.get(session_id) + assert flask_login.current_user.id == es.user_id + incoming = request.json + + for f in 'edit_list', 'comment': + if f not in incoming: + continue + setattr(es, f, incoming[f]) + database.session.commit() + + response = jsonify(success=True, session_id=session_id) + response.headers["Access-Control-Allow-Origin"] = "*" + return response + +@app.route("/api/1/real_save/") +def api_save_changeset(session_id): + es = model.EditSession.query.get(session_id) + + def send_message(event, **data): + data["type"] = event + return f"data: {json.dumps(data)}\n\n" + + def stream(): + changeset = edit.new_changeset(es.comment) + r = edit.create_changeset(changeset) + reply = r.text.strip() + + if reply == "Couldn't authenticate you": + mail.open_changeset_error(session_id, changeset, r) + yield send_message("auth-fail", error=reply) + return + + if not reply.isdigit(): + mail.open_changeset_error(session_id, changeset, r) + yield send_message("changeset-error", error=reply) + return + + changeset_id = int(reply) + yield send_message("open", id=changeset_id) + + update_count = 0 + + edit.record_changeset( + id=changeset_id, comment=es.comment, update_count=update_count + ) + + for e in es.edit_list: + pass + + return Response(stream(), mimetype='text/event-stream') + +@app.route("/api/1/save/") +def mock_api_save_changeset(session_id): + es = model.EditSession.query.get(session_id) + + def send(event, **data): + data["type"] = event + return f"data: {json.dumps(data)}\n\n" + + def stream(user): + print('stream') + changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar() + sleep(1) + yield send("open", id=changeset_id) + sleep(1) + + update_count = 0 + + print('record_changeset', changeset_id) + edit.record_changeset( + id=changeset_id, user=user, comment=es.comment, update_count=update_count + ) + + print('edits') + + for num, e in enumerate(es.edit_list): + print(num, e) + yield send("progress", edit=e, num=num) + sleep(1) + yield send("saved", edit=e, num=num) + sleep(1) + + print('closing') + yield send("closing") + sleep(1) + yield send("done") + + return Response(stream(g.user), mimetype='text/event-stream') + + if __name__ == "__main__": app.run(host="0.0.0.0")