Add code for uploading, mocked for now.

This commit is contained in:
Edward Betts 2021-06-24 18:40:59 +02:00
parent b03ae32a9e
commit 2c88e5402c
3 changed files with 409 additions and 9 deletions

View file

@ -48,7 +48,7 @@
</span> </span>
</button> </button>
<div id="edit-count" class="p-2" v-if="edits.length"> <div id="edit-count" class="p-2" v-if="upload_state === undefined && edits.length">
<span>edits: {{ edits.length }}</span> <span>edits: {{ edits.length }}</span>
<button class="btn btn-primary btn-sm ms-2" @click="close_item(); view_edits=true"> <button class="btn btn-primary btn-sm ms-2" @click="close_item(); view_edits=true">
<i class="fa fa-upload"></i> save <i class="fa fa-upload"></i> save
@ -73,21 +73,68 @@
<div v-if="view_edits" class="p-2"> <div v-if="view_edits" class="p-2">
<div class="h3"> <div class="h3">
Upload to OpenStreetMap Upload to OpenStreetMap
<button type="button" class="btn-close float-end" @click="view_edits=false"></button> <button :disabled="upload_state !== undefined && upload_state != 'done'"
type="button"
class="btn-close float-end"
@click="close_edit_list"></button>
</div> </div>
<div class="card w-100 bg-light"> <div class="card w-100 bg-light mb-2">
<div class="card-body"> <div class="card-body">
<form> <p class="card-text">{{ edits.length }} edits to upload</p>
<form @submit.prevent="upload">
<div class="mb-3"> <div class="mb-3">
<label for="changesetComment" class="form-label">Changeset comment</label> <label for="changesetComment" class="form-label">Changeset comment</label>
<input type="text" class="form-control" id="changesetComment" :value="changeset_comment"> <input
:disabled="upload_state !== undefined"
type="text"
class="form-control"
id="changesetComment"
v-model="changeset_comment">
</div> </div>
<button type="submit" class="btn btn-primary">Upload tags</button> <button
:disabled="changeset_comment && upload_state !== undefined"
type="submit"
class="btn btn-primary">
<i class="fa fa-upload"></i> Save to OpenStreetMap
</button>
</form> </form>
<div class="progress mt-2">
<div :style="{ width: upload_progress + '%' }"
class="progress-bar"
role="progressbar"></div>
</div>
</div> </div>
</div> </div>
<div v-if="upload_state == 'auth-fail'" class="alert alert-danger" role="alert">
<p>The OpenStreetMap returned an error: "Couldn't authenticate you".</p>
<p>To workaround this error you need to logout and login again.</p>
</div>
<div v-if="upload_state == 'init'" class="alert alert-info" role="alert">
Starting upload.
</div>
<div v-if="upload_state == 'uploading'" class="alert alert-info" role="alert">
Uploading changes.
</div>
<div v-if="upload_state == 'closing'" class="alert alert-info" role="alert">
Closing changeset.
</div>
<div v-if="upload_state == 'done'" class="alert alert-success" role="alert">
Changes saved.
<a :href="`https://www.openstreetmap.org/changeset/${changeset_id}`"
target="_blank">
view your changeset
</a>
</div>
<div> <div>
<div> <div>
<div class="card my-2 w-100" v-for="edit in edits_grouped_by_qid"> <div class="card my-2 w-100" v-for="edit in edits_grouped_by_qid">
@ -166,6 +213,11 @@
remove tag: <span class="badge bg-danger">wikidata={{ edit.qid }}</span> remove tag: <span class="badge bg-danger">wikidata={{ edit.qid }}</span>
</span> </span>
<span v-if="osm.upload_state == 'current'"
class="ms-2 badge bg-info">uploading</span>
<span v-if="osm.upload_state == 'saved'"
class="ms-2 badge bg-success">saved</span>
</td> </td>
</tr> </tr>
</tbody> </tbody>
@ -440,6 +492,9 @@ export default {
edits: [], edits: [],
view_edits: false, view_edits: false,
changeset_comment: "Add wikidata tag", changeset_comment: "Add wikidata tag",
changeset_id: undefined,
upload_state: undefined,
upload_progress: 0,
}; };
}, },
computed: { computed: {
@ -472,6 +527,7 @@ export default {
selected_items() { selected_items() {
var ret = {}; var ret = {};
for (const qid in this.items) { for (const qid in this.items) {
if (this.items[qid] === undefined) continue;
var item = this.items[qid]; var item = this.items[qid];
if (!item.wikidata) continue; if (!item.wikidata) continue;
@ -569,6 +625,82 @@ export default {
} }
}, },
methods: { methods: {
close_edit_list() {
this.view_edits = false;
if (this.upload_state == 'done') {
this.edits = [];
this.upload_progress = 0;
this.upload_state = undefined;
}
},
upload() {
console.log('upload triggered');
this.upload_state = "init";
var edit_list = [];
this.edits.forEach((edit) => {
var e = {
'qid': edit.item.qid,
'osm': edit.osm.identifier,
'op': (edit.osm.selected ? 'add' : 'remove'),
};
edit_list.push(e);
});
var post_json = {
'comment': this.changeset_comment,
'edit_list': edit_list,
}
console.log('post new session');
var edit_session_url = `${this.api_base_url}/api/1/edit`;
axios.post(edit_session_url, post_json).then((response) => {
var session_id = response.data.session_id;
var save_url = `${this.api_base_url}/api/1/save/${session_id}`;
console.log('new event source');
const es = new EventSource(save_url);
es.onerror = function(event) {
console.log('event source:', es);
console.log('ready state:', es.readyState);
}
var app = this;
es.onmessage = function(event) {
const data = JSON.parse(event.data);
switch(data.type) {
case "auth-fail":
app.upload_state = "auth-fail";
console.log("auth-fail");
es.close();
break;
case "changeset-error":
app.upload_state = "changeset-error";
app.upload_error = data.error;
console.log("changeset-error", data.error);
es.close();
break;
case "open":
app.upload_state = "uploading";
app.changeset_id = data.id;
break;
case "progress":
var edit = app.edits[data.num];
app.upload_progress = ((edit.num + 1) * 100) / app.edits.length;
console.log(app.upload_progress);
edit.osm.upload_state = "progress";
break;
case "saved":
var edit = app.edits[data.num];
edit.osm.upload_state = "saved";
break;
case "closing":
app.upload_state = "closing";
break;
case "done":
app.upload_state = "done";
es.close();
break;
}
console.log('upload state:', app.upload_state);
}
});
},
edit_list_index(item, osm) { edit_list_index(item, osm) {
var index = -1; var index = -1;
for (var i = 0; i < this.edits.length; i++) { for (var i = 0; i < this.edits.length; i++) {

143
matcher/mail.py Normal file
View file

@ -0,0 +1,143 @@
from flask import current_app, g, request, has_request_context
from email.mime.text import MIMEText
from email.utils import formatdate, make_msgid
from pprint import pformat
import smtplib
import traceback
import sys
def send_mail(subject, body, config=None):
try:
send_mail_main(subject, body, config=config)
except smtplib.SMTPDataError:
pass # ignore email errors
def send_mail_main(subject, body, config=None):
return
if config is None:
config = current_app.config
mail_to = config["ADMIN_EMAIL"]
mail_from = config["MAIL_FROM"]
msg = MIMEText(body, "plain", "UTF-8")
msg["Subject"] = subject
msg["To"] = mail_to
msg["From"] = mail_from
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid()
s = smtplib.SMTP(config["SMTP_HOST"])
s.sendmail(mail_from, [mail_to], msg.as_string())
s.quit()
def get_username():
if hasattr(g, "user"):
if g.user.is_authenticated:
user = g.user.username
else:
user = "not authenticated"
else:
user = "no user"
return user
def get_area(place):
return f"{place.area_in_sq_km:,.2f} sq km" if place.area else "n/a"
def error_mail(subject, data, r, via_web=True):
body = f"""
remote URL: {r.url}
status code: {r.status_code}
request data:
{data}
status code: {r.status_code}
content-type: {r.headers["content-type"]}
reply:
{r.text}
"""
if has_request_context():
body = f"site URL: {request.url}\nuser: {get_username()}\n" + body
send_mail(subject, body)
def announce_change(change):
body = f"""
user: {change.user.username}
name: {change.place.display_name}
page: {change.place.candidates_url(_external=True)}
items: {change.update_count}
comment: {change.comment}
https://www.openstreetmap.org/changeset/{change.id}
"""
send_mail(f"tags added: {change.place.name_for_changeset}", body)
def place_error(place, error_type, error_detail):
body = f"""
user: {get_username()}
name: {place.display_name}
page: {place.candidates_url(_external=True)}
area: {get_area(place)}
error:
{error_detail}
"""
if error_detail is None:
error_detail = "[None]"
elif len(error_detail) > 100:
error_detail = "[long error message]"
subject = f"{error_type}: {place.name} - {error_detail}"
send_mail(subject, body)
def open_changeset_error(place, changeset, r):
url = place.candidates_url(_external=True)
username = g.user.username
body = f"""
user: {username}
name: {place.display_name}
page: {url}
message user: https://www.openstreetmap.org/message/new/{username}
sent:
{changeset}
reply:
{r.text}
"""
send_mail("error creating changeset:" + place.name, body)
def send_traceback(info, prefix="osm-wikidata"):
exception_name = sys.exc_info()[0].__name__
subject = f"{prefix} error: {exception_name}"
body = f"user: {get_username()}\n" + info + "\n" + traceback.format_exc()
send_mail(subject, body)
def datavalue_missing(field, entity):
qid = entity["title"]
body = f"https://www.wikidata.org/wiki/{qid}\n\n{pformat(entity)}"
subject = f"{qid}: datavalue missing in {field}"
send_mail(subject, body)

View file

@ -1,12 +1,13 @@
#!/usr/bin/python3 #!/usr/bin/python3
from flask import (Flask, render_template, request, jsonify, redirect, url_for, g, from flask import (Flask, render_template, request, jsonify, redirect, url_for, g,
flash, session) flash, session, Response)
from sqlalchemy import func, or_ from sqlalchemy import func, or_
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from matcher import nominatim, model, database, commons, wikidata, wikidata_api, osm_oauth from matcher import (nominatim, model, database, commons, wikidata, wikidata_api,
osm_oauth, edit, mail)
from collections import Counter from collections import Counter
from time import time from time import time, sleep
from geoalchemy2 import Geography from geoalchemy2 import Geography
from requests_oauthlib import OAuth1Session from requests_oauthlib import OAuth1Session
import flask_login import flask_login
@ -1282,5 +1283,129 @@ def oauth_callback():
return redirect(next_page) return redirect(next_page)
def validate_edit_list(edits):
for e in edits:
assert model.Item.get_by_qid(e["qid"])
assert e["op"] in {"add", "remove"}
osm_type, _, osm_id = e['osm'].partition('/')
osm_id = int(osm_id)
if osm_type == 'node':
assert model.Point.get(osm_id)
else:
src_id = osm_id if osm_type == "way" else -osm_id
assert (model.Line.query.get(src_id)
or model.Polygon.query.get(src_id))
@app.route("/api/1/edit", methods=["POST"])
def api_new_edit_session():
user = flask_login.current_user
incoming = request.json
validate_edit_list(incoming["edit_list"])
es = model.EditSession(user=user,
edit_list=incoming['edit_list'],
comment=incoming['comment'])
database.session.add(es)
database.session.commit()
session_id = es.id
response = jsonify(success=True, session_id=session_id)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
@app.route("/api/1/edit/<int:session_id>", methods=["POST"])
def api_edit_session(session_id):
es = model.EditSession.query.get(session_id)
assert flask_login.current_user.id == es.user_id
incoming = request.json
for f in 'edit_list', 'comment':
if f not in incoming:
continue
setattr(es, f, incoming[f])
database.session.commit()
response = jsonify(success=True, session_id=session_id)
response.headers["Access-Control-Allow-Origin"] = "*"
return response
@app.route("/api/1/real_save/<int:session_id>")
def api_save_changeset(session_id):
es = model.EditSession.query.get(session_id)
def send_message(event, **data):
data["type"] = event
return f"data: {json.dumps(data)}\n\n"
def stream():
changeset = edit.new_changeset(es.comment)
r = edit.create_changeset(changeset)
reply = r.text.strip()
if reply == "Couldn't authenticate you":
mail.open_changeset_error(session_id, changeset, r)
yield send_message("auth-fail", error=reply)
return
if not reply.isdigit():
mail.open_changeset_error(session_id, changeset, r)
yield send_message("changeset-error", error=reply)
return
changeset_id = int(reply)
yield send_message("open", id=changeset_id)
update_count = 0
edit.record_changeset(
id=changeset_id, comment=es.comment, update_count=update_count
)
for e in es.edit_list:
pass
return Response(stream(), mimetype='text/event-stream')
@app.route("/api/1/save/<int:session_id>")
def mock_api_save_changeset(session_id):
es = model.EditSession.query.get(session_id)
def send(event, **data):
data["type"] = event
return f"data: {json.dumps(data)}\n\n"
def stream(user):
print('stream')
changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar()
sleep(1)
yield send("open", id=changeset_id)
sleep(1)
update_count = 0
print('record_changeset', changeset_id)
edit.record_changeset(
id=changeset_id, user=user, comment=es.comment, update_count=update_count
)
print('edits')
for num, e in enumerate(es.edit_list):
print(num, e)
yield send("progress", edit=e, num=num)
sleep(1)
yield send("saved", edit=e, num=num)
sleep(1)
print('closing')
yield send("closing")
sleep(1)
yield send("done")
return Response(stream(g.user), mimetype='text/event-stream')
if __name__ == "__main__": if __name__ == "__main__":
app.run(host="0.0.0.0") app.run(host="0.0.0.0")