Add code for uploading, mocked for now.

frontend/App.vue

@@ -48,7 +48,7 @@
     </span>
   </button>
 
-  <div id="edit-count" class="p-2" v-if="edits.length">
+  <div id="edit-count" class="p-2" v-if="upload_state === undefined && edits.length">
     <span>edits: {{ edits.length }}</span>
     <button class="btn btn-primary btn-sm ms-2" @click="close_item(); view_edits=true">
       <i class="fa fa-upload"></i> save
@@ -73,21 +73,68 @@
     <div v-if="view_edits" class="p-2">
       <div class="h3">
         Upload to OpenStreetMap
-        <button type="button" class="btn-close float-end" @click="view_edits=false"></button>
+        <button :disabled="upload_state !== undefined && upload_state != 'done'"
+           type="button"
+           class="btn-close float-end"
+           @click="close_edit_list"></button>
       </div>
 
-      <div class="card w-100 bg-light">
+      <div class="card w-100 bg-light mb-2">
         <div class="card-body">
-          <form>
+          <p class="card-text">{{ edits.length }} edits to upload</p>
+          <form @submit.prevent="upload">
             <div class="mb-3">
               <label for="changesetComment" class="form-label">Changeset comment</label>
-              <input type="text" class="form-control" id="changesetComment" :value="changeset_comment">
+              <input
+                :disabled="upload_state !== undefined"
+                type="text"
+                class="form-control"
+                id="changesetComment"
+                v-model="changeset_comment">
             </div>
-            <button type="submit" class="btn btn-primary">Upload tags</button>
+            <button
+              :disabled="changeset_comment && upload_state !== undefined"
+              type="submit"
+              class="btn btn-primary">
+              <i class="fa fa-upload"></i> Save to OpenStreetMap
+            </button>
           </form>
+
+          <div class="progress mt-2">
+            <div :style="{ width: upload_progress + '%' }"
+                 class="progress-bar"
+                 role="progressbar"></div>
+          </div>
+
+
         </div>
       </div>
 
+      <div v-if="upload_state == 'auth-fail'" class="alert alert-danger" role="alert">
+        <p>The OpenStreetMap returned an error: "Couldn't authenticate you".</p>
+        <p>To workaround this error you need to logout and login again.</p>
+      </div>
+
+      <div v-if="upload_state == 'init'" class="alert alert-info" role="alert">
+        Starting upload.
+      </div>
+
+      <div v-if="upload_state == 'uploading'" class="alert alert-info" role="alert">
+        Uploading changes.
+      </div>
+
+      <div v-if="upload_state == 'closing'" class="alert alert-info" role="alert">
+        Closing changeset.
+      </div>
+
+      <div v-if="upload_state == 'done'" class="alert alert-success" role="alert">
+        Changes saved.
+        <a :href="`https://www.openstreetmap.org/changeset/${changeset_id}`"
+          target="_blank">
+          view your changeset
+        </a>
+      </div>
+
       <div>
         <div>
           <div class="card my-2 w-100" v-for="edit in edits_grouped_by_qid">
@@ -166,6 +213,11 @@
                         remove tag: <span class="badge bg-danger">wikidata={{ edit.qid }}</span>
                       </span>
 
+                      <span v-if="osm.upload_state == 'current'"
+                            class="ms-2 badge bg-info">uploading</span>
+                      <span v-if="osm.upload_state == 'saved'"
+                            class="ms-2 badge bg-success">saved</span>
+
                     </td>
                   </tr>
                 </tbody>
@@ -440,6 +492,9 @@ export default {
       edits: [],
       view_edits: false,
       changeset_comment: "Add wikidata tag",
+      changeset_id: undefined,
+      upload_state: undefined,
+      upload_progress: 0,
     };
   },
   computed: {
@@ -472,6 +527,7 @@ export default {
     selected_items() {
       var ret = {};
       for (const qid in this.items) {
+        if (this.items[qid] === undefined) continue;
         var item = this.items[qid];
         if (!item.wikidata) continue;
 
@@ -569,6 +625,82 @@ export default {
     }
   },
   methods: {
+    close_edit_list() {
+      this.view_edits = false;
+      if (this.upload_state == 'done') {
+        this.edits = [];
+        this.upload_progress = 0;
+        this.upload_state = undefined;
+      }
+    },
+    upload() {
+      console.log('upload triggered');
+      this.upload_state = "init";
+      var edit_list = [];
+      this.edits.forEach((edit) => {
+        var e = {
+          'qid': edit.item.qid,
+          'osm': edit.osm.identifier,
+          'op': (edit.osm.selected ? 'add' : 'remove'),
+        };
+        edit_list.push(e);
+      });
+      var post_json = {
+        'comment': this.changeset_comment,
+        'edit_list': edit_list,
+      }
+      console.log('post new session');
+      var edit_session_url = `${this.api_base_url}/api/1/edit`;
+      axios.post(edit_session_url, post_json).then((response) => {
+        var session_id = response.data.session_id;
+        var save_url = `${this.api_base_url}/api/1/save/${session_id}`;
+        console.log('new event source');
+        const es = new EventSource(save_url);
+        es.onerror = function(event) {
+          console.log('event source:', es);
+          console.log('ready state:', es.readyState);
+        }
+        var app = this;
+        es.onmessage = function(event) {
+          const data = JSON.parse(event.data);
+          switch(data.type) {
+            case "auth-fail":
+              app.upload_state = "auth-fail";
+              console.log("auth-fail");
+              es.close();
+              break;
+            case "changeset-error":
+              app.upload_state = "changeset-error";
+              app.upload_error = data.error;
+              console.log("changeset-error", data.error);
+              es.close();
+              break;
+            case "open":
+              app.upload_state = "uploading";
+              app.changeset_id = data.id;
+              break;
+            case "progress":
+              var edit = app.edits[data.num];
+              app.upload_progress = ((edit.num + 1) * 100) / app.edits.length;
+              console.log(app.upload_progress);
+              edit.osm.upload_state = "progress";
+              break;
+            case "saved":
+              var edit = app.edits[data.num];
+              edit.osm.upload_state = "saved";
+              break;
+            case "closing":
+              app.upload_state = "closing";
+              break;
+            case "done":
+              app.upload_state = "done";
+              es.close();
+              break;
+          }
+          console.log('upload state:', app.upload_state);
+        }
+      });
+    },
     edit_list_index(item, osm) {
       var index = -1;
       for (var i = 0; i < this.edits.length; i++) {

matcher/mail.py

@@ -0,0 +1,143 @@
+from flask import current_app, g, request, has_request_context
+from email.mime.text import MIMEText
+from email.utils import formatdate, make_msgid
+from pprint import pformat
+import smtplib
+import traceback
+import sys
+
+
+def send_mail(subject, body, config=None):
+    try:
+        send_mail_main(subject, body, config=config)
+    except smtplib.SMTPDataError:
+        pass  # ignore email errors
+
+
+def send_mail_main(subject, body, config=None):
+    return
+    if config is None:
+        config = current_app.config
+
+    mail_to = config["ADMIN_EMAIL"]
+    mail_from = config["MAIL_FROM"]
+    msg = MIMEText(body, "plain", "UTF-8")
+
+    msg["Subject"] = subject
+    msg["To"] = mail_to
+    msg["From"] = mail_from
+    msg["Date"] = formatdate()
+    msg["Message-ID"] = make_msgid()
+
+    s = smtplib.SMTP(config["SMTP_HOST"])
+    s.sendmail(mail_from, [mail_to], msg.as_string())
+    s.quit()
+
+
+def get_username():
+    if hasattr(g, "user"):
+        if g.user.is_authenticated:
+            user = g.user.username
+        else:
+            user = "not authenticated"
+    else:
+        user = "no user"
+
+    return user
+
+
+def get_area(place):
+    return f"{place.area_in_sq_km:,.2f} sq km" if place.area else "n/a"
+
+
+def error_mail(subject, data, r, via_web=True):
+    body = f"""
+remote URL: {r.url}
+status code: {r.status_code}
+
+request data:
+{data}
+
+status code: {r.status_code}
+content-type: {r.headers["content-type"]}
+
+reply:
+{r.text}
+"""
+
+    if has_request_context():
+        body = f"site URL: {request.url}\nuser: {get_username()}\n" + body
+
+    send_mail(subject, body)
+
+
+def announce_change(change):
+    body = f"""
+user: {change.user.username}
+name: {change.place.display_name}
+page: {change.place.candidates_url(_external=True)}
+items: {change.update_count}
+comment: {change.comment}
+
+https://www.openstreetmap.org/changeset/{change.id}
+
+"""
+
+    send_mail(f"tags added: {change.place.name_for_changeset}", body)
+
+
+def place_error(place, error_type, error_detail):
+    body = f"""
+user: {get_username()}
+name: {place.display_name}
+page: {place.candidates_url(_external=True)}
+area: {get_area(place)}
+error:
+{error_detail}
+"""
+
+    if error_detail is None:
+        error_detail = "[None]"
+    elif len(error_detail) > 100:
+        error_detail = "[long error message]"
+
+    subject = f"{error_type}: {place.name} - {error_detail}"
+    send_mail(subject, body)
+
+
+def open_changeset_error(place, changeset, r):
+    url = place.candidates_url(_external=True)
+    username = g.user.username
+    body = f"""
+user: {username}
+name: {place.display_name}
+page: {url}
+
+message user: https://www.openstreetmap.org/message/new/{username}
+
+sent:
+
+{changeset}
+
+reply:
+
+{r.text}
+
+"""
+
+    send_mail("error creating changeset:" + place.name, body)
+
+
+def send_traceback(info, prefix="osm-wikidata"):
+    exception_name = sys.exc_info()[0].__name__
+    subject = f"{prefix} error: {exception_name}"
+    body = f"user: {get_username()}\n" + info + "\n" + traceback.format_exc()
+    send_mail(subject, body)
+
+
+def datavalue_missing(field, entity):
+    qid = entity["title"]
+    body = f"https://www.wikidata.org/wiki/{qid}\n\n{pformat(entity)}"
+
+    subject = f"{qid}: datavalue missing in {field}"
+    send_mail(subject, body)

web_view.py

@@ -1,12 +1,13 @@
 #!/usr/bin/python3
 
 from flask import (Flask, render_template, request, jsonify, redirect, url_for, g,
-                   flash, session)
+                   flash, session, Response)
 from sqlalchemy import func, or_
 from sqlalchemy.orm import selectinload
-from matcher import nominatim, model, database, commons, wikidata, wikidata_api, osm_oauth
+from matcher import (nominatim, model, database, commons, wikidata, wikidata_api,
+                     osm_oauth, edit, mail)
 from collections import Counter
-from time import time
+from time import time, sleep
 from geoalchemy2 import Geography
 from requests_oauthlib import OAuth1Session
 import flask_login
@@ -1282,5 +1283,129 @@ def oauth_callback():
     return redirect(next_page)
 
 
+def validate_edit_list(edits):
+    for e in edits:
+        assert model.Item.get_by_qid(e["qid"])
+        assert e["op"] in {"add", "remove"}
+        osm_type, _, osm_id = e['osm'].partition('/')
+        osm_id = int(osm_id)
+        if osm_type == 'node':
+            assert model.Point.get(osm_id)
+        else:
+            src_id = osm_id if osm_type == "way" else -osm_id
+            assert (model.Line.query.get(src_id)
+                    or model.Polygon.query.get(src_id))
+
+
+@app.route("/api/1/edit", methods=["POST"])
+def api_new_edit_session():
+    user = flask_login.current_user
+    incoming = request.json
+
+    validate_edit_list(incoming["edit_list"])
+    es = model.EditSession(user=user,
+                           edit_list=incoming['edit_list'],
+                           comment=incoming['comment'])
+    database.session.add(es)
+    database.session.commit()
+
+    session_id = es.id
+
+    response = jsonify(success=True, session_id=session_id)
+    response.headers["Access-Control-Allow-Origin"] = "*"
+    return response
+
+@app.route("/api/1/edit/<int:session_id>", methods=["POST"])
+def api_edit_session(session_id):
+    es = model.EditSession.query.get(session_id)
+    assert flask_login.current_user.id == es.user_id
+    incoming = request.json
+
+    for f in 'edit_list', 'comment':
+        if f not in incoming:
+            continue
+        setattr(es, f, incoming[f])
+    database.session.commit()
+
+    response = jsonify(success=True, session_id=session_id)
+    response.headers["Access-Control-Allow-Origin"] = "*"
+    return response
+
+@app.route("/api/1/real_save/<int:session_id>")
+def api_save_changeset(session_id):
+    es = model.EditSession.query.get(session_id)
+
+    def send_message(event, **data):
+        data["type"] = event
+        return f"data: {json.dumps(data)}\n\n"
+
+    def stream():
+        changeset = edit.new_changeset(es.comment)
+        r = edit.create_changeset(changeset)
+        reply = r.text.strip()
+
+        if reply == "Couldn't authenticate you":
+            mail.open_changeset_error(session_id, changeset, r)
+            yield send_message("auth-fail", error=reply)
+            return
+
+        if not reply.isdigit():
+            mail.open_changeset_error(session_id, changeset, r)
+            yield send_message("changeset-error", error=reply)
+            return
+
+        changeset_id = int(reply)
+        yield send_message("open", id=changeset_id)
+
+        update_count = 0
+
+        edit.record_changeset(
+            id=changeset_id, comment=es.comment, update_count=update_count
+        )
+
+        for e in es.edit_list:
+            pass
+
+    return Response(stream(), mimetype='text/event-stream')
+
+@app.route("/api/1/save/<int:session_id>")
+def mock_api_save_changeset(session_id):
+    es = model.EditSession.query.get(session_id)
+
+    def send(event, **data):
+        data["type"] = event
+        return f"data: {json.dumps(data)}\n\n"
+
+    def stream(user):
+        print('stream')
+        changeset_id = database.session.query(func.max(model.Changeset.id) + 1).scalar()
+        sleep(1)
+        yield send("open", id=changeset_id)
+        sleep(1)
+
+        update_count = 0
+
+        print('record_changeset', changeset_id)
+        edit.record_changeset(
+            id=changeset_id, user=user, comment=es.comment, update_count=update_count
+        )
+
+        print('edits')
+
+        for num, e in enumerate(es.edit_list):
+            print(num, e)
+            yield send("progress", edit=e, num=num)
+            sleep(1)
+            yield send("saved", edit=e, num=num)
+            sleep(1)
+
+        print('closing')
+        yield send("closing")
+        sleep(1)
+        yield send("done")
+
+    return Response(stream(g.user), mimetype='text/event-stream')
+
+
 if __name__ == "__main__":
     app.run(host="0.0.0.0")