From c649179cafb7b43eda41830dfc19f89a73bf9fbe Mon Sep 17 00:00:00 2001
From: Edward Betts <edward@4angle.com>
Date: Tue, 26 Nov 2019 14:01:14 +0000
Subject: [PATCH] Record WQDS details and show report.

---
 app.py                    |  7 ++++-
 depicts/model.py          | 19 ++++++++++++
 depicts/wdqs.py           | 20 +++++++++++-
 templates/query_list.html | 64 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 108 insertions(+), 2 deletions(-)
 create mode 100644 templates/query_list.html

diff --git a/app.py b/app.py
index 9a23811..c31f8bf 100755
--- a/app.py
+++ b/app.py
@@ -5,7 +5,7 @@ from depicts import (utils, wdqs, commons, mediawiki, artwork, database,
                      wd_catalog, human, wikibase, wikidata_oauth, wikidata_edit)
 from depicts.pager import Pagination, init_pager
 from depicts.model import (DepictsItem, DepictsItemAltLabel, Edit, ArtworkItem,
-                           Language)
+                           Language, WikidataQuery)
 from depicts.error_mail import setup_error_mail
 from requests_oauthlib import OAuth1Session
 from werkzeug.exceptions import InternalServerError
@@ -896,6 +896,11 @@ def missing_image_report():
 
     return render_template('missing_image.html', item_list=item_list)
 
+@app.route('/report/query')
+def wikidata_query_list():
+    q = WikidataQuery.query.order_by(WikidataQuery.start_time.desc())
+    return render_template('query_list.html', q=q)
+
 
 if __name__ == "__main__":
     app.debug = True
diff --git a/depicts/model.py b/depicts/model.py
index 00449e3..a075335 100644
--- a/depicts/model.py
+++ b/depicts/model.py
@@ -4,6 +4,7 @@ from sqlalchemy.schema import Column, ForeignKey
 from sqlalchemy.types import Integer, String, DateTime, Boolean
 from sqlalchemy.orm import column_property, relationship, synonym
 from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.sql.expression import cast
 from sqlalchemy.dialects import postgresql
 from urllib.parse import quote
@@ -98,3 +99,21 @@ class Edit(Base):
     @property
     def user_wikidata_url(self):
         return 'https://www.wikidata.org/wiki/User:' + self.url_norm_username
+
+class WikidataQuery(Base):
+    __tablename__ = 'wikidata_query'
+    id = Column(Integer, primary_key=True)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)
+    sparql_query = Column(String)
+    path = Column(String)
+    status_code = Column(Integer)
+    error_text = Column(String)
+
+    @hybrid_property
+    def duration(self):
+        return self.end_time - self.start_time
+
+    @property
+    def display_seconds(self):
+        return f'{self.duration.total_seconds():.1f}'
diff --git a/depicts/wdqs.py b/depicts/wdqs.py
index c71b0b2..febdfd6 100644
--- a/depicts/wdqs.py
+++ b/depicts/wdqs.py
@@ -4,8 +4,11 @@ import urllib.parse
 import os
 import dateutil.parser
 import hashlib
+from flask import request
 from collections import defaultdict
-from . import utils
+from datetime import datetime
+from .model import WikidataQuery
+from . import utils, database
 
 query_url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'
 url_start = 'http://www.wikidata.org/entity/Q'
@@ -26,9 +29,24 @@ def commons_uri_to_filename(uri):
 
 def run_query(query):
     params = {'query': query, 'format': 'json'}
+    start = datetime.utcnow()
     r = requests.post(query_url, data=params, stream=True)
+    end = datetime.utcnow()
+
+    db_query = WikidataQuery(
+        start_time=start,
+        end_time=end,
+        sparql_query=query,
+        path=request.full_path.rstrip('?'),
+        status_code=r.status_code)
+
     if r.status_code != 200:
         print(r.text)
+        db_query.error_text = r.text
+
+    database.session.add(db_query)
+    database.session.commit()
+
     assert r.status_code == 200
     return r
 
diff --git a/templates/query_list.html b/templates/query_list.html
new file mode 100644
index 0000000..1e6ca5c
--- /dev/null
+++ b/templates/query_list.html
@@ -0,0 +1,64 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="p-2">
+
+  <div class="container">
+    <div class="row">
+      <div class="col-2">when</div>
+      <div class="col-3">page</div>
+      <div class="col-2">query time</div>
+      <div class="col">options</div>
+    </div>
+  {% for obj in q %}
+    {% if obj.status_code != 200 %}{% set class=" text-light bg-danger"%}{% endif %}
+    <div class="row">
+      <div class="col-2{{ class }}">
+        {{ obj.start_time.strftime('%Y %b %d %H:%M') }}
+      </div>
+      <div class="col-3"><a href="{{ obj.path }}">{{ obj.path }}</a></div>
+
+      <div class="col-2{{ class }}">
+        {{ obj.display_seconds }}&nbsp;seconds
+      </div>
+
+      <div class="col">
+        [<a href="#" class="toggle" data-id="{{ obj.id }}">show/hide query</a>]
+        [<a href="https://query.wikidata.org/#{{ obj.sparql_query | urlencode }}">Wikidata Query Service</a>]
+      </div>
+      </div>
+
+      <div id="query_{{ obj.id }}" class="row sparql">
+        <div class="col">
+          <pre>{{ obj.sparql_query }}</pre>
+        </div>
+      </div>
+
+      <div class="row error">
+        <div class="col">
+        {% if obj.status_code != 200 %}
+          <div>status code: {{ obj.status_code }}</div>
+          <pre>{{ obj.error_text }}</pre>
+        {% endif %}
+        </div>
+      </div>
+  {% endfor %}
+  </div>
+
+</div>
+
+{% endblock %}
+
+{% block script %}
+<script>
+  $('.sparql').hide();
+  $('.error').hide();
+
+  $('.toggle').click(function() {
+    var query_id = $(this).data('id');
+
+    $('#query_' + query_id).toggle();
+  });
+
+</script>
+{% endblock %}