From c649179cafb7b43eda41830dfc19f89a73bf9fbe Mon Sep 17 00:00:00 2001
From: Edward Betts <edward@4angle.com>
Date: Tue, 26 Nov 2019 14:01:14 +0000
Subject: [PATCH] Record WQDS details and show report.
---
app.py | 7 ++++-
depicts/model.py | 19 ++++++++++++
depicts/wdqs.py | 20 +++++++++++-
templates/query_list.html | 64 +++++++++++++++++++++++++++++++++++++++
4 files changed, 108 insertions(+), 2 deletions(-)
create mode 100644 templates/query_list.html
diff --git a/app.py b/app.py
index 9a23811..c31f8bf 100755
--- a/app.py
+++ b/app.py
@@ -5,7 +5,7 @@ from depicts import (utils, wdqs, commons, mediawiki, artwork, database,
wd_catalog, human, wikibase, wikidata_oauth, wikidata_edit)
from depicts.pager import Pagination, init_pager
from depicts.model import (DepictsItem, DepictsItemAltLabel, Edit, ArtworkItem,
- Language)
+ Language, WikidataQuery)
from depicts.error_mail import setup_error_mail
from requests_oauthlib import OAuth1Session
from werkzeug.exceptions import InternalServerError
@@ -896,6 +896,11 @@ def missing_image_report():
return render_template('missing_image.html', item_list=item_list)
+@app.route('/report/query')
+def wikidata_query_list():
+ q = WikidataQuery.query.order_by(WikidataQuery.start_time.desc())
+ return render_template('query_list.html', q=q)
+
if __name__ == "__main__":
app.debug = True
diff --git a/depicts/model.py b/depicts/model.py
index 00449e3..a075335 100644
--- a/depicts/model.py
+++ b/depicts/model.py
@@ -4,6 +4,7 @@ from sqlalchemy.schema import Column, ForeignKey
from sqlalchemy.types import Integer, String, DateTime, Boolean
from sqlalchemy.orm import column_property, relationship, synonym
from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.sql.expression import cast
from sqlalchemy.dialects import postgresql
from urllib.parse import quote
@@ -98,3 +99,21 @@ class Edit(Base):
@property
def user_wikidata_url(self):
return 'https://www.wikidata.org/wiki/User:' + self.url_norm_username
+
+class WikidataQuery(Base):
+ __tablename__ = 'wikidata_query'
+ id = Column(Integer, primary_key=True)
+ start_time = Column(DateTime)
+ end_time = Column(DateTime)
+ sparql_query = Column(String)
+ path = Column(String)
+ status_code = Column(Integer)
+ error_text = Column(String)
+
+ @hybrid_property
+ def duration(self):
+ return self.end_time - self.start_time
+
+ @property
+ def display_seconds(self):
+ return f'{self.duration.total_seconds():.1f}'
diff --git a/depicts/wdqs.py b/depicts/wdqs.py
index c71b0b2..febdfd6 100644
--- a/depicts/wdqs.py
+++ b/depicts/wdqs.py
@@ -4,8 +4,11 @@ import urllib.parse
import os
import dateutil.parser
import hashlib
+from flask import request
from collections import defaultdict
-from . import utils
+from datetime import datetime
+from .model import WikidataQuery
+from . import utils, database
query_url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'
url_start = 'http://www.wikidata.org/entity/Q'
@@ -26,9 +29,24 @@ def commons_uri_to_filename(uri):
def run_query(query):
params = {'query': query, 'format': 'json'}
+ start = datetime.utcnow()
r = requests.post(query_url, data=params, stream=True)
+ end = datetime.utcnow()
+
+ db_query = WikidataQuery(
+ start_time=start,
+ end_time=end,
+ sparql_query=query,
+ path=request.full_path.rstrip('?'),
+ status_code=r.status_code)
+
if r.status_code != 200:
print(r.text)
+ db_query.error_text = r.text
+
+ database.session.add(db_query)
+ database.session.commit()
+
assert r.status_code == 200
return r
diff --git a/templates/query_list.html b/templates/query_list.html
new file mode 100644
index 0000000..1e6ca5c
--- /dev/null
+++ b/templates/query_list.html
@@ -0,0 +1,64 @@
+{% extends "base.html" %}
+
+{% block content %}
+<div class="p-2">
+
+ <div class="container">
+ <div class="row">
+ <div class="col-2">when</div>
+ <div class="col-3">page</div>
+ <div class="col-2">query time</div>
+ <div class="col">options</div>
+ </div>
+ {% for obj in q %}
+ {% if obj.status_code != 200 %}{% set class=" text-light bg-danger"%}{% endif %}
+ <div class="row">
+ <div class="col-2{{ class }}">
+ {{ obj.start_time.strftime('%Y %b %d %H:%M') }}
+ </div>
+ <div class="col-3"><a href="{{ obj.path }}">{{ obj.path }}</a></div>
+
+ <div class="col-2{{ class }}">
+ {{ obj.display_seconds }} seconds
+ </div>
+
+ <div class="col">
+ [<a href="#" class="toggle" data-id="{{ obj.id }}">show/hide query</a>]
+ [<a href="https://query.wikidata.org/#{{ obj.sparql_query | urlencode }}">Wikidata Query Service</a>]
+ </div>
+ </div>
+
+ <div id="query_{{ obj.id }}" class="row sparql">
+ <div class="col">
+ <pre>{{ obj.sparql_query }}</pre>
+ </div>
+ </div>
+
+ <div class="row error">
+ <div class="col">
+ {% if obj.status_code != 200 %}
+ <div>status code: {{ obj.status_code }}</div>
+ <pre>{{ obj.error_text }}</pre>
+ {% endif %}
+ </div>
+ </div>
+ {% endfor %}
+ </div>
+
+</div>
+
+{% endblock %}
+
+{% block script %}
+<script>
+ $('.sparql').hide();
+ $('.error').hide();
+
+ $('.toggle').click(function() {
+ var query_id = $(this).data('id');
+
+ $('#query_' + query_id).toggle();
+ });
+
+</script>
+{% endblock %}