diff --git a/app.py b/app.py index 9a23811..c31f8bf 100755 --- a/app.py +++ b/app.py @@ -5,7 +5,7 @@ from depicts import (utils, wdqs, commons, mediawiki, artwork, database, wd_catalog, human, wikibase, wikidata_oauth, wikidata_edit) from depicts.pager import Pagination, init_pager from depicts.model import (DepictsItem, DepictsItemAltLabel, Edit, ArtworkItem, - Language) + Language, WikidataQuery) from depicts.error_mail import setup_error_mail from requests_oauthlib import OAuth1Session from werkzeug.exceptions import InternalServerError @@ -896,6 +896,11 @@ def missing_image_report(): return render_template('missing_image.html', item_list=item_list) +@app.route('/report/query') +def wikidata_query_list(): + q = WikidataQuery.query.order_by(WikidataQuery.start_time.desc()) + return render_template('query_list.html', q=q) + if __name__ == "__main__": app.debug = True diff --git a/depicts/model.py b/depicts/model.py index 00449e3..a075335 100644 --- a/depicts/model.py +++ b/depicts/model.py @@ -4,6 +4,7 @@ from sqlalchemy.schema import Column, ForeignKey from sqlalchemy.types import Integer, String, DateTime, Boolean from sqlalchemy.orm import column_property, relationship, synonym from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.sql.expression import cast from sqlalchemy.dialects import postgresql from urllib.parse import quote @@ -98,3 +99,21 @@ class Edit(Base): @property def user_wikidata_url(self): return 'https://www.wikidata.org/wiki/User:' + self.url_norm_username + +class WikidataQuery(Base): + __tablename__ = 'wikidata_query' + id = Column(Integer, primary_key=True) + start_time = Column(DateTime) + end_time = Column(DateTime) + sparql_query = Column(String) + path = Column(String) + status_code = Column(Integer) + error_text = Column(String) + + @hybrid_property + def duration(self): + return self.end_time - self.start_time + + @property + def display_seconds(self): + return f'{self.duration.total_seconds():.1f}' diff --git a/depicts/wdqs.py b/depicts/wdqs.py index c71b0b2..febdfd6 100644 --- a/depicts/wdqs.py +++ b/depicts/wdqs.py @@ -4,8 +4,11 @@ import urllib.parse import os import dateutil.parser import hashlib +from flask import request from collections import defaultdict -from . import utils +from datetime import datetime +from .model import WikidataQuery +from . import utils, database query_url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql' url_start = 'http://www.wikidata.org/entity/Q' @@ -26,9 +29,24 @@ def commons_uri_to_filename(uri): def run_query(query): params = {'query': query, 'format': 'json'} + start = datetime.utcnow() r = requests.post(query_url, data=params, stream=True) + end = datetime.utcnow() + + db_query = WikidataQuery( + start_time=start, + end_time=end, + sparql_query=query, + path=request.full_path.rstrip('?'), + status_code=r.status_code) + if r.status_code != 200: print(r.text) + db_query.error_text = r.text + + database.session.add(db_query) + database.session.commit() + assert r.status_code == 200 return r diff --git a/templates/query_list.html b/templates/query_list.html new file mode 100644 index 0000000..1e6ca5c --- /dev/null +++ b/templates/query_list.html @@ -0,0 +1,64 @@ +{% extends "base.html" %} + +{% block content %} +
{{ obj.sparql_query }}+
{{ obj.error_text }}+ {% endif %} +