diff --git a/add_links/api.py b/add_links/api.py index 6f57def..05fc7b1 100644 --- a/add_links/api.py +++ b/add_links/api.py @@ -1,5 +1,4 @@ import re -import sys import typing import requests @@ -73,33 +72,18 @@ webpage_error = ( ) -def _get_active_session() -> requests.sessions.Session: - """Return OAuth session if one is available in Flask context, else plain session.""" - try: - from flask import g - if hasattr(g, "oauth_session") and g.oauth_session is not None: - return g.oauth_session # type: ignore[return-value] - except RuntimeError: - pass - print("WARNING: using unauthenticated session", file=sys.stderr) - return get_session() - - def api_get(params: StrDict) -> StrDict: """Make call to Wikipedia API.""" - s = _get_active_session() + s = get_session() r = s.get(get_query_url(), params=params) try: ret: StrDict = r.json() except JSONDecodeError: - print(f"API request failed: HTTP {r.status_code}", file=sys.stderr) - print(f"Response body: {r.text!r}", file=sys.stderr) if webpage_error in r.text: raise MediawikiError(webpage_error) - if r.status_code == 429: - raise MediawikiError("Wikipedia rate limit exceeded — wait a moment and try again.") - raise MediawikiError(f"HTTP {r.status_code}: {r.text[:200]!r}") + else: + raise MediawikiError("unknown error") check_for_error(ret) return ret @@ -287,7 +271,7 @@ def call_get_diff(title: str, section_num: int, section_text: str) -> str: "rvdifftotext": section_text.strip(), } - s = _get_active_session() + s = get_session() r = s.post(get_query_url(), data=data) try: ret = r.json() diff --git a/add_links/match.py b/add_links/match.py index a3440e7..9d00404 100644 --- a/add_links/match.py +++ b/add_links/match.py @@ -78,7 +78,7 @@ re_cite = re.compile( re.I | re.S, ) -re_cite_template_start = re.compile(r"\{\{(?:cite|citation|short description|gli|defn|annotated link|excerpt|main|see|for)\b", re.I) +re_cite_template_start = re.compile(r"\{\{(?:cite|citation|short description|gli|defn|annotated link|excerpt|main|see)\b", re.I) re_no_param_template = re.compile(r"\{\{[^|{}]+\}\}") re_external_link = re.compile(r"\[https?://[^\]]+\]") # Italic text (work titles in bibliographies). Handles apostrophes in content @@ -252,14 +252,6 @@ def add_link(m: re.Match[str], replacement: str, text: str) -> str: if matched_text.startswith("[[") and matched_text.endswith("|"): return m.re.sub(lambda m: f"[[{replacement}|", text, count=1) - split_links = matched_text.find("]] [[") - if split_links > 0 and m.start() >= 2 and text[m.start() - 2 : m.start()] == "[[": - # Match starts inside one link and continues into the next opening link. - # Link only the text from the first link span and leave the second link as-is. - link_dest = replacement.split("|")[0] if "|" in replacement else replacement - visible = matched_text[:split_links] - return text[: m.start() - 2] + f"[[{link_dest}|{visible}]]" + text[m.start() + split_links + 2 :] - inner_bracket = matched_text.find("[[") if inner_bracket > 0: prefix = matched_text[:inner_bracket].rstrip() @@ -559,6 +551,4 @@ def get_diff(q: str, title: str, linkto: str | None) -> dict[str, typing.Any]: ) found["diff"] = call_get_diff(title, found["section_num"], section_text) - if not found["diff"]: - raise NoMatch return found diff --git a/add_links/mediawiki_oauth.py b/add_links/mediawiki_oauth.py index 12ebd58..39aecb0 100644 --- a/add_links/mediawiki_oauth.py +++ b/add_links/mediawiki_oauth.py @@ -1,6 +1,5 @@ """Wikipedia OAuth.""" -import sys import typing import urllib from typing import cast @@ -74,8 +73,9 @@ def api_request(params: typing.Mapping[str, str | int]) -> dict[str, typing.Any] try: return cast(dict[str, typing.Any], r.json()) except Exception: - print(f"API request failed: HTTP {r.status_code}", file=sys.stderr) - print(f"Response body: {r.text!r}", file=sys.stderr) + print("text") + print(r.text) + print("---") raise @@ -99,40 +99,13 @@ def userinfo_call() -> typing.Mapping[str, typing.Any]: return api_request(params) -def get_oauth_session() -> OAuth1Session | None: - """Return an OAuth1Session for the current user, or None if not logged in.""" - if "owner_key" not in session or "owner_secret" not in session: - return None - app = current_app - client_key = app.config["CLIENT_KEY"] - client_secret = app.config["CLIENT_SECRET"] - oauth = OAuth1Session( - client_key, - client_secret=client_secret, - resource_owner_key=session["owner_key"], - resource_owner_secret=session["owner_secret"], - ) - oauth.headers.update({"User-Agent": ua}) - oauth.params = typing.cast( - dict[str, str | int], - {"format": "json", "action": "query", "formatversion": 2}, - ) - return oauth - - def get_username() -> None | str: """Get the username or None if not logged in.""" if "owner_key" not in session: return None # not authorized if "username" not in session: - try: - reply = userinfo_call() - except Exception as e: - print(f"get_username failed, clearing session: {e}", file=sys.stderr) - session.pop("owner_key", None) - session.pop("owner_secret", None) - return None + reply = userinfo_call() if "query" not in reply: return None session["username"] = reply["query"]["userinfo"]["name"] diff --git a/static/css/diff.css b/static/css/diff.css index c7009f2..65d5ef9 100644 --- a/static/css/diff.css +++ b/static/css/diff.css @@ -5,16 +5,19 @@ span.searchmatch { font-weight: bold; } table.diff,td.diff-otitle,td.diff-ntitle{background-color:white} td.diff-otitle,td.diff-ntitle{text-align:center} -td.diff-marker{width:1.5em;text-align:center;font-weight:bold;font-size:1.25em;padding:0 0.3em} +td.diff-marker{text-align:right;font-weight:bold;font-size:1.25em} td.diff-lineno{font-weight:bold} td.diff-addedline,td.diff-deletedline,td.diff-context{font-size:88%;vertical-align:top;white-space:-moz-pre-wrap;white-space:pre-wrap} -td.diff-addedline,td.diff-deletedline{border-left:3px solid} -td.diff-addedline{border-color:#a3d3ff;background:#f0f8ff} -td.diff-deletedline{border-color:#ffe49c;background:#fffaf0} -td.diff-context{color:#555} +td.diff-addedline,td.diff-deletedline{border-style:solid;border-width:1px 1px 1px 4px;border-radius:0.33em} +td.diff-addedline{border-color:#a3d3ff} +td.diff-deletedline{border-color:#ffe49c} +td.diff-context{background:#f3f3f3;color:#333333;border-style:solid;border-width:1px 1px 1px 4px;border-color:#e6e6e6;border-radius:0.33em} .diffchange{font-weight:bold;text-decoration:none} -table.diff{border:none;width:100%;border-spacing:0;border-collapse:collapse;table-layout:auto} +table.diff{border:none;width:98%;border-spacing:4px; table-layout:fixed} +td.diff-addedline .diffchange,td.diff-deletedline .diffchange{border-radius:0.33em;padding:0.25em 0} td.diff-addedline .diffchange{background:#d8ecff} td.diff-deletedline .diffchange{background:#feeec8} -table.diff td{padding:0.2em 0.5em} -table.diff td div{word-wrap:break-word;overflow:auto} +table.diff td{padding:0.33em 0.66em} +table.diff col.diff-marker{width:2%} +table.diff col.diff-content{width:48%} +table.diff td div{ word-wrap:break-word; overflow:auto} diff --git a/static/favicon.svg b/static/favicon.svg deleted file mode 100644 index 181c415..0000000 --- a/static/favicon.svg +++ /dev/null @@ -1,3 +0,0 @@ - - 🔗 - diff --git a/templates/all_done.html b/templates/all_done.html index e3135f7..c346482 100644 --- a/templates/all_done.html +++ b/templates/all_done.html @@ -1,11 +1,10 @@ {% extends "base.html" %} -{% block title %}All done{% endblock %} +{% block title %}Index{% endblock %} {% block content %} -
-

All done

-

No more candidates found for this article.

- Search another article -
+
+

All done

+
back to index
+
{% endblock %} diff --git a/templates/article.html b/templates/article.html index a96770c..55d05b0 100644 --- a/templates/article.html +++ b/templates/article.html @@ -1,152 +1,48 @@ {% extends "base.html" %} -{% block title %}{{ title }}{% endblock %} +{% block title %}Link '{{ title }}' in '{{ hit_title }}'{% endblock %} {% block style %} {% endblock %} {% block content %} -
- +
+

Link '{{ title }}' in '{{ hit_title }}'

+
+ + +
-
-

Find links to "{{ title }}"

- {{ title }} ↗ - {% if redirect_to %} - → redirects to {{ redirect_to }} ↗ - {% endif %} -
+
Username: {{ g.user }}
-
- {{ total }} mentions total - {{ with_link }} already linked{% if total > 0 %} ({{ "{:.0%}".format(with_link / total) }}){% endif %} - {% if saves_this_session %} - {{ saves_this_session }} added this session - {% endif %} -
+
view article
-
-
-
- Searching… -
- Searching… -
-
+
back to index
- +
{% endblock %} -{% block script %} - -{% endblock %} diff --git a/templates/base.html b/templates/base.html index 0328311..3804a16 100644 --- a/templates/base.html +++ b/templates/base.html @@ -2,38 +2,21 @@ - + - - {% block title %}{% endblock %} – Missing Link + + + {% block title %}{% endblock %} + + {% block style %}{% endblock %} - - {% block content %}{% endblock %} - + + {% block script %}{% endblock %} diff --git a/templates/error.html b/templates/error.html deleted file mode 100644 index c1a5018..0000000 --- a/templates/error.html +++ /dev/null @@ -1,17 +0,0 @@ -{% extends "base.html" %} - -{% block title %}Error{% endblock %} - -{% block content %} -
-
-
-
-

Something went wrong

-

{{ message }}

-
- Back to home -
-
-
-{% endblock %} diff --git a/templates/index.html b/templates/index.html index 11c14ec..feab172 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,44 +1,25 @@ {% extends "base.html" %} -{% block title %}Missing Link{% endblock %} +{% block title %}Index{% endblock %} {% block content %} -
-
-
-

Missing Link

-

Find unlinked mentions of a Wikipedia article and add the links.

-
- - -
-
-
+
+

Index

+
+ + +
- {% if debug %} -
-
-

Examples

- - - - - - - - - - {% for item in examples %} - - - - - - {% endfor %} - -
ArticleTotal% linked
{{ item.title }}{{ item.total }}{{ "{:.0%}".format(item.with_links / item.total) }}
-
+
Username: {{ g.user }}
+ + + {% for item in examples %} + + + + + + {% endfor %} +
{{ item.title }}{{ item.total }}{{ "{:.1%}".format(item.with_links / item.total) }}
- {% endif %} -
{% endblock %} diff --git a/templates/save_done.html b/templates/save_done.html index ffca063..c96cc66 100644 --- a/templates/save_done.html +++ b/templates/save_done.html @@ -1,11 +1,10 @@ {% extends "base.html" %} -{% block title %}Edit saved{% endblock %} +{% block title %}Index{% endblock %} {% block content %} -
-

Edit saved

-

Your edit has been saved to Wikipedia.

- Search another article -
+
+

Save done

+
Save is complete.
+
{% endblock %} diff --git a/web_view.py b/web_view.py index 6cc669c..32fb1da 100755 --- a/web_view.py +++ b/web_view.py @@ -4,13 +4,11 @@ import html import itertools import json import re -import sys import typing import flask import werkzeug from requests_oauthlib import OAuth1Session -from requests_oauthlib.oauth1_session import TokenRequestDenied from werkzeug.wrappers.response import Response from add_links import api, core, mediawiki_api, mediawiki_oauth @@ -96,20 +94,14 @@ def search_count(q: str) -> int: return get_hit_count(article_title_to_search_query(q)) - 1 -def search_count_with_link(q: str, redirect_to: str | None = None) -> int: +def search_count_with_link(q: str) -> int: """Articles in Wikipedia that include this search term and a link.""" - count = get_hit_count(article_title_to_search_query(q) + f' linksto:"{q}"') - if redirect_to: - count += get_hit_count(article_title_to_search_query(q) + f' linksto:"{redirect_to}"') - return count + return get_hit_count(article_title_to_search_query(q) + f' linksto:"{q}"') -def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hit]]: +def search_no_link(q: str) -> tuple[int, list[Hit]]: """Search for mentions of article title with no link included.""" - exclude = f' -linksto:"{q}"' - if redirect_to: - exclude += f' -linksto:"{redirect_to}"' - query = run_search(article_title_to_search_query(q) + exclude, "max") + query = run_search(article_title_to_search_query(q) + f' -linksto:"{q}"', "max") return (query["searchinfo"]["totalhits"], query["search"]) @@ -117,7 +109,6 @@ def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hi def global_user() -> None: """Make username available everywhere.""" flask.g.user = mediawiki_oauth.get_username() - flask.g.oauth_session = mediawiki_oauth.get_oauth_session() @app.route("/") @@ -127,20 +118,17 @@ def index() -> str | Response: url = flask.url_for("oauth_callback", **flask.request.args) # type: ignore return flask.redirect(url) + examples = load_examples() + examples.sort( + key=lambda i: float(i["with_links"]) / float(i["total"]), reverse=True + ) + if q := flask.request.args.get("q"): if q_trimmed := q.strip(): return flask.redirect(article_url(q_trimmed)) - debug = flask.request.args.get("debug") - examples: list[dict[str, str | int]] = [] - if debug: - examples = load_examples() - examples.sort( - key=lambda i: float(i["with_links"]) / float(i["total"]), reverse=True - ) - return flask.render_template( - "index.html", examples=examples, article_url=article_url, debug=debug + "index.html", examples=examples, article_url=article_url ) @@ -199,12 +187,7 @@ def start_oauth() -> Response: oauth = OAuth1Session(client_key, client_secret=client_secret, callback_uri="oob") oauth.headers.update({"User-Agent": api.ua}) - try: - fetch_response = oauth.fetch_request_token(request_token_url) - except TokenRequestDenied as e: - return flask.make_response( - flask.render_template("error.html", message=str(e)), 502 - ) + fetch_response = oauth.fetch_request_token(request_token_url) flask.session["owner_key"] = fetch_response.get("oauth_token") flask.session["owner_secret"] = fetch_response.get("oauth_token_secret") @@ -246,8 +229,7 @@ def oauth_callback() -> werkzeug.wrappers.response.Response: flask.session["owner_key"] = oauth_tokens.get("oauth_token") flask.session["owner_secret"] = oauth_tokens.get("oauth_token_secret") - username = mediawiki_oauth.get_username() - print(f"login successful: {username}", file=sys.stderr) + print("login successful") next_page = flask.session.get("after_login") return flask.redirect(next_page if next_page else flask.url_for("index")) @@ -299,16 +281,31 @@ def match_type(q: str, snippet: str) -> str | None: return match +class NoGoodHit(Exception): + """No good hit.""" -def _record_skip(from_title: str, hit_title: str) -> None: - """Record that a candidate was skipped or saved for this article.""" - skipped: dict[str, list[str]] = flask.session.get("skipped", {}) - article_skipped = skipped.get(from_title, []) - if hit_title not in article_skipped: - skipped[from_title] = article_skipped + [hit_title] - flask.session["skipped"] = skipped - flask.session.modified = True +def get_best_hit(title: str, hits: list[Hit]) -> tuple[Hit, dict[str, typing.Any]]: + """Find the best hit within the search results.""" + for hit in hits: + if hit["title"].lower() == title.lower(): + continue + # if match_type(title, hit["snippet"]) != "exact": + # continue + + try: + print(f'get diff: {hit["title"]}, {title}') + found = get_diff(title, hit["title"], None) + except NoMatch: + print("no match") + continue + except api.MediawikiError as e: + print(f"MediawikiError for {hit['title']!r}: {e}") + continue + + return (hit, found) + + raise NoGoodHit def handle_post(url_title: str) -> Response: @@ -319,16 +316,8 @@ def handle_post(url_title: str) -> Response: do_save(from_title, hit_title) except mediawiki_oauth.LoginNeeded: return flask.redirect(flask.url_for("start_oauth")) - except (mediawiki_api.APIError, api.MediawikiError) as e: - return flask.make_response( - flask.render_template("error.html", message=f"Save failed: {e}"), 502 - ) - flask.session["saves"] = flask.session.get("saves", 0) + 1 - saves_by_title: dict[str, int] = flask.session.get("saves_by_title", {}) - saves_by_title[from_title] = saves_by_title.get(from_title, 0) + 1 - flask.session["saves_by_title"] = saves_by_title - flask.session.modified = True - _record_skip(from_title, hit_title) + except mediawiki_api.APIError as e: + return flask.make_response(f"Save failed: {e}", 502) return flask.redirect( flask.url_for("article_page", url_title=url_title, after=hit_title) ) @@ -341,55 +330,48 @@ def article_page(url_title: str) -> str | Response: return handle_post(url_title) from_title = url_title.replace("_", " ").strip() + article_title = flask.request.args.get("title") - try: - redirect_to = api.get_wiki_info(from_title) - except (api.MissingPage, api.MultipleRedirects, api.MediawikiError): - redirect_to = None + total = search_count(from_title) + with_link = search_count_with_link(from_title) - try: - total = search_count(from_title) - with_link = search_count_with_link(from_title, redirect_to) - _no_link_count, hits = search_no_link(from_title, redirect_to) - except api.MediawikiError as e: - return flask.make_response( - flask.render_template("error.html", message=str(e)), 502 - ) + no_link_count, hits = search_no_link(from_title) - # Filter out candidates already processed this session - session_skipped: set[str] = set( - flask.session.get("skipped", {}).get(from_title, []) - ) + by_title = {hit["title"]: hit for hit in hits} - # If a specific candidate was requested, move it to the front - title_param = flask.request.args.get("title") - if title_param: - hits = [h for h in hits if h["title"] == title_param] + \ - [h for h in hits if h["title"] != title_param] + found = None + if article_title in by_title: + hit = by_title[article_title] + try: + found = get_diff(from_title, hit["title"], None) + except NoMatch: + pass - # Record and apply explicit skip-past - after = flask.request.args.get("after") - if after: - _record_skip(from_title, after) - session_skipped.add(after) + if not found: + after = flask.request.args.get("after") + if after: + print(after) + hits_iter = itertools.dropwhile(lambda hit: hit["title"] != after, hits) + skip = next(hits_iter, None) + if skip: + hits = list(hits_iter) - hits = [h for h in hits if h["title"] not in session_skipped - and h["title"] != from_title and h["title"] != case_flip_first(from_title)] - - if not hits: - return flask.render_template("all_done.html") - - saves_this_session = flask.session.get("saves_by_title", {}).get(from_title, 0) + try: + hit, found = get_best_hit(from_title, hits) + except NoGoodHit: + return flask.render_template("all_done.html") return flask.render_template( "article.html", title=from_title, - redirect_to=redirect_to, total=total, with_link=with_link, + hit_title=hit["title"], hits=hits, + replacement=found["replacement"], + diff=found["diff"], + found=found, url_title=url_title, - saves_this_session=saves_this_session, ) @@ -397,12 +379,7 @@ def do_save(title: str, hit_title: str) -> str: """Update page on Wikipedia.""" token = mediawiki_oauth.get_token() - try: - redirect_to = api.get_wiki_info(title) - except (api.MissingPage, api.MultipleRedirects, api.MediawikiError): - redirect_to = None - - found = get_match(title, hit_title, redirect_to) + found = get_match(title, hit_title, None) summary = ( f"link [[{found['replacement']}]] using [[:en:User:Edward/Find link|Find link]]" @@ -440,20 +417,16 @@ def api_hits() -> werkzeug.wrappers.response.Response: @app.route("/api/1/valid_hit") def api_valid_hit() -> werkzeug.wrappers.response.Response: - """Check if a candidate article has a valid unlinked mention.""" - link_to = flask.request.args["link_to"] + """Return candidates for the given article title.""" link_from = flask.request.args["link_from"] - redirect_to = flask.request.args.get("redirect_to") or None + link_to = flask.request.args["link_to"] try: - found = get_diff(link_to, link_from, redirect_to) + diff, replacement = get_diff(link_to, link_from, None) except NoMatch: - _record_skip(link_to, link_from) return flask.jsonify(valid=False) - except api.MediawikiError as e: - return flask.jsonify(valid=False, error=str(e)) - return flask.jsonify(valid=True, diff=found["diff"], replacement=found["replacement"]) + return flask.jsonify(valid=True, diff=diff, replacement=replacement) @app.route("/favicon.ico")