edward/add-links
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: edward:25056aaf33738249b61b7657287261a0addc995e
Branches Tags
edward:main
..
compare: edward:fe89db11bdff14f1132f1142f966f39c542ced62
Branches Tags
edward:main

No commits in common. "25056aaf33738249b61b7657287261a0addc995e" and "fe89db11bdff14f1132f1142f966f39c542ced62" have entirely different histories.
25056aaf33 ... fe89db11bd

12 changed files with 154 additions and 393 deletions
Download patch file Download diff file Expand all files Collapse all files

24
add_links/api.py
View file

@ -1,5 +1,4 @@
import re import re
import sys
import typing import typing
import requests import requests
@ -73,33 +72,18 @@ webpage_error = (
) )
def _get_active_session() -> requests.sessions.Session:
"""Return OAuth session if one is available in Flask context, else plain session."""
try:
from flask import g
if hasattr(g, "oauth_session") and g.oauth_session is not None:
return g.oauth_session # type: ignore[return-value]
except RuntimeError:
pass
print("WARNING: using unauthenticated session", file=sys.stderr)
return get_session()
def api_get(params: StrDict) -> StrDict: def api_get(params: StrDict) -> StrDict:
"""Make call to Wikipedia API.""" """Make call to Wikipedia API."""
s = _get_active_session() s = get_session()
r = s.get(get_query_url(), params=params) r = s.get(get_query_url(), params=params)
try: try:
ret: StrDict = r.json() ret: StrDict = r.json()
except JSONDecodeError: except JSONDecodeError:
print(f"API request failed: HTTP {r.status_code}", file=sys.stderr)
print(f"Response body: {r.text!r}", file=sys.stderr)
if webpage_error in r.text: if webpage_error in r.text:
raise MediawikiError(webpage_error) raise MediawikiError(webpage_error)
if r.status_code == 429: else:
raise MediawikiError("Wikipedia rate limit exceeded — wait a moment and try again.") raise MediawikiError("unknown error")
raise MediawikiError(f"HTTP {r.status_code}: {r.text[:200]!r}")
check_for_error(ret) check_for_error(ret)
return ret return ret
@ -287,7 +271,7 @@ def call_get_diff(title: str, section_num: int, section_text: str) -> str:
"rvdifftotext": section_text.strip(), "rvdifftotext": section_text.strip(),
} }
s = _get_active_session() s = get_session()
r = s.post(get_query_url(), data=data) r = s.post(get_query_url(), data=data)
try: try:
ret = r.json() ret = r.json()

12
add_links/match.py
View file

@ -78,7 +78,7 @@ re_cite = re.compile(
re.I | re.S, re.I | re.S,
) )
re_cite_template_start = re.compile(r"\{\{(?:cite|citation|short description|gli|defn|annotated link|excerpt|main|see|for)\b", re.I) re_cite_template_start = re.compile(r"\{\{(?:cite|citation|short description|gli|defn|annotated link|excerpt|main|see)\b", re.I)
re_no_param_template = re.compile(r"\{\{[^|{}]+\}\}") re_no_param_template = re.compile(r"\{\{[^|{}]+\}\}")
re_external_link = re.compile(r"\[https?://[^\]]+\]") re_external_link = re.compile(r"\[https?://[^\]]+\]")
# Italic text (work titles in bibliographies). Handles apostrophes in content # Italic text (work titles in bibliographies). Handles apostrophes in content
@ -252,14 +252,6 @@ def add_link(m: re.Match[str], replacement: str, text: str) -> str:
if matched_text.startswith("[[") and matched_text.endswith("|"): if matched_text.startswith("[[") and matched_text.endswith("|"):
return m.re.sub(lambda m: f"[[{replacement}|", text, count=1) return m.re.sub(lambda m: f"[[{replacement}|", text, count=1)
split_links = matched_text.find("]] [[")
if split_links > 0 and m.start() >= 2 and text[m.start() - 2 : m.start()] == "[[":
# Match starts inside one link and continues into the next opening link.
# Link only the text from the first link span and leave the second link as-is.
link_dest = replacement.split("|")[0] if "|" in replacement else replacement
visible = matched_text[:split_links]
return text[: m.start() - 2] + f"[[{link_dest}|{visible}]]" + text[m.start() + split_links + 2 :]
inner_bracket = matched_text.find("[[") inner_bracket = matched_text.find("[[")
if inner_bracket > 0: if inner_bracket > 0:
prefix = matched_text[:inner_bracket].rstrip() prefix = matched_text[:inner_bracket].rstrip()
@ -559,6 +551,4 @@ def get_diff(q: str, title: str, linkto: str | None) -> dict[str, typing.Any]:
) )
found["diff"] = call_get_diff(title, found["section_num"], section_text) found["diff"] = call_get_diff(title, found["section_num"], section_text)
if not found["diff"]:
raise NoMatch
return found return found

35
add_links/mediawiki_oauth.py
View file

@ -1,6 +1,5 @@
"""Wikipedia OAuth.""" """Wikipedia OAuth."""
import sys
import typing import typing
import urllib import urllib
from typing import cast from typing import cast
@ -74,8 +73,9 @@ def api_request(params: typing.Mapping[str, str | int]) -> dict[str, typing.Any]
try: try:
return cast(dict[str, typing.Any], r.json()) return cast(dict[str, typing.Any], r.json())
except Exception: except Exception:
print(f"API request failed: HTTP {r.status_code}", file=sys.stderr) print("text")
print(f"Response body: {r.text!r}", file=sys.stderr) print(r.text)
print("---")
raise raise
@ -99,40 +99,13 @@ def userinfo_call() -> typing.Mapping[str, typing.Any]:
return api_request(params) return api_request(params)
def get_oauth_session() -> OAuth1Session | None:
"""Return an OAuth1Session for the current user, or None if not logged in."""
if "owner_key" not in session or "owner_secret" not in session:
return None
app = current_app
client_key = app.config["CLIENT_KEY"]
client_secret = app.config["CLIENT_SECRET"]
oauth = OAuth1Session(
client_key,
client_secret=client_secret,
resource_owner_key=session["owner_key"],
resource_owner_secret=session["owner_secret"],
)
oauth.headers.update({"User-Agent": ua})
oauth.params = typing.cast(
dict[str, str | int],
{"format": "json", "action": "query", "formatversion": 2},
)
return oauth
def get_username() -> None | str: def get_username() -> None | str:
"""Get the username or None if not logged in.""" """Get the username or None if not logged in."""
if "owner_key" not in session: if "owner_key" not in session:
return None # not authorized return None # not authorized
if "username" not in session: if "username" not in session:
try: reply = userinfo_call()
reply = userinfo_call()
except Exception as e:
print(f"get_username failed, clearing session: {e}", file=sys.stderr)
session.pop("owner_key", None)
session.pop("owner_secret", None)
return None
if "query" not in reply: if "query" not in reply:
return None return None
session["username"] = reply["query"]["userinfo"]["name"] session["username"] = reply["query"]["userinfo"]["name"]

19
static/css/diff.css
View file

@ -5,16 +5,19 @@ span.searchmatch { font-weight: bold; }
table.diff,td.diff-otitle,td.diff-ntitle{background-color:white} table.diff,td.diff-otitle,td.diff-ntitle{background-color:white}
td.diff-otitle,td.diff-ntitle{text-align:center} td.diff-otitle,td.diff-ntitle{text-align:center}
td.diff-marker{width:1.5em;text-align:center;font-weight:bold;font-size:1.25em;padding:0 0.3em} td.diff-marker{text-align:right;font-weight:bold;font-size:1.25em}
td.diff-lineno{font-weight:bold} td.diff-lineno{font-weight:bold}
td.diff-addedline,td.diff-deletedline,td.diff-context{font-size:88%;vertical-align:top;white-space:-moz-pre-wrap;white-space:pre-wrap} td.diff-addedline,td.diff-deletedline,td.diff-context{font-size:88%;vertical-align:top;white-space:-moz-pre-wrap;white-space:pre-wrap}
td.diff-addedline,td.diff-deletedline{border-left:3px solid} td.diff-addedline,td.diff-deletedline{border-style:solid;border-width:1px 1px 1px 4px;border-radius:0.33em}
td.diff-addedline{border-color:#a3d3ff;background:#f0f8ff} td.diff-addedline{border-color:#a3d3ff}
td.diff-deletedline{border-color:#ffe49c;background:#fffaf0} td.diff-deletedline{border-color:#ffe49c}
td.diff-context{color:#555} td.diff-context{background:#f3f3f3;color:#333333;border-style:solid;border-width:1px 1px 1px 4px;border-color:#e6e6e6;border-radius:0.33em}
.diffchange{font-weight:bold;text-decoration:none} .diffchange{font-weight:bold;text-decoration:none}
table.diff{border:none;width:100%;border-spacing:0;border-collapse:collapse;table-layout:auto} table.diff{border:none;width:98%;border-spacing:4px; table-layout:fixed}
td.diff-addedline .diffchange,td.diff-deletedline .diffchange{border-radius:0.33em;padding:0.25em 0}
td.diff-addedline .diffchange{background:#d8ecff} td.diff-addedline .diffchange{background:#d8ecff}
td.diff-deletedline .diffchange{background:#feeec8} td.diff-deletedline .diffchange{background:#feeec8}
table.diff td{padding:0.2em 0.5em} table.diff td{padding:0.33em 0.66em}
table.diff td div{word-wrap:break-word;overflow:auto} table.diff col.diff-marker{width:2%}
table.diff col.diff-content{width:48%}
table.diff td div{ word-wrap:break-word; overflow:auto}

3
static/favicon.svg
View file

@ -1,3 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<text y=".9em" font-size="90">🔗</text>
</svg>
Side by side

Before

Width:  |  Height:  |  Size: 114 B

11
templates/all_done.html
View file

@ -1,11 +1,10 @@
{% extends "base.html" %} {% extends "base.html" %}
{% block title %}All done{% endblock %} {% block title %}Index{% endblock %}
{% block content %} {% block content %}
<div class="container text-center mt-5"> <div class="container">
<h1 class="mb-3">All done</h1> <h1>All done</h1>
<p class="text-muted mb-4">No more candidates found for this article.</p> <div><a href="{{ url_for('index') }}">back to index </a></div>
<a href="{{ url_for('index') }}" class="btn btn-primary">Search another article</a> </div>
</div>
{% endblock %} {% endblock %}

160
templates/article.html
View file

@ -1,152 +1,48 @@
{% extends "base.html" %} {% extends "base.html" %}
{% block title %}{{ title }}{% endblock %} {% block title %}Link '{{ title }}' in '{{ hit_title }}'{% endblock %}
{% block style %} {% block style %}
<link href="{{ url_for("static", filename="css/diff.css") }}" rel="stylesheet"/> <link href="{{ url_for("static", filename="css/diff.css") }}" rel="stylesheet"/>
{% endblock %} {% endblock %}
{% block content %} {% block content %}
<div class="container"> <div class="container">
<nav aria-label="breadcrumb" class="mb-3"> <h1>Link '{{ title }}' in '{{ hit_title }}'</h1>
<ol class="breadcrumb"> <form action="{{ url_for("index") }}">
<li class="breadcrumb-item"><a href="{{ url_for('index') }}">Home</a></li> <input name="q">
<li class="breadcrumb-item active">{{ title }}</li> <input type="submit" value="search">
</ol> </form>
</nav>
<div class="d-flex flex-wrap align-items-baseline gap-3 mb-1"> <div>Username: {{ g.user }}</div>
<h1 class="h4 mb-0">Find links to "{{ title }}"</h1>
<a href="https://en.wikipedia.org/wiki/{{ title }}" target="_blank" class="text-muted small">{{ title }} ↗</a>
{% if redirect_to %}
<span class="text-muted small">→ redirects to <a href="https://en.wikipedia.org/wiki/{{ redirect_to }}" target="_blank">{{ redirect_to }} ↗</a></span>
{% endif %}
</div>
<div class="d-flex gap-3 mb-3 text-muted small"> <div><a href="https://en.wikipedia.org/wiki/{{ title }}" target="_blank">view article</a></div>
<span>{{ total }} mentions total</span>
<span>{{ with_link }} already linked{% if total > 0 %} ({{ "{:.0%}".format(with_link / total) }}){% endif %}</span>
{% if saves_this_session %}
<span class="text-success">{{ saves_this_session }} added this session</span>
{% endif %}
</div>
<div id="search-progress" class="my-4"> <div><a href="{{ url_for('index') }}">back to index </a></div>
<div class="d-flex align-items-center gap-2 text-muted">
<div class="spinner-border spinner-border-sm" role="status">
<span class="visually-hidden">Searching…</span>
</div>
<span id="search-status">Searching…</span>
</div>
</div>
<div id="result" hidden> <div>total: {{ total }}</div>
<div class="d-flex flex-wrap align-items-baseline gap-2 mb-3"> <div>with link: {{ with_link }}</div>
<span class="text-muted small">Adding link in</span> <div>ratio: {{ "{:.1%}".format(with_link / total) }}</div>
<a id="result-hit-link" href="#" target="_blank" class="small"><span id="result-hit-title"></span> ↗</a> {# <div>hit: {{ hit }}</div> #}
</div> <div>replacement: {{ found.replacement }}</div>
<div class="mb-4"> <div>section: {{ found.section }}</div>
<table class="diff" id="diff-table"></table> <table>
</div> {{ diff | safe }}
<form method="POST" class="mb-4"> </table>
<input type="hidden" name="hit" id="hit-input"> <form method="POST">
<div class="d-flex gap-2"> <input type="hidden" name="hit" value="{{ hit_title }}">
<button type="submit" class="btn btn-success">Save edit</button> <div class="my-3">
<a id="skip-link" href="#" class="btn btn-outline-secondary">Skip</a> <input type="submit" class="btn btn-primary" value="save"/>
<a href="{{url_for("article_page", url_title=url_title, after=hit_title)}}" class="btn btn-primary">skip</a>
</div> </div>
</form> </form>
</div>
<div id="all-done" hidden class="text-center mt-4"> <ol>
<p class="text-muted mb-4">No more candidates found for this article.</p>
<a href="{{ url_for('index') }}" class="btn btn-primary">Search another article</a>
</div>
{% if hits %}
<details id="candidates-section" class="border rounded p-3 mt-2">
<summary class="text-muted small" style="cursor:pointer"><span id="candidates-count">{{ hits | length }}</span> candidates</summary>
<ol class="mt-3 mb-0 small" id="candidates-list">
{% for hit in hits %} {% for hit in hits %}
<li class="mb-1" data-title="{{ hit.title }}"> {% set url = url_for("article_page", url_title=url_title, title=hit.title) %}
<a href="{{ url_for("article_page", url_title=url_title, title=hit.title) }}">{{ hit.title }}</a> <li><a href="{{ url }}">{{ hit.title }}</a> &ndash; {{ hit.snippet | safe }}</li>
</li>
{% endfor %} {% endfor %}
</ol> </ol>
</details> </div>
{% endif %}
</div>
{% endblock %} {% endblock %}
{% block script %}
<script>
(function () {
const hits = {{ hits | map(attribute='title') | list | tojson }};
const linkTo = {{ title | tojson }};
const redirectTo = {{ redirect_to | tojson }};
const apiUrl = {{ url_for('api_valid_hit') | tojson }};
const pageUrl = new URL(window.location.href);
const elProgress = document.getElementById('search-progress');
const elStatus = document.getElementById('search-status');
const elResult = document.getElementById('result');
const elAllDone = document.getElementById('all-done');
const elList = document.getElementById('candidates-list');
function removeCandidate(title) {
if (!elList) return;
const li = elList.querySelector(`li[data-title="${CSS.escape(title)}"]`);
if (!li) return;
li.remove();
const elCount = document.getElementById('candidates-count');
if (elCount) elCount.textContent = elList.children.length;
}
async function search() {
for (const hitTitle of hits) {
elStatus.textContent = `Checking "${hitTitle}"…`;
let data;
try {
const params = new URLSearchParams({ link_to: linkTo, link_from: hitTitle });
if (redirectTo) params.append('redirect_to', redirectTo);
const resp = await fetch(apiUrl + '?' + params);
if (!resp.ok) continue;
data = await resp.json();
} catch (e) {
continue;
}
if (!data.valid) { removeCandidate(hitTitle); continue; }
elProgress.hidden = true;
document.getElementById('result-hit-title').textContent = hitTitle;
document.getElementById('result-hit-link').href =
'https://en.wikipedia.org/wiki/' + encodeURIComponent(hitTitle.replace(/ /g, '_'));
document.getElementById('diff-table').innerHTML = data.diff;
document.getElementById('hit-input').value = hitTitle;
const skipUrl = new URL(pageUrl);
skipUrl.searchParams.delete('title');
skipUrl.searchParams.set('after', hitTitle);
document.getElementById('skip-link').href = skipUrl.toString();
const currentUrl = new URL(pageUrl);
currentUrl.searchParams.delete('after');
currentUrl.searchParams.set('title', hitTitle);
history.replaceState(null, '', currentUrl.toString());
elResult.hidden = false;
return;
}
elProgress.hidden = true;
const elCandidates = document.getElementById('candidates-section');
if (elCandidates) elCandidates.hidden = true;
elAllDone.hidden = false;
}
search();
}());
</script>
{% endblock %}

33
templates/base.html
View file

@ -2,38 +2,21 @@
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<link href="{{ url_for("static", filename="bootstrap5/css/bootstrap.min.css") }}" rel="stylesheet"> <link href="{{ url_for("static", filename="bootstrap/css/bootstrap.min.css") }}" rel="stylesheet">
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" type="image/svg+xml" href="{{ url_for('static', filename='favicon.svg') }}">
<title>{% block title %}{% endblock %} – Missing Link</title> <title>
{% block title %}{% endblock %}
</title>
{% block style %}{% endblock %} {% block style %}{% endblock %}
</head> </head>
<body> <body>
<nav class="navbar navbar-expand-md navbar-dark bg-dark mb-4">
<div class="container">
<a class="navbar-brand fw-semibold" href="{{ url_for('index') }}">Missing Link</a>
<div class="d-flex align-items-center gap-2">
<form class="d-flex" action="{{ url_for('index') }}">
<input class="form-control form-control-sm me-2" name="q" placeholder="Article title…" style="width:240px">
<button class="btn btn-outline-light btn-sm" type="submit">Go</button>
</form>
{% if g.user %}
<span class="text-light small opacity-75">{{ g.user }}</span>
{% if session.get("saves") %}
<span class="badge bg-success">{{ session["saves"] }} saved</span>
{% endif %}
<a class="btn btn-outline-light btn-sm" href="{{ url_for('oauth_disconnect') }}">Log out</a>
{% else %}
<a class="btn btn-outline-light btn-sm" href="{{ url_for('start_oauth') }}">Log in with Wikipedia</a>
{% endif %}
</div>
</div>
</nav>
{% block content %}{% endblock %} {% block content %}{% endblock %}
<script src="{{ url_for("static", filename="bootstrap5/js/bootstrap.bundle.min.js") }}"></script> <script src="{{ url_for("static", filename="bootstrap/js/bootstrap.bundle.min.js")}}></script>
{% block script %}{% endblock %} {% block script %}{% endblock %}
</body> </body>
</html> </html>

17
templates/error.html
View file

@ -1,17 +0,0 @@
{% extends "base.html" %}
{% block title %}Error{% endblock %}
{% block content %}
<div class="container mt-5">
<div class="row justify-content-center">
<div class="col-md-7">
<div class="alert alert-danger">
<h4 class="alert-heading">Something went wrong</h4>
<p class="mb-0"><code>{{ message }}</code></p>
</div>
<a href="{{ url_for('index') }}" class="btn btn-outline-secondary btn-sm">Back to home</a>
</div>
</div>
</div>
{% endblock %}

55
templates/index.html
View file

@ -1,44 +1,25 @@
{% extends "base.html" %} {% extends "base.html" %}
{% block title %}Missing Link{% endblock %} {% block title %}Index{% endblock %}
{% block content %} {% block content %}
<div class="container"> <div class="container">
<div class="row justify-content-center mt-5"> <h1>Index</h1>
<div class="col-md-6 text-center"> <form>
<h1 class="mb-2">Missing Link</h1> <input name="q">
<p class="text-muted mb-4">Find unlinked mentions of a Wikipedia article and add the links.</p> <input type="submit" value="search">
<form class="d-flex gap-2 justify-content-center" action="{{ url_for('index') }}"> </form>
<input class="form-control" name="q" placeholder="Article title…" style="max-width:360px" autofocus>
<button class="btn btn-primary" type="submit">Search</button>
</form>
</div>
</div>
{% if debug %} <div>Username: {{ g.user }}</div>
<div class="row mt-5">
<div class="col"> <table class="table w-auto">
<h2 class="h6 text-muted text-uppercase mb-3">Examples</h2> {% for item in examples %}
<table class="table table-sm table-hover w-auto"> <tr>
<thead class="table-light"> <td><a href="{{ article_url(item.title) }}">{{ item.title }}</a></td>
<tr> <td>{{ item.total }}</td>
<th>Article</th> <td>{{ "{:.1%}".format(item.with_links / item.total) }}</td>
<th class="text-end">Total</th> </tr>
<th class="text-end">% linked</th> {% endfor %}
</tr> </table>
</thead>
<tbody>
{% for item in examples %}
<tr>
<td><a href="{{ article_url(item.title) }}">{{ item.title }}</a></td>
<td class="text-end text-muted">{{ item.total }}</td>
<td class="text-end text-muted">{{ "{:.0%}".format(item.with_links / item.total) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div> </div>
{% endif %}
</div>
{% endblock %} {% endblock %}

11
templates/save_done.html
View file

@ -1,11 +1,10 @@
{% extends "base.html" %} {% extends "base.html" %}
{% block title %}Edit saved{% endblock %} {% block title %}Index{% endblock %}
{% block content %} {% block content %}
<div class="container text-center mt-5"> <div class="container">
<h1 class="mb-3">Edit saved</h1> <h1>Save done</h1>
<p class="text-muted mb-4">Your edit has been saved to Wikipedia.</p> <div>Save is complete.</div>
<a href="{{ url_for('index') }}" class="btn btn-primary">Search another article</a> </div>
</div>
{% endblock %} {% endblock %}

167
web_view.py
View file

@ -4,13 +4,11 @@ import html
import itertools import itertools
import json import json
import re import re
import sys
import typing import typing
import flask import flask
import werkzeug import werkzeug
from requests_oauthlib import OAuth1Session from requests_oauthlib import OAuth1Session
from requests_oauthlib.oauth1_session import TokenRequestDenied
from werkzeug.wrappers.response import Response from werkzeug.wrappers.response import Response
from add_links import api, core, mediawiki_api, mediawiki_oauth from add_links import api, core, mediawiki_api, mediawiki_oauth
@ -96,20 +94,14 @@ def search_count(q: str) -> int:
return get_hit_count(article_title_to_search_query(q)) - 1 return get_hit_count(article_title_to_search_query(q)) - 1
def search_count_with_link(q: str, redirect_to: str | None = None) -> int: def search_count_with_link(q: str) -> int:
"""Articles in Wikipedia that include this search term and a link.""" """Articles in Wikipedia that include this search term and a link."""
count = get_hit_count(article_title_to_search_query(q) + f' linksto:"{q}"') return get_hit_count(article_title_to_search_query(q) + f' linksto:"{q}"')
if redirect_to:
count += get_hit_count(article_title_to_search_query(q) + f' linksto:"{redirect_to}"')
return count
def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hit]]: def search_no_link(q: str) -> tuple[int, list[Hit]]:
"""Search for mentions of article title with no link included.""" """Search for mentions of article title with no link included."""
exclude = f' -linksto:"{q}"' query = run_search(article_title_to_search_query(q) + f' -linksto:"{q}"', "max")
if redirect_to:
exclude += f' -linksto:"{redirect_to}"'
query = run_search(article_title_to_search_query(q) + exclude, "max")
return (query["searchinfo"]["totalhits"], query["search"]) return (query["searchinfo"]["totalhits"], query["search"])
@ -117,7 +109,6 @@ def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hi
def global_user() -> None: def global_user() -> None:
"""Make username available everywhere.""" """Make username available everywhere."""
flask.g.user = mediawiki_oauth.get_username() flask.g.user = mediawiki_oauth.get_username()
flask.g.oauth_session = mediawiki_oauth.get_oauth_session()
@app.route("/") @app.route("/")
@ -127,20 +118,17 @@ def index() -> str | Response:
url = flask.url_for("oauth_callback", **flask.request.args) # type: ignore url = flask.url_for("oauth_callback", **flask.request.args) # type: ignore
return flask.redirect(url) return flask.redirect(url)
examples = load_examples()
examples.sort(
key=lambda i: float(i["with_links"]) / float(i["total"]), reverse=True
)
if q := flask.request.args.get("q"): if q := flask.request.args.get("q"):
if q_trimmed := q.strip(): if q_trimmed := q.strip():
return flask.redirect(article_url(q_trimmed)) return flask.redirect(article_url(q_trimmed))
debug = flask.request.args.get("debug")
examples: list[dict[str, str | int]] = []
if debug:
examples = load_examples()
examples.sort(
key=lambda i: float(i["with_links"]) / float(i["total"]), reverse=True
)
return flask.render_template( return flask.render_template(
"index.html", examples=examples, article_url=article_url, debug=debug "index.html", examples=examples, article_url=article_url
) )
@ -199,12 +187,7 @@ def start_oauth() -> Response:
oauth = OAuth1Session(client_key, client_secret=client_secret, callback_uri="oob") oauth = OAuth1Session(client_key, client_secret=client_secret, callback_uri="oob")
oauth.headers.update({"User-Agent": api.ua}) oauth.headers.update({"User-Agent": api.ua})
try: fetch_response = oauth.fetch_request_token(request_token_url)
fetch_response = oauth.fetch_request_token(request_token_url)
except TokenRequestDenied as e:
return flask.make_response(
flask.render_template("error.html", message=str(e)), 502
)
flask.session["owner_key"] = fetch_response.get("oauth_token") flask.session["owner_key"] = fetch_response.get("oauth_token")
flask.session["owner_secret"] = fetch_response.get("oauth_token_secret") flask.session["owner_secret"] = fetch_response.get("oauth_token_secret")
@ -246,8 +229,7 @@ def oauth_callback() -> werkzeug.wrappers.response.Response:
flask.session["owner_key"] = oauth_tokens.get("oauth_token") flask.session["owner_key"] = oauth_tokens.get("oauth_token")
flask.session["owner_secret"] = oauth_tokens.get("oauth_token_secret") flask.session["owner_secret"] = oauth_tokens.get("oauth_token_secret")
username = mediawiki_oauth.get_username() print("login successful")
print(f"login successful: {username}", file=sys.stderr)
next_page = flask.session.get("after_login") next_page = flask.session.get("after_login")
return flask.redirect(next_page if next_page else flask.url_for("index")) return flask.redirect(next_page if next_page else flask.url_for("index"))
@ -299,16 +281,31 @@ def match_type(q: str, snippet: str) -> str | None:
return match return match
class NoGoodHit(Exception):
"""No good hit."""
def _record_skip(from_title: str, hit_title: str) -> None: def get_best_hit(title: str, hits: list[Hit]) -> tuple[Hit, dict[str, typing.Any]]:
"""Record that a candidate was skipped or saved for this article.""" """Find the best hit within the search results."""
skipped: dict[str, list[str]] = flask.session.get("skipped", {}) for hit in hits:
article_skipped = skipped.get(from_title, []) if hit["title"].lower() == title.lower():
if hit_title not in article_skipped: continue
skipped[from_title] = article_skipped + [hit_title] # if match_type(title, hit["snippet"]) != "exact":
flask.session["skipped"] = skipped # continue
flask.session.modified = True
try:
print(f'get diff: {hit["title"]}, {title}')
found = get_diff(title, hit["title"], None)
except NoMatch:
print("no match")
continue
except api.MediawikiError as e:
print(f"MediawikiError for {hit['title']!r}: {e}")
continue
return (hit, found)
raise NoGoodHit
def handle_post(url_title: str) -> Response: def handle_post(url_title: str) -> Response:
@ -319,16 +316,8 @@ def handle_post(url_title: str) -> Response:
do_save(from_title, hit_title) do_save(from_title, hit_title)
except mediawiki_oauth.LoginNeeded: except mediawiki_oauth.LoginNeeded:
return flask.redirect(flask.url_for("start_oauth")) return flask.redirect(flask.url_for("start_oauth"))
except (mediawiki_api.APIError, api.MediawikiError) as e: except mediawiki_api.APIError as e:
return flask.make_response( return flask.make_response(f"Save failed: {e}", 502)
flask.render_template("error.html", message=f"Save failed: {e}"), 502
)
flask.session["saves"] = flask.session.get("saves", 0) + 1
saves_by_title: dict[str, int] = flask.session.get("saves_by_title", {})
saves_by_title[from_title] = saves_by_title.get(from_title, 0) + 1
flask.session["saves_by_title"] = saves_by_title
flask.session.modified = True
_record_skip(from_title, hit_title)
return flask.redirect( return flask.redirect(
flask.url_for("article_page", url_title=url_title, after=hit_title) flask.url_for("article_page", url_title=url_title, after=hit_title)
) )
@ -341,55 +330,48 @@ def article_page(url_title: str) -> str | Response:
return handle_post(url_title) return handle_post(url_title)
from_title = url_title.replace("_", " ").strip() from_title = url_title.replace("_", " ").strip()
article_title = flask.request.args.get("title")
try: total = search_count(from_title)
redirect_to = api.get_wiki_info(from_title) with_link = search_count_with_link(from_title)
except (api.MissingPage, api.MultipleRedirects, api.MediawikiError):
redirect_to = None
try: no_link_count, hits = search_no_link(from_title)
total = search_count(from_title)
with_link = search_count_with_link(from_title, redirect_to)
_no_link_count, hits = search_no_link(from_title, redirect_to)
except api.MediawikiError as e:
return flask.make_response(
flask.render_template("error.html", message=str(e)), 502
)
# Filter out candidates already processed this session by_title = {hit["title"]: hit for hit in hits}
session_skipped: set[str] = set(
flask.session.get("skipped", {}).get(from_title, [])
)
# If a specific candidate was requested, move it to the front found = None
title_param = flask.request.args.get("title") if article_title in by_title:
if title_param: hit = by_title[article_title]
hits = [h for h in hits if h["title"] == title_param] + \ try:
[h for h in hits if h["title"] != title_param] found = get_diff(from_title, hit["title"], None)
except NoMatch:
pass
# Record and apply explicit skip-past if not found:
after = flask.request.args.get("after") after = flask.request.args.get("after")
if after: if after:
_record_skip(from_title, after) print(after)
session_skipped.add(after) hits_iter = itertools.dropwhile(lambda hit: hit["title"] != after, hits)
skip = next(hits_iter, None)
if skip:
hits = list(hits_iter)
hits = [h for h in hits if h["title"] not in session_skipped try:
and h["title"] != from_title and h["title"] != case_flip_first(from_title)] hit, found = get_best_hit(from_title, hits)
except NoGoodHit:
if not hits: return flask.render_template("all_done.html")
return flask.render_template("all_done.html")
saves_this_session = flask.session.get("saves_by_title", {}).get(from_title, 0)
return flask.render_template( return flask.render_template(
"article.html", "article.html",
title=from_title, title=from_title,
redirect_to=redirect_to,
total=total, total=total,
with_link=with_link, with_link=with_link,
hit_title=hit["title"],
hits=hits, hits=hits,
replacement=found["replacement"],
diff=found["diff"],
found=found,
url_title=url_title, url_title=url_title,
saves_this_session=saves_this_session,
) )
@ -397,12 +379,7 @@ def do_save(title: str, hit_title: str) -> str:
"""Update page on Wikipedia.""" """Update page on Wikipedia."""
token = mediawiki_oauth.get_token() token = mediawiki_oauth.get_token()
try: found = get_match(title, hit_title, None)
redirect_to = api.get_wiki_info(title)
except (api.MissingPage, api.MultipleRedirects, api.MediawikiError):
redirect_to = None
found = get_match(title, hit_title, redirect_to)
summary = ( summary = (
f"link [[{found['replacement']}]] using [[:en:User:Edward/Find link|Find link]]" f"link [[{found['replacement']}]] using [[:en:User:Edward/Find link|Find link]]"
@ -440,20 +417,16 @@ def api_hits() -> werkzeug.wrappers.response.Response:
@app.route("/api/1/valid_hit") @app.route("/api/1/valid_hit")
def api_valid_hit() -> werkzeug.wrappers.response.Response: def api_valid_hit() -> werkzeug.wrappers.response.Response:
"""Check if a candidate article has a valid unlinked mention.""" """Return candidates for the given article title."""
link_to = flask.request.args["link_to"]
link_from = flask.request.args["link_from"] link_from = flask.request.args["link_from"]
redirect_to = flask.request.args.get("redirect_to") or None link_to = flask.request.args["link_to"]
try: try:
found = get_diff(link_to, link_from, redirect_to) diff, replacement = get_diff(link_to, link_from, None)
except NoMatch: except NoMatch:
_record_skip(link_to, link_from)
return flask.jsonify(valid=False) return flask.jsonify(valid=False)
except api.MediawikiError as e:
return flask.jsonify(valid=False, error=str(e))
return flask.jsonify(valid=True, diff=found["diff"], replacement=found["replacement"]) return flask.jsonify(valid=True, diff=diff, replacement=replacement)
@app.route("/favicon.ico") @app.route("/favicon.ico")