edward/MissingLink
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity

Skip invalid and protected candidates

Browse source
This commit is contained in:
Edward Betts 2026-05-18 23:20:42 +01:00
parent bd0c1a26c2
commit 93f49c5554
3 changed files with 223 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

53
templates/article.html
View file

@ -85,6 +85,12 @@
const redirectTo = {{ redirect_to | tojson }};
const apiUrl = {{ url_for('api_valid_hit') | tojson }};
const pageUrl = new URL(window.location.href);
const cleanPageUrl = new URL(pageUrl);
cleanPageUrl.searchParams.delete('title');
cleanPageUrl.searchParams.delete('after');
cleanPageUrl.searchParams.delete('skip');
const storageKey = 'missinglink.skipped.' + linkTo;
const skippedTitles = [];
const elProgress = document.getElementById('search-progress');
const elStatus = document.getElementById('search-status');
@ -101,8 +107,39 @@
if (elCount) elCount.textContent = elList.children.length;
}
function loadSkippedTitles() {
try {
const stored = JSON.parse(localStorage.getItem(storageKey) || '[]');
if (Array.isArray(stored)) {
for (const title of stored) rememberSkipped(title, false);
}
} catch (e) {
try { localStorage.removeItem(storageKey); } catch (e) {}
}
for (const title of {{ skipped_titles | tojson }}) rememberSkipped(title, false);
saveSkippedTitles();
}
function saveSkippedTitles() {
try {
localStorage.setItem(storageKey, JSON.stringify(skippedTitles));
} catch (e) {}
}
function rememberSkipped(title, save = true) {
if (!title || skippedTitles.includes(title)) return;
skippedTitles.push(title);
if (save) saveSkippedTitles();
}
async function search() {
loadSkippedTitles();
for (const hitTitle of skippedTitles) removeCandidate(hitTitle);
for (const hitTitle of hits) {
if (skippedTitles.includes(hitTitle)) continue;
elStatus.textContent = `Checking "${hitTitle}"…`;
let data;
@ -116,7 +153,11 @@
continue;
}
if (!data.valid) { removeCandidate(hitTitle); continue; }
if (!data.valid) {
rememberSkipped(hitTitle);
removeCandidate(hitTitle);
continue;
}
elProgress.hidden = true;
@ -126,13 +167,15 @@
document.getElementById('diff-table').innerHTML = data.diff;
document.getElementById('hit-input').value = hitTitle;
const skipUrl = new URL(pageUrl);
skipUrl.searchParams.delete('title');
skipUrl.searchParams.set('after', hitTitle);
document.getElementById('skip-link').href = skipUrl.toString();
const elSkipLink = document.getElementById('skip-link');
elSkipLink.href = cleanPageUrl.toString();
elSkipLink.onclick = function () {
rememberSkipped(hitTitle);
};
const currentUrl = new URL(pageUrl);
currentUrl.searchParams.delete('after');
currentUrl.searchParams.delete('skip');
currentUrl.searchParams.set('title', hitTitle);
history.replaceState(null, '', currentUrl.toString());

113
test_web_view.py Normal file
View file

@ -0,0 +1,113 @@
import unittest
from unittest.mock import patch
import web_view
def hit(title: str) -> web_view.Hit:
return {
"ns": 0,
"title": title,
"pageid": 1,
"size": 1,
"wordcount": 1,
"snippet": "",
"timestamp": "",
}
class ArticlePageSkipTests(unittest.TestCase):
def setUp(self) -> None:
web_view.app.config["TESTING"] = True
self.client = web_view.app.test_client()
def test_session_skips_are_left_out_of_candidates(self) -> None:
hits = [
hit("Mass racial violence in the United States"),
hit("National Register of Historic Places listings in Brookhaven (town), New York"),
hit("Useful candidate"),
]
with self.client.session_transaction() as session:
session["skipped"] = {
"African-American neighborhood": [
"Mass racial violence in the United States",
"National Register of Historic Places listings in Brookhaven (town), New York",
]
}
with (
patch("web_view.api.get_wiki_info", return_value=None),
patch("web_view.search_count", return_value=3),
patch("web_view.search_count_with_link", return_value=0),
patch("web_view.search_no_link", return_value=(3, hits)),
):
response = self.client.get("/link/African-American_neighborhood")
body = response.get_data(as_text=True)
self.assertEqual(response.status_code, 200)
self.assertNotIn(
'data-title="Mass racial violence in the United States"', body
)
self.assertNotIn(
'data-title="National Register of Historic Places listings in Brookhaven (town), New York"',
body,
)
self.assertIn("Useful candidate", body)
class ProtectedCandidateTests(unittest.TestCase):
def test_edit_protected_hits_are_filtered_out(self) -> None:
hits = [hit("Editable"), hit("Protected")]
hits[0]["pageid"] = 10
hits[1]["pageid"] = 20
with patch(
"web_view.get_page_info_by_id",
return_value={
10: {"pageid": 10, "title": "Editable", "protection": []},
20: {
"pageid": 20,
"title": "Protected",
"protection": [
{"type": "edit", "level": "sysop", "expiry": "infinity"}
],
},
},
):
filtered = web_view.filter_edit_protected_hits(hits)
self.assertEqual([item["title"] for item in filtered], ["Editable"])
def test_search_no_link_filters_edit_protected_results(self) -> None:
hits = [hit("Editable"), hit("Protected")]
hits[0]["pageid"] = 10
hits[1]["pageid"] = 20
with (
patch(
"web_view.run_search",
return_value={"searchinfo": {"totalhits": 2}, "search": hits},
),
patch(
"web_view.get_page_info_by_id",
return_value={
10: {"pageid": 10, "title": "Editable", "protection": []},
20: {
"pageid": 20,
"title": "Protected",
"protection": [
{"type": "edit", "level": "sysop", "expiry": "infinity"}
],
},
},
),
):
totalhits, filtered = web_view.search_no_link("Target")
self.assertEqual(totalhits, 2)
self.assertEqual([item["title"] for item in filtered], ["Editable"])
if __name__ == "__main__":
unittest.main()

74
web_view.py
View file

@ -1,7 +1,6 @@
#!/usr/bin/python3
import html
import itertools
import json
import re
import sys
@ -37,6 +36,14 @@ class Hit(typing.TypedDict):
timestamp: str
class PageInfo(typing.TypedDict, total=False):
"""Wikipedia page metadata."""
pageid: int
title: str
protection: list[dict[str, str]]
def render_error(message: str) -> str:
"""Render shared error page."""
return flask.render_template("error.html", message=message)
@ -115,13 +122,60 @@ def search_count_with_link(q: str, redirect_to: str | None = None) -> int:
return count
def chunks(items: list[int], size: int) -> typing.Iterator[list[int]]:
"""Yield fixed-size chunks from a list."""
for offset in range(0, len(items), size):
yield items[offset : offset + size]
def get_page_info_by_id(pageids: list[int]) -> dict[int, PageInfo]:
"""Fetch metadata for pages by page ID."""
if not pageids:
return {}
pages: dict[int, PageInfo] = {}
for pageid_chunk in chunks(pageids, 50):
ret = api.api_get(
{
"prop": "info",
"inprop": "protection",
"pageids": "|".join(str(pageid) for pageid in pageid_chunk),
}
)
for page in ret["query"]["pages"]:
pageid = typing.cast(int, page["pageid"])
pages[pageid] = typing.cast(PageInfo, page)
return pages
def is_edit_protected(page: PageInfo | None) -> bool:
"""Check if a page has edit protection."""
if not page:
return False
return any(
protection.get("type") == "edit" or protection.get("action") == "edit"
for protection in page.get("protection", [])
)
def filter_edit_protected_hits(hits: list[Hit]) -> list[Hit]:
"""Remove candidates that cannot be edited because of page protection."""
page_info = get_page_info_by_id([hit["pageid"] for hit in hits])
return [
hit for hit in hits if not is_edit_protected(page_info.get(hit["pageid"]))
]
def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hit]]:
"""Search for mentions of article title with no link included."""
exclude = f' -linksto:"{q}"'
if redirect_to:
exclude += f' -linksto:"{redirect_to}"'
query = run_search(article_title_to_search_query(q) + exclude, "max")
return (query["searchinfo"]["totalhits"], query["search"])
return (
query["searchinfo"]["totalhits"],
filter_edit_protected_hits(query["search"]),
)
@app.before_request
@ -317,6 +371,9 @@ def match_type(q: str, snippet: str) -> str | None:
def _record_skip(from_title: str, hit_title: str) -> None:
"""Record that a candidate was skipped or saved for this article."""
hit_title = hit_title.strip()
if not hit_title:
return
skipped: dict[str, list[str]] = flask.session.get("skipped", {})
article_skipped = skipped.get(from_title, [])
if hit_title not in article_skipped:
@ -371,10 +428,8 @@ def article_page(url_title: str) -> str | Response:
render_mediawiki_error(e), 502
)
# Filter out candidates already processed this session
session_skipped: set[str] = set(
flask.session.get("skipped", {}).get(from_title, [])
)
article_skipped = flask.session.get("skipped", {}).get(from_title, [])
session_skipped: set[str] = set(article_skipped)
# If a specific candidate was requested, move it to the front
title_param = flask.request.args.get("title")
@ -382,12 +437,6 @@ def article_page(url_title: str) -> str | Response:
hits = [h for h in hits if h["title"] == title_param] + \
[h for h in hits if h["title"] != title_param]
# Record and apply explicit skip-past
after = flask.request.args.get("after")
if after:
_record_skip(from_title, after)
session_skipped.add(after)
hits = [h for h in hits if h["title"] not in session_skipped
and h["title"] != from_title and h["title"] != case_flip_first(from_title)]
@ -404,6 +453,7 @@ def article_page(url_title: str) -> str | Response:
with_link=with_link,
hits=hits,
url_title=url_title,
skipped_titles=article_skipped,
saves_this_session=saves_this_session,
)