diff --git a/templates/article.html b/templates/article.html
index a96770c..18269c4 100644
--- a/templates/article.html
+++ b/templates/article.html
@@ -85,6 +85,12 @@
const redirectTo = {{ redirect_to | tojson }};
const apiUrl = {{ url_for('api_valid_hit') | tojson }};
const pageUrl = new URL(window.location.href);
+ const cleanPageUrl = new URL(pageUrl);
+ cleanPageUrl.searchParams.delete('title');
+ cleanPageUrl.searchParams.delete('after');
+ cleanPageUrl.searchParams.delete('skip');
+ const storageKey = 'missinglink.skipped.' + linkTo;
+ const skippedTitles = [];
const elProgress = document.getElementById('search-progress');
const elStatus = document.getElementById('search-status');
@@ -101,8 +107,39 @@
if (elCount) elCount.textContent = elList.children.length;
}
+ function loadSkippedTitles() {
+ try {
+ const stored = JSON.parse(localStorage.getItem(storageKey) || '[]');
+ if (Array.isArray(stored)) {
+ for (const title of stored) rememberSkipped(title, false);
+ }
+ } catch (e) {
+ try { localStorage.removeItem(storageKey); } catch (e) {}
+ }
+
+ for (const title of {{ skipped_titles | tojson }}) rememberSkipped(title, false);
+ saveSkippedTitles();
+ }
+
+ function saveSkippedTitles() {
+ try {
+ localStorage.setItem(storageKey, JSON.stringify(skippedTitles));
+ } catch (e) {}
+ }
+
+ function rememberSkipped(title, save = true) {
+ if (!title || skippedTitles.includes(title)) return;
+ skippedTitles.push(title);
+ if (save) saveSkippedTitles();
+ }
+
async function search() {
+ loadSkippedTitles();
+ for (const hitTitle of skippedTitles) removeCandidate(hitTitle);
+
for (const hitTitle of hits) {
+ if (skippedTitles.includes(hitTitle)) continue;
+
elStatus.textContent = `Checking "${hitTitle}"…`;
let data;
@@ -116,7 +153,11 @@
continue;
}
- if (!data.valid) { removeCandidate(hitTitle); continue; }
+ if (!data.valid) {
+ rememberSkipped(hitTitle);
+ removeCandidate(hitTitle);
+ continue;
+ }
elProgress.hidden = true;
@@ -126,13 +167,15 @@
document.getElementById('diff-table').innerHTML = data.diff;
document.getElementById('hit-input').value = hitTitle;
- const skipUrl = new URL(pageUrl);
- skipUrl.searchParams.delete('title');
- skipUrl.searchParams.set('after', hitTitle);
- document.getElementById('skip-link').href = skipUrl.toString();
+ const elSkipLink = document.getElementById('skip-link');
+ elSkipLink.href = cleanPageUrl.toString();
+ elSkipLink.onclick = function () {
+ rememberSkipped(hitTitle);
+ };
const currentUrl = new URL(pageUrl);
currentUrl.searchParams.delete('after');
+ currentUrl.searchParams.delete('skip');
currentUrl.searchParams.set('title', hitTitle);
history.replaceState(null, '', currentUrl.toString());
diff --git a/test_web_view.py b/test_web_view.py
new file mode 100644
index 0000000..af1e203
--- /dev/null
+++ b/test_web_view.py
@@ -0,0 +1,113 @@
+import unittest
+from unittest.mock import patch
+
+import web_view
+
+
+def hit(title: str) -> web_view.Hit:
+ return {
+ "ns": 0,
+ "title": title,
+ "pageid": 1,
+ "size": 1,
+ "wordcount": 1,
+ "snippet": "",
+ "timestamp": "",
+ }
+
+
+class ArticlePageSkipTests(unittest.TestCase):
+ def setUp(self) -> None:
+ web_view.app.config["TESTING"] = True
+ self.client = web_view.app.test_client()
+
+ def test_session_skips_are_left_out_of_candidates(self) -> None:
+ hits = [
+ hit("Mass racial violence in the United States"),
+ hit("National Register of Historic Places listings in Brookhaven (town), New York"),
+ hit("Useful candidate"),
+ ]
+
+ with self.client.session_transaction() as session:
+ session["skipped"] = {
+ "African-American neighborhood": [
+ "Mass racial violence in the United States",
+ "National Register of Historic Places listings in Brookhaven (town), New York",
+ ]
+ }
+
+ with (
+ patch("web_view.api.get_wiki_info", return_value=None),
+ patch("web_view.search_count", return_value=3),
+ patch("web_view.search_count_with_link", return_value=0),
+ patch("web_view.search_no_link", return_value=(3, hits)),
+ ):
+ response = self.client.get("/link/African-American_neighborhood")
+
+ body = response.get_data(as_text=True)
+ self.assertEqual(response.status_code, 200)
+ self.assertNotIn(
+ 'data-title="Mass racial violence in the United States"', body
+ )
+ self.assertNotIn(
+ 'data-title="National Register of Historic Places listings in Brookhaven (town), New York"',
+ body,
+ )
+ self.assertIn("Useful candidate", body)
+
+
+class ProtectedCandidateTests(unittest.TestCase):
+ def test_edit_protected_hits_are_filtered_out(self) -> None:
+ hits = [hit("Editable"), hit("Protected")]
+ hits[0]["pageid"] = 10
+ hits[1]["pageid"] = 20
+
+ with patch(
+ "web_view.get_page_info_by_id",
+ return_value={
+ 10: {"pageid": 10, "title": "Editable", "protection": []},
+ 20: {
+ "pageid": 20,
+ "title": "Protected",
+ "protection": [
+ {"type": "edit", "level": "sysop", "expiry": "infinity"}
+ ],
+ },
+ },
+ ):
+ filtered = web_view.filter_edit_protected_hits(hits)
+
+ self.assertEqual([item["title"] for item in filtered], ["Editable"])
+
+ def test_search_no_link_filters_edit_protected_results(self) -> None:
+ hits = [hit("Editable"), hit("Protected")]
+ hits[0]["pageid"] = 10
+ hits[1]["pageid"] = 20
+
+ with (
+ patch(
+ "web_view.run_search",
+ return_value={"searchinfo": {"totalhits": 2}, "search": hits},
+ ),
+ patch(
+ "web_view.get_page_info_by_id",
+ return_value={
+ 10: {"pageid": 10, "title": "Editable", "protection": []},
+ 20: {
+ "pageid": 20,
+ "title": "Protected",
+ "protection": [
+ {"type": "edit", "level": "sysop", "expiry": "infinity"}
+ ],
+ },
+ },
+ ),
+ ):
+ totalhits, filtered = web_view.search_no_link("Target")
+
+ self.assertEqual(totalhits, 2)
+ self.assertEqual([item["title"] for item in filtered], ["Editable"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/web_view.py b/web_view.py
index 6f7c9c5..748857d 100755
--- a/web_view.py
+++ b/web_view.py
@@ -1,7 +1,6 @@
#!/usr/bin/python3
import html
-import itertools
import json
import re
import sys
@@ -37,6 +36,14 @@ class Hit(typing.TypedDict):
timestamp: str
+class PageInfo(typing.TypedDict, total=False):
+ """Wikipedia page metadata."""
+
+ pageid: int
+ title: str
+ protection: list[dict[str, str]]
+
+
def render_error(message: str) -> str:
"""Render shared error page."""
return flask.render_template("error.html", message=message)
@@ -115,13 +122,60 @@ def search_count_with_link(q: str, redirect_to: str | None = None) -> int:
return count
+def chunks(items: list[int], size: int) -> typing.Iterator[list[int]]:
+ """Yield fixed-size chunks from a list."""
+ for offset in range(0, len(items), size):
+ yield items[offset : offset + size]
+
+
+def get_page_info_by_id(pageids: list[int]) -> dict[int, PageInfo]:
+ """Fetch metadata for pages by page ID."""
+ if not pageids:
+ return {}
+
+ pages: dict[int, PageInfo] = {}
+ for pageid_chunk in chunks(pageids, 50):
+ ret = api.api_get(
+ {
+ "prop": "info",
+ "inprop": "protection",
+ "pageids": "|".join(str(pageid) for pageid in pageid_chunk),
+ }
+ )
+ for page in ret["query"]["pages"]:
+ pageid = typing.cast(int, page["pageid"])
+ pages[pageid] = typing.cast(PageInfo, page)
+ return pages
+
+
+def is_edit_protected(page: PageInfo | None) -> bool:
+ """Check if a page has edit protection."""
+ if not page:
+ return False
+ return any(
+ protection.get("type") == "edit" or protection.get("action") == "edit"
+ for protection in page.get("protection", [])
+ )
+
+
+def filter_edit_protected_hits(hits: list[Hit]) -> list[Hit]:
+ """Remove candidates that cannot be edited because of page protection."""
+ page_info = get_page_info_by_id([hit["pageid"] for hit in hits])
+ return [
+ hit for hit in hits if not is_edit_protected(page_info.get(hit["pageid"]))
+ ]
+
+
def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hit]]:
"""Search for mentions of article title with no link included."""
exclude = f' -linksto:"{q}"'
if redirect_to:
exclude += f' -linksto:"{redirect_to}"'
query = run_search(article_title_to_search_query(q) + exclude, "max")
- return (query["searchinfo"]["totalhits"], query["search"])
+ return (
+ query["searchinfo"]["totalhits"],
+ filter_edit_protected_hits(query["search"]),
+ )
@app.before_request
@@ -317,6 +371,9 @@ def match_type(q: str, snippet: str) -> str | None:
def _record_skip(from_title: str, hit_title: str) -> None:
"""Record that a candidate was skipped or saved for this article."""
+ hit_title = hit_title.strip()
+ if not hit_title:
+ return
skipped: dict[str, list[str]] = flask.session.get("skipped", {})
article_skipped = skipped.get(from_title, [])
if hit_title not in article_skipped:
@@ -371,10 +428,8 @@ def article_page(url_title: str) -> str | Response:
render_mediawiki_error(e), 502
)
- # Filter out candidates already processed this session
- session_skipped: set[str] = set(
- flask.session.get("skipped", {}).get(from_title, [])
- )
+ article_skipped = flask.session.get("skipped", {}).get(from_title, [])
+ session_skipped: set[str] = set(article_skipped)
# If a specific candidate was requested, move it to the front
title_param = flask.request.args.get("title")
@@ -382,12 +437,6 @@ def article_page(url_title: str) -> str | Response:
hits = [h for h in hits if h["title"] == title_param] + \
[h for h in hits if h["title"] != title_param]
- # Record and apply explicit skip-past
- after = flask.request.args.get("after")
- if after:
- _record_skip(from_title, after)
- session_skipped.add(after)
-
hits = [h for h in hits if h["title"] not in session_skipped
and h["title"] != from_title and h["title"] != case_flip_first(from_title)]
@@ -404,6 +453,7 @@ def article_page(url_title: str) -> str | Response:
with_link=with_link,
hits=hits,
url_title=url_title,
+ skipped_titles=article_skipped,
saves_this_session=saves_this_session,
)