Skip invalid and protected candidates

templates/article.html

@@ -85,6 +85,12 @@
   const redirectTo = {{ redirect_to | tojson }};
   const apiUrl = {{ url_for('api_valid_hit') | tojson }};
   const pageUrl = new URL(window.location.href);
+  const cleanPageUrl = new URL(pageUrl);
+  cleanPageUrl.searchParams.delete('title');
+  cleanPageUrl.searchParams.delete('after');
+  cleanPageUrl.searchParams.delete('skip');
+  const storageKey = 'missinglink.skipped.' + linkTo;
+  const skippedTitles = [];
 
   const elProgress = document.getElementById('search-progress');
   const elStatus   = document.getElementById('search-status');
@@ -101,8 +107,39 @@
     if (elCount) elCount.textContent = elList.children.length;
   }
 
+  function loadSkippedTitles() {
+    try {
+      const stored = JSON.parse(localStorage.getItem(storageKey) || '[]');
+      if (Array.isArray(stored)) {
+        for (const title of stored) rememberSkipped(title, false);
+      }
+    } catch (e) {
+      try { localStorage.removeItem(storageKey); } catch (e) {}
+    }
+
+    for (const title of {{ skipped_titles | tojson }}) rememberSkipped(title, false);
+    saveSkippedTitles();
+  }
+
+  function saveSkippedTitles() {
+    try {
+      localStorage.setItem(storageKey, JSON.stringify(skippedTitles));
+    } catch (e) {}
+  }
+
+  function rememberSkipped(title, save = true) {
+    if (!title || skippedTitles.includes(title)) return;
+    skippedTitles.push(title);
+    if (save) saveSkippedTitles();
+  }
+
   async function search() {
+    loadSkippedTitles();
+    for (const hitTitle of skippedTitles) removeCandidate(hitTitle);
+
     for (const hitTitle of hits) {
+      if (skippedTitles.includes(hitTitle)) continue;
+
       elStatus.textContent = `Checking "${hitTitle}"…`;
 
       let data;
@@ -116,7 +153,11 @@
         continue;
       }
 
-      if (!data.valid) { removeCandidate(hitTitle); continue; }
+      if (!data.valid) {
+        rememberSkipped(hitTitle);
+        removeCandidate(hitTitle);
+        continue;
+      }
 
       elProgress.hidden = true;
 
@@ -126,13 +167,15 @@
       document.getElementById('diff-table').innerHTML = data.diff;
       document.getElementById('hit-input').value = hitTitle;
 
-      const skipUrl = new URL(pageUrl);
-      skipUrl.searchParams.delete('title');
-      skipUrl.searchParams.set('after', hitTitle);
-      document.getElementById('skip-link').href = skipUrl.toString();
+      const elSkipLink = document.getElementById('skip-link');
+      elSkipLink.href = cleanPageUrl.toString();
+      elSkipLink.onclick = function () {
+        rememberSkipped(hitTitle);
+      };
 
       const currentUrl = new URL(pageUrl);
       currentUrl.searchParams.delete('after');
+      currentUrl.searchParams.delete('skip');
       currentUrl.searchParams.set('title', hitTitle);
       history.replaceState(null, '', currentUrl.toString());
 

test_web_view.py

@@ -0,0 +1,113 @@
+import unittest
+from unittest.mock import patch
+
+import web_view
+
+
+def hit(title: str) -> web_view.Hit:
+    return {
+        "ns": 0,
+        "title": title,
+        "pageid": 1,
+        "size": 1,
+        "wordcount": 1,
+        "snippet": "",
+        "timestamp": "",
+    }
+
+
+class ArticlePageSkipTests(unittest.TestCase):
+    def setUp(self) -> None:
+        web_view.app.config["TESTING"] = True
+        self.client = web_view.app.test_client()
+
+    def test_session_skips_are_left_out_of_candidates(self) -> None:
+        hits = [
+            hit("Mass racial violence in the United States"),
+            hit("National Register of Historic Places listings in Brookhaven (town), New York"),
+            hit("Useful candidate"),
+        ]
+
+        with self.client.session_transaction() as session:
+            session["skipped"] = {
+                "African-American neighborhood": [
+                    "Mass racial violence in the United States",
+                    "National Register of Historic Places listings in Brookhaven (town), New York",
+                ]
+            }
+
+        with (
+            patch("web_view.api.get_wiki_info", return_value=None),
+            patch("web_view.search_count", return_value=3),
+            patch("web_view.search_count_with_link", return_value=0),
+            patch("web_view.search_no_link", return_value=(3, hits)),
+        ):
+            response = self.client.get("/link/African-American_neighborhood")
+
+        body = response.get_data(as_text=True)
+        self.assertEqual(response.status_code, 200)
+        self.assertNotIn(
+            'data-title="Mass racial violence in the United States"', body
+        )
+        self.assertNotIn(
+            'data-title="National Register of Historic Places listings in Brookhaven (town), New York"',
+            body,
+        )
+        self.assertIn("Useful candidate", body)
+
+
+class ProtectedCandidateTests(unittest.TestCase):
+    def test_edit_protected_hits_are_filtered_out(self) -> None:
+        hits = [hit("Editable"), hit("Protected")]
+        hits[0]["pageid"] = 10
+        hits[1]["pageid"] = 20
+
+        with patch(
+            "web_view.get_page_info_by_id",
+            return_value={
+                10: {"pageid": 10, "title": "Editable", "protection": []},
+                20: {
+                    "pageid": 20,
+                    "title": "Protected",
+                    "protection": [
+                        {"type": "edit", "level": "sysop", "expiry": "infinity"}
+                    ],
+                },
+            },
+        ):
+            filtered = web_view.filter_edit_protected_hits(hits)
+
+        self.assertEqual([item["title"] for item in filtered], ["Editable"])
+
+    def test_search_no_link_filters_edit_protected_results(self) -> None:
+        hits = [hit("Editable"), hit("Protected")]
+        hits[0]["pageid"] = 10
+        hits[1]["pageid"] = 20
+
+        with (
+            patch(
+                "web_view.run_search",
+                return_value={"searchinfo": {"totalhits": 2}, "search": hits},
+            ),
+            patch(
+                "web_view.get_page_info_by_id",
+                return_value={
+                    10: {"pageid": 10, "title": "Editable", "protection": []},
+                    20: {
+                        "pageid": 20,
+                        "title": "Protected",
+                        "protection": [
+                            {"type": "edit", "level": "sysop", "expiry": "infinity"}
+                        ],
+                    },
+                },
+            ),
+        ):
+            totalhits, filtered = web_view.search_no_link("Target")
+
+        self.assertEqual(totalhits, 2)
+        self.assertEqual([item["title"] for item in filtered], ["Editable"])
+
+
+if __name__ == "__main__":
+    unittest.main()

web_view.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python3
 
 import html
-import itertools
 import json
 import re
 import sys
@@ -37,6 +36,14 @@ class Hit(typing.TypedDict):
     timestamp: str
 
 
+class PageInfo(typing.TypedDict, total=False):
+    """Wikipedia page metadata."""
+
+    pageid: int
+    title: str
+    protection: list[dict[str, str]]
+
+
 def render_error(message: str) -> str:
     """Render shared error page."""
     return flask.render_template("error.html", message=message)
@@ -115,13 +122,60 @@ def search_count_with_link(q: str, redirect_to: str | None = None) -> int:
     return count
 
 
+def chunks(items: list[int], size: int) -> typing.Iterator[list[int]]:
+    """Yield fixed-size chunks from a list."""
+    for offset in range(0, len(items), size):
+        yield items[offset : offset + size]
+
+
+def get_page_info_by_id(pageids: list[int]) -> dict[int, PageInfo]:
+    """Fetch metadata for pages by page ID."""
+    if not pageids:
+        return {}
+
+    pages: dict[int, PageInfo] = {}
+    for pageid_chunk in chunks(pageids, 50):
+        ret = api.api_get(
+            {
+                "prop": "info",
+                "inprop": "protection",
+                "pageids": "|".join(str(pageid) for pageid in pageid_chunk),
+            }
+        )
+        for page in ret["query"]["pages"]:
+            pageid = typing.cast(int, page["pageid"])
+            pages[pageid] = typing.cast(PageInfo, page)
+    return pages
+
+
+def is_edit_protected(page: PageInfo | None) -> bool:
+    """Check if a page has edit protection."""
+    if not page:
+        return False
+    return any(
+        protection.get("type") == "edit" or protection.get("action") == "edit"
+        for protection in page.get("protection", [])
+    )
+
+
+def filter_edit_protected_hits(hits: list[Hit]) -> list[Hit]:
+    """Remove candidates that cannot be edited because of page protection."""
+    page_info = get_page_info_by_id([hit["pageid"] for hit in hits])
+    return [
+        hit for hit in hits if not is_edit_protected(page_info.get(hit["pageid"]))
+    ]
+
+
 def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hit]]:
     """Search for mentions of article title with no link included."""
     exclude = f' -linksto:"{q}"'
     if redirect_to:
         exclude += f' -linksto:"{redirect_to}"'
     query = run_search(article_title_to_search_query(q) + exclude, "max")
-    return (query["searchinfo"]["totalhits"], query["search"])
+    return (
+        query["searchinfo"]["totalhits"],
+        filter_edit_protected_hits(query["search"]),
+    )
 
 
 @app.before_request
@@ -317,6 +371,9 @@ def match_type(q: str, snippet: str) -> str | None:
 
 def _record_skip(from_title: str, hit_title: str) -> None:
     """Record that a candidate was skipped or saved for this article."""
+    hit_title = hit_title.strip()
+    if not hit_title:
+        return
     skipped: dict[str, list[str]] = flask.session.get("skipped", {})
     article_skipped = skipped.get(from_title, [])
     if hit_title not in article_skipped:
@@ -371,10 +428,8 @@ def article_page(url_title: str) -> str | Response:
             render_mediawiki_error(e), 502
         )
 
-    # Filter out candidates already processed this session
-    session_skipped: set[str] = set(
-        flask.session.get("skipped", {}).get(from_title, [])
-    )
+    article_skipped = flask.session.get("skipped", {}).get(from_title, [])
+    session_skipped: set[str] = set(article_skipped)
 
     # If a specific candidate was requested, move it to the front
     title_param = flask.request.args.get("title")
@@ -382,12 +437,6 @@ def article_page(url_title: str) -> str | Response:
         hits = [h for h in hits if h["title"] == title_param] + \
                [h for h in hits if h["title"] != title_param]
 
-    # Record and apply explicit skip-past
-    after = flask.request.args.get("after")
-    if after:
-        _record_skip(from_title, after)
-        session_skipped.add(after)
-
     hits = [h for h in hits if h["title"] not in session_skipped
             and h["title"] != from_title and h["title"] != case_flip_first(from_title)]
 
@@ -404,6 +453,7 @@ def article_page(url_title: str) -> str | Response:
         with_link=with_link,
         hits=hits,
         url_title=url_title,
+        skipped_titles=article_skipped,
         saves_this_session=saves_this_session,
     )