diff --git a/templates/article.html b/templates/article.html index a96770c..18269c4 100644 --- a/templates/article.html +++ b/templates/article.html @@ -85,6 +85,12 @@ const redirectTo = {{ redirect_to | tojson }}; const apiUrl = {{ url_for('api_valid_hit') | tojson }}; const pageUrl = new URL(window.location.href); + const cleanPageUrl = new URL(pageUrl); + cleanPageUrl.searchParams.delete('title'); + cleanPageUrl.searchParams.delete('after'); + cleanPageUrl.searchParams.delete('skip'); + const storageKey = 'missinglink.skipped.' + linkTo; + const skippedTitles = []; const elProgress = document.getElementById('search-progress'); const elStatus = document.getElementById('search-status'); @@ -101,8 +107,39 @@ if (elCount) elCount.textContent = elList.children.length; } + function loadSkippedTitles() { + try { + const stored = JSON.parse(localStorage.getItem(storageKey) || '[]'); + if (Array.isArray(stored)) { + for (const title of stored) rememberSkipped(title, false); + } + } catch (e) { + try { localStorage.removeItem(storageKey); } catch (e) {} + } + + for (const title of {{ skipped_titles | tojson }}) rememberSkipped(title, false); + saveSkippedTitles(); + } + + function saveSkippedTitles() { + try { + localStorage.setItem(storageKey, JSON.stringify(skippedTitles)); + } catch (e) {} + } + + function rememberSkipped(title, save = true) { + if (!title || skippedTitles.includes(title)) return; + skippedTitles.push(title); + if (save) saveSkippedTitles(); + } + async function search() { + loadSkippedTitles(); + for (const hitTitle of skippedTitles) removeCandidate(hitTitle); + for (const hitTitle of hits) { + if (skippedTitles.includes(hitTitle)) continue; + elStatus.textContent = `Checking "${hitTitle}"…`; let data; @@ -116,7 +153,11 @@ continue; } - if (!data.valid) { removeCandidate(hitTitle); continue; } + if (!data.valid) { + rememberSkipped(hitTitle); + removeCandidate(hitTitle); + continue; + } elProgress.hidden = true; @@ -126,13 +167,15 @@ document.getElementById('diff-table').innerHTML = data.diff; document.getElementById('hit-input').value = hitTitle; - const skipUrl = new URL(pageUrl); - skipUrl.searchParams.delete('title'); - skipUrl.searchParams.set('after', hitTitle); - document.getElementById('skip-link').href = skipUrl.toString(); + const elSkipLink = document.getElementById('skip-link'); + elSkipLink.href = cleanPageUrl.toString(); + elSkipLink.onclick = function () { + rememberSkipped(hitTitle); + }; const currentUrl = new URL(pageUrl); currentUrl.searchParams.delete('after'); + currentUrl.searchParams.delete('skip'); currentUrl.searchParams.set('title', hitTitle); history.replaceState(null, '', currentUrl.toString()); diff --git a/test_web_view.py b/test_web_view.py new file mode 100644 index 0000000..af1e203 --- /dev/null +++ b/test_web_view.py @@ -0,0 +1,113 @@ +import unittest +from unittest.mock import patch + +import web_view + + +def hit(title: str) -> web_view.Hit: + return { + "ns": 0, + "title": title, + "pageid": 1, + "size": 1, + "wordcount": 1, + "snippet": "", + "timestamp": "", + } + + +class ArticlePageSkipTests(unittest.TestCase): + def setUp(self) -> None: + web_view.app.config["TESTING"] = True + self.client = web_view.app.test_client() + + def test_session_skips_are_left_out_of_candidates(self) -> None: + hits = [ + hit("Mass racial violence in the United States"), + hit("National Register of Historic Places listings in Brookhaven (town), New York"), + hit("Useful candidate"), + ] + + with self.client.session_transaction() as session: + session["skipped"] = { + "African-American neighborhood": [ + "Mass racial violence in the United States", + "National Register of Historic Places listings in Brookhaven (town), New York", + ] + } + + with ( + patch("web_view.api.get_wiki_info", return_value=None), + patch("web_view.search_count", return_value=3), + patch("web_view.search_count_with_link", return_value=0), + patch("web_view.search_no_link", return_value=(3, hits)), + ): + response = self.client.get("/link/African-American_neighborhood") + + body = response.get_data(as_text=True) + self.assertEqual(response.status_code, 200) + self.assertNotIn( + 'data-title="Mass racial violence in the United States"', body + ) + self.assertNotIn( + 'data-title="National Register of Historic Places listings in Brookhaven (town), New York"', + body, + ) + self.assertIn("Useful candidate", body) + + +class ProtectedCandidateTests(unittest.TestCase): + def test_edit_protected_hits_are_filtered_out(self) -> None: + hits = [hit("Editable"), hit("Protected")] + hits[0]["pageid"] = 10 + hits[1]["pageid"] = 20 + + with patch( + "web_view.get_page_info_by_id", + return_value={ + 10: {"pageid": 10, "title": "Editable", "protection": []}, + 20: { + "pageid": 20, + "title": "Protected", + "protection": [ + {"type": "edit", "level": "sysop", "expiry": "infinity"} + ], + }, + }, + ): + filtered = web_view.filter_edit_protected_hits(hits) + + self.assertEqual([item["title"] for item in filtered], ["Editable"]) + + def test_search_no_link_filters_edit_protected_results(self) -> None: + hits = [hit("Editable"), hit("Protected")] + hits[0]["pageid"] = 10 + hits[1]["pageid"] = 20 + + with ( + patch( + "web_view.run_search", + return_value={"searchinfo": {"totalhits": 2}, "search": hits}, + ), + patch( + "web_view.get_page_info_by_id", + return_value={ + 10: {"pageid": 10, "title": "Editable", "protection": []}, + 20: { + "pageid": 20, + "title": "Protected", + "protection": [ + {"type": "edit", "level": "sysop", "expiry": "infinity"} + ], + }, + }, + ), + ): + totalhits, filtered = web_view.search_no_link("Target") + + self.assertEqual(totalhits, 2) + self.assertEqual([item["title"] for item in filtered], ["Editable"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/web_view.py b/web_view.py index 6f7c9c5..748857d 100755 --- a/web_view.py +++ b/web_view.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 import html -import itertools import json import re import sys @@ -37,6 +36,14 @@ class Hit(typing.TypedDict): timestamp: str +class PageInfo(typing.TypedDict, total=False): + """Wikipedia page metadata.""" + + pageid: int + title: str + protection: list[dict[str, str]] + + def render_error(message: str) -> str: """Render shared error page.""" return flask.render_template("error.html", message=message) @@ -115,13 +122,60 @@ def search_count_with_link(q: str, redirect_to: str | None = None) -> int: return count +def chunks(items: list[int], size: int) -> typing.Iterator[list[int]]: + """Yield fixed-size chunks from a list.""" + for offset in range(0, len(items), size): + yield items[offset : offset + size] + + +def get_page_info_by_id(pageids: list[int]) -> dict[int, PageInfo]: + """Fetch metadata for pages by page ID.""" + if not pageids: + return {} + + pages: dict[int, PageInfo] = {} + for pageid_chunk in chunks(pageids, 50): + ret = api.api_get( + { + "prop": "info", + "inprop": "protection", + "pageids": "|".join(str(pageid) for pageid in pageid_chunk), + } + ) + for page in ret["query"]["pages"]: + pageid = typing.cast(int, page["pageid"]) + pages[pageid] = typing.cast(PageInfo, page) + return pages + + +def is_edit_protected(page: PageInfo | None) -> bool: + """Check if a page has edit protection.""" + if not page: + return False + return any( + protection.get("type") == "edit" or protection.get("action") == "edit" + for protection in page.get("protection", []) + ) + + +def filter_edit_protected_hits(hits: list[Hit]) -> list[Hit]: + """Remove candidates that cannot be edited because of page protection.""" + page_info = get_page_info_by_id([hit["pageid"] for hit in hits]) + return [ + hit for hit in hits if not is_edit_protected(page_info.get(hit["pageid"])) + ] + + def search_no_link(q: str, redirect_to: str | None = None) -> tuple[int, list[Hit]]: """Search for mentions of article title with no link included.""" exclude = f' -linksto:"{q}"' if redirect_to: exclude += f' -linksto:"{redirect_to}"' query = run_search(article_title_to_search_query(q) + exclude, "max") - return (query["searchinfo"]["totalhits"], query["search"]) + return ( + query["searchinfo"]["totalhits"], + filter_edit_protected_hits(query["search"]), + ) @app.before_request @@ -317,6 +371,9 @@ def match_type(q: str, snippet: str) -> str | None: def _record_skip(from_title: str, hit_title: str) -> None: """Record that a candidate was skipped or saved for this article.""" + hit_title = hit_title.strip() + if not hit_title: + return skipped: dict[str, list[str]] = flask.session.get("skipped", {}) article_skipped = skipped.get(from_title, []) if hit_title not in article_skipped: @@ -371,10 +428,8 @@ def article_page(url_title: str) -> str | Response: render_mediawiki_error(e), 502 ) - # Filter out candidates already processed this session - session_skipped: set[str] = set( - flask.session.get("skipped", {}).get(from_title, []) - ) + article_skipped = flask.session.get("skipped", {}).get(from_title, []) + session_skipped: set[str] = set(article_skipped) # If a specific candidate was requested, move it to the front title_param = flask.request.args.get("title") @@ -382,12 +437,6 @@ def article_page(url_title: str) -> str | Response: hits = [h for h in hits if h["title"] == title_param] + \ [h for h in hits if h["title"] != title_param] - # Record and apply explicit skip-past - after = flask.request.args.get("after") - if after: - _record_skip(from_title, after) - session_skipped.add(after) - hits = [h for h in hits if h["title"] not in session_skipped and h["title"] != from_title and h["title"] != case_flip_first(from_title)] @@ -404,6 +453,7 @@ def article_page(url_title: str) -> str | Response: with_link=with_link, hits=hits, url_title=url_title, + skipped_titles=article_skipped, saves_this_session=saves_this_session, )