diff --git a/add_links/api.py b/add_links/api.py index 6bd7577..0742fe2 100644 --- a/add_links/api.py +++ b/add_links/api.py @@ -1,5 +1,3 @@ -import os -from pathlib import Path import re import sys import typing @@ -10,7 +8,6 @@ from simplejson.scanner import JSONDecodeError from .language import get_current_language from .util import is_disambig -from .wikimedia_api_logging import WikimediaApiLogConfig, logged_request StrDict = dict[str, typing.Any] @@ -18,15 +15,6 @@ ua = ( "find-link/2.2 " + "(https://github.com/EdwardBetts/find_link; contact: edward@4angle.com)" ) -wikimedia_log_config = WikimediaApiLogConfig( - tool="missinglink", - log_path=Path( - os.environ.get( - "MISSINGLINK_WIKIMEDIA_API_LOG", "/var/log/missinglink/wikimedia-api.jsonl" - ) - ), - user_agent=ua, -) re_disambig = re.compile(r"^(.*) \((.*)\)$") @@ -85,40 +73,23 @@ webpage_error = ( ) -def _get_request_oauth_session() -> requests.sessions.Session | None: - """Return the logged-in user's OAuth session in a Flask request, if present.""" +def _get_active_session() -> requests.sessions.Session: + """Return OAuth session if one is available in Flask context, else plain session.""" try: - from flask import g, has_request_context - - if not has_request_context(): - return None - - oauth_session = getattr(g, "oauth_session", None) - if oauth_session is not None: - return typing.cast(requests.sessions.Session, oauth_session) - - from . import mediawiki_oauth - - return typing.cast( - requests.sessions.Session | None, mediawiki_oauth.get_oauth_session() - ) + from flask import g + if hasattr(g, "oauth_session") and g.oauth_session is not None: + return g.oauth_session # type: ignore[return-value] except RuntimeError: - return None - - -def _get_active_session() -> tuple[requests.sessions.Session, str]: - """Return active session and auth mode.""" - if oauth_session := _get_request_oauth_session(): - return oauth_session, "oauth" - + pass print("WARNING: using unauthenticated session", file=sys.stderr) - return get_session(), "anon" + return get_session() def api_get(params: StrDict) -> StrDict: """Make call to Wikipedia API.""" - s, _auth = _get_active_session() - r = logged_request(s, wikimedia_log_config, "GET", get_query_url(), params=params) + s = _get_active_session() + + r = s.get(get_query_url(), params=params) try: ret: StrDict = r.json() except JSONDecodeError: @@ -316,8 +287,8 @@ def call_get_diff(title: str, section_num: int, section_text: str) -> str: "rvdifftotext": section_text.strip(), } - s, _auth = _get_active_session() - r = logged_request(s, wikimedia_log_config, "POST", get_query_url(), data=data) + s = _get_active_session() + r = s.post(get_query_url(), data=data) try: ret = r.json() except JSONDecodeError: diff --git a/add_links/match.py b/add_links/match.py index 4c6af97..732e3ba 100644 --- a/add_links/match.py +++ b/add_links/match.py @@ -116,15 +116,10 @@ def find_cite_template_spans(text: str) -> list[tuple[int, int]]: def parse_cite(text: str) -> typing.Iterator[tuple[str, str]]: """Parse citations yielding (type, chunk) tuples, skipping ref tags, cite templates, and external links.""" regions = [(m.start(), m.end()) for m in re_cite.finditer(text)] - link_spans = [(m.start(), m.end()) for m in re_link_in_text.finditer(text)] regions.extend(find_cite_template_spans(text)) regions.extend((m.start(), m.end()) for m in re_no_param_template.finditer(text)) regions.extend((m.start(), m.end()) for m in re_external_link.finditer(text)) - regions.extend( - (m.start(), m.end()) - for m in re_italic.finditer(text) - if not any(start <= m.start() and m.end() <= end for start, end in link_spans) - ) + regions.extend((m.start(), m.end()) for m in re_italic.finditer(text)) regions.extend((m.start(), m.end()) for m in re_bullet_with_url.finditer(text)) regions.sort() diff --git a/add_links/mediawiki_api.py b/add_links/mediawiki_api.py index 8319c37..611ab5b 100644 --- a/add_links/mediawiki_api.py +++ b/add_links/mediawiki_api.py @@ -1,6 +1,7 @@ """Interface with the mediawiki API.""" import typing +from pprint import pprint from typing import Any, cast import requests @@ -11,17 +12,6 @@ from . import mediawiki_oauth class APIError(Exception): """Unexpected response from the MediaWiki API.""" - -def _format_api_error(data: dict[str, Any]) -> str: - """Format a MediaWiki API error response.""" - error = data.get("error") - if isinstance(error, dict): - message = error.get("info") or error.get("code") - if message: - return str(message) - return f"Unexpected MediaWiki API response: {data!r}" - - wiki_hostname = "en.wikipedia.org" wiki_api_php = f"https://{wiki_hostname}/w/api.php" user_agent = "add-links/0.1" @@ -113,5 +103,8 @@ def edit_page( } ret = call(params, timeout=30) if "edit" not in ret: - raise APIError(_format_api_error(ret)) + print("params") + pprint(params) + print() + pprint(ret) return typing.cast(str, ret["edit"]) diff --git a/add_links/mediawiki_api_old.py b/add_links/mediawiki_api_old.py index 851a43b..0196207 100644 --- a/add_links/mediawiki_api_old.py +++ b/add_links/mediawiki_api_old.py @@ -4,9 +4,6 @@ from typing import Any import requests -from .api import wikimedia_log_config -from .wikimedia_api_logging import logged_request - wiki_hostname = "en.wikipedia.org" wiki_api_php = f"https://{wiki_hostname}/w/api.php" user_agent = "dab-mechanic/0.1" @@ -30,10 +27,8 @@ def parse_page(enwiki: str) -> dict[str, Any]: def get(params: dict[str, str | int]) -> dict[str, Any]: """Make GET request to mediawiki API.""" - session = requests.Session() - session.headers = {"User-Agent": user_agent} - data: dict[str, Any] = logged_request( - session, wikimedia_log_config, "GET", wiki_api_php, params=params + data: dict[str, Any] = requests.get( + wiki_api_php, headers={"User-Agent": user_agent}, params=params ).json() return data diff --git a/add_links/mediawiki_oauth.py b/add_links/mediawiki_oauth.py index 69f7d9c..12ebd58 100644 --- a/add_links/mediawiki_oauth.py +++ b/add_links/mediawiki_oauth.py @@ -9,8 +9,7 @@ import requests from flask import current_app, session from requests_oauthlib import OAuth1Session -from .api import ua, wikimedia_log_config -from .wikimedia_api_logging import logged_request +from .api import ua wiki_hostname = "en.wikipedia.org" api_url = f"https://{wiki_hostname}/w/api.php" @@ -45,15 +44,7 @@ def api_post_request(params: dict[str, str | int], timeout: int = 4) -> requests ) oauth.headers.update({"User-Agent": ua}) proxies = get_edit_proxy() - return logged_request( - oauth, - wikimedia_log_config, - "POST", - api_url, - data=params, - timeout=timeout, - proxies=proxies, - ) + return oauth.post(api_url, data=params, timeout=timeout, proxies=proxies) def raw_request(params: typing.Mapping[str, str | int]) -> requests.Response: @@ -72,13 +63,8 @@ def raw_request(params: typing.Mapping[str, str | int]) -> requests.Response: ) oauth.headers.update({"User-Agent": ua}) proxies = get_edit_proxy() - return logged_request( - oauth, - wikimedia_log_config, - "GET", - api_url + "?" + urllib.parse.urlencode(params), - timeout=4, - proxies=proxies, + return oauth.get( + api_url + "?" + urllib.parse.urlencode(params), timeout=4, proxies=proxies ) diff --git a/add_links/wikimedia_api_logging.py b/add_links/wikimedia_api_logging.py deleted file mode 100644 index 8936930..0000000 --- a/add_links/wikimedia_api_logging.py +++ /dev/null @@ -1,222 +0,0 @@ -"""JSONL logging helpers for Wikimedia API request metrics.""" - -import json -import logging -import os -import socket -import time -from collections.abc import Mapping -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from types import TracebackType -from urllib.parse import parse_qs, urlparse - -import requests - - -@dataclass(frozen=True) -class WikimediaApiLogConfig: - """Configuration for Wikimedia API request logging.""" - - tool: str - log_path: Path - user_agent: str - - -@dataclass(frozen=True) -class WikimediaApiRequestMetric: - """Details of one Wikimedia API request.""" - - tool: str - url: str - method: str - status_code: int | None - elapsed_ms: int - user_agent: str - request_params: Mapping[str, object] | None = None - error: str | None = None - - -_logger_cache: dict[Path, logging.Logger] = {} - - -def setup_wikimedia_api_logger(log_path: Path) -> logging.Logger: - """Create a JSONL logger for Wikimedia API request metrics.""" - if log_path in _logger_cache: - return _logger_cache[log_path] - - logger_name = f"wikimedia_api_metrics.{log_path}" - logger = logging.getLogger(logger_name) - logger.setLevel(logging.INFO) - logger.propagate = False - - if not logger.handlers: - try: - log_path.parent.mkdir(parents=True, exist_ok=True) - handler: logging.Handler = logging.FileHandler(log_path) - except OSError: - handler = logging.NullHandler() - handler.setFormatter(logging.Formatter("%(message)s")) - logger.addHandler(handler) - - _logger_cache[log_path] = logger - return logger - - -def get_mediawiki_action( - url: str, request_params: Mapping[str, object] | None = None -) -> str | None: - """Extract the MediaWiki API action from params or a URL, if present.""" - if request_params is not None: - action = request_params.get("action") - if isinstance(action, str): - return action - - parsed = urlparse(url) - query = parse_qs(parsed.query) - values = query.get("action") - - if not values: - return None - - return values[0] - - -def get_flask_request_context() -> tuple[str | None, str | None]: - """Return Flask endpoint and browser URL when running in a request.""" - try: - from flask import has_request_context, request - - if not has_request_context(): - return None, None - return request.endpoint, request.url - except RuntimeError: - return None, None - - -def build_log_record(metric: WikimediaApiRequestMetric) -> dict[str, object]: - """Build a JSON-serialisable log record for one API request.""" - parsed = urlparse(metric.url) - flask_endpoint, flask_url = get_flask_request_context() - - record: dict[str, object] = { - "ts": datetime.now(UTC).isoformat(), - "tool": metric.tool, - "host": socket.gethostname(), - "pid": os.getpid(), - "method": metric.method, - "api_host": parsed.netloc, - "path": parsed.path, - "action": get_mediawiki_action(metric.url, metric.request_params), - "status_code": metric.status_code, - "elapsed_ms": metric.elapsed_ms, - "user_agent": metric.user_agent, - "flask_endpoint": flask_endpoint, - "flask_url": flask_url, - } - - if metric.error is not None: - record["error"] = metric.error - - return record - - -def log_wikimedia_api_request( - logger: logging.Logger, - metric: WikimediaApiRequestMetric, -) -> None: - """Write one Wikimedia API request metric as a JSONL log line.""" - record = build_log_record(metric) - logger.info(json.dumps(record, separators=(",", ":"), sort_keys=True)) - - -class WikimediaRequestTimer: - """Context manager for timing and logging a Wikimedia API request.""" - - def __init__( - self, - config: WikimediaApiLogConfig, - method: str, - url: str, - request_params: Mapping[str, object] | None = None, - ) -> None: - self.config = config - self.method = method - self.url = url - self.request_params = request_params - self.started = 0.0 - self.logger = setup_wikimedia_api_logger(config.log_path) - - def __enter__(self) -> "WikimediaRequestTimer": - """Start timing a request.""" - self.started = time.monotonic() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc: BaseException | None, - traceback: TracebackType | None, - ) -> bool: - """Log failed requests when an exception escapes.""" - if exc is None: - return False - - elapsed_ms = int((time.monotonic() - self.started) * 1000) - - log_wikimedia_api_request( - self.logger, - WikimediaApiRequestMetric( - tool=self.config.tool, - url=self.url, - method=self.method, - status_code=None, - elapsed_ms=elapsed_ms, - user_agent=self.config.user_agent, - request_params=self.request_params, - error=type(exc).__name__, - ), - ) - - return False - - def log_response(self, status_code: int, final_url: str | None = None) -> None: - """Log a completed request.""" - elapsed_ms = int((time.monotonic() - self.started) * 1000) - - log_wikimedia_api_request( - self.logger, - WikimediaApiRequestMetric( - tool=self.config.tool, - url=final_url or self.url, - method=self.method, - status_code=status_code, - elapsed_ms=elapsed_ms, - user_agent=self.config.user_agent, - request_params=self.request_params, - ), - ) - - -def logged_request( - session: requests.sessions.Session, - config: WikimediaApiLogConfig, - method: str, - url: str, - **kwargs: object, -) -> requests.Response: - """Make a Wikimedia API request and log one JSONL metric line.""" - request_params = kwargs.get("params") or kwargs.get("data") - if not isinstance(request_params, Mapping): - request_params = None - - with WikimediaRequestTimer( - config, method.upper(), url, request_params=request_params - ) as timer: - response = getattr(session, method.lower())(url, **kwargs) - final_url = getattr(response, "url", None) - timer.log_response( - response.status_code, final_url if isinstance(final_url, str) else None - ) - return response diff --git a/templates/article.html b/templates/article.html index 18269c4..a96770c 100644 --- a/templates/article.html +++ b/templates/article.html @@ -85,12 +85,6 @@ const redirectTo = {{ redirect_to | tojson }}; const apiUrl = {{ url_for('api_valid_hit') | tojson }}; const pageUrl = new URL(window.location.href); - const cleanPageUrl = new URL(pageUrl); - cleanPageUrl.searchParams.delete('title'); - cleanPageUrl.searchParams.delete('after'); - cleanPageUrl.searchParams.delete('skip'); - const storageKey = 'missinglink.skipped.' + linkTo; - const skippedTitles = []; const elProgress = document.getElementById('search-progress'); const elStatus = document.getElementById('search-status'); @@ -107,39 +101,8 @@ if (elCount) elCount.textContent = elList.children.length; } - function loadSkippedTitles() { - try { - const stored = JSON.parse(localStorage.getItem(storageKey) || '[]'); - if (Array.isArray(stored)) { - for (const title of stored) rememberSkipped(title, false); - } - } catch (e) { - try { localStorage.removeItem(storageKey); } catch (e) {} - } - - for (const title of {{ skipped_titles | tojson }}) rememberSkipped(title, false); - saveSkippedTitles(); - } - - function saveSkippedTitles() { - try { - localStorage.setItem(storageKey, JSON.stringify(skippedTitles)); - } catch (e) {} - } - - function rememberSkipped(title, save = true) { - if (!title || skippedTitles.includes(title)) return; - skippedTitles.push(title); - if (save) saveSkippedTitles(); - } - async function search() { - loadSkippedTitles(); - for (const hitTitle of skippedTitles) removeCandidate(hitTitle); - for (const hitTitle of hits) { - if (skippedTitles.includes(hitTitle)) continue; - elStatus.textContent = `Checking "${hitTitle}"…`; let data; @@ -153,11 +116,7 @@ continue; } - if (!data.valid) { - rememberSkipped(hitTitle); - removeCandidate(hitTitle); - continue; - } + if (!data.valid) { removeCandidate(hitTitle); continue; } elProgress.hidden = true; @@ -167,15 +126,13 @@ document.getElementById('diff-table').innerHTML = data.diff; document.getElementById('hit-input').value = hitTitle; - const elSkipLink = document.getElementById('skip-link'); - elSkipLink.href = cleanPageUrl.toString(); - elSkipLink.onclick = function () { - rememberSkipped(hitTitle); - }; + const skipUrl = new URL(pageUrl); + skipUrl.searchParams.delete('title'); + skipUrl.searchParams.set('after', hitTitle); + document.getElementById('skip-link').href = skipUrl.toString(); const currentUrl = new URL(pageUrl); currentUrl.searchParams.delete('after'); - currentUrl.searchParams.delete('skip'); currentUrl.searchParams.set('title', hitTitle); history.replaceState(null, '', currentUrl.toString()); diff --git a/templates/base.html b/templates/base.html index 590f7e2..0328311 100644 --- a/templates/base.html +++ b/templates/base.html @@ -6,44 +6,15 @@ {% block title %}{% endblock %} – Missing Link - {% block style %}{% endblock %} -