dab-mechanic/web_view.py

310 lines
8.7 KiB
Python
Raw Normal View History

2022-08-13 13:16:49 +01:00
#!/usr/bin/python3
2022-08-15 17:56:21 +01:00
import inspect
2022-08-13 16:25:07 +01:00
import json
2022-08-14 17:44:07 +01:00
import re
2022-08-21 11:31:21 +01:00
from typing import Optional, TypedDict
import mwparserfromhell
2023-09-29 14:17:56 +01:00
from pprint import pprint
2022-08-13 13:16:49 +01:00
import flask
import lxml.html
import requests
2022-08-15 17:56:21 +01:00
import werkzeug.exceptions
2022-08-15 11:43:22 +01:00
from requests_oauthlib import OAuth1Session
2022-08-15 17:56:21 +01:00
from werkzeug.debug.tbtools import get_current_traceback
2022-08-13 13:16:49 +01:00
from werkzeug.wrappers import Response
2022-08-17 14:38:30 +01:00
from dab_mechanic import mediawiki_api, wikidata_oauth, wikipedia
2022-08-13 13:16:49 +01:00
app = flask.Flask(__name__)
2022-08-15 13:11:29 +01:00
app.config.from_object("config.default")
2022-08-13 13:16:49 +01:00
2022-08-16 12:43:03 +01:00
wiki_hostname = "en.wikipedia.org"
wiki_api_php = f"https://{wiki_hostname}/w/api.php"
wiki_index_php = f"https://{wiki_hostname}/w/index.php"
2022-08-17 13:34:55 +01:00
awdl_url = "https://dplbot.toolforge.org/articles_with_dab_links.php"
2022-08-16 12:43:03 +01:00
@app.before_request
def global_user():
"""Make username available everywhere."""
flask.g.user = wikidata_oauth.get_username()
2022-08-13 13:16:49 +01:00
2022-08-14 17:44:07 +01:00
2022-08-15 17:56:21 +01:00
@app.errorhandler(werkzeug.exceptions.InternalServerError)
def exception_handler(e):
tb = get_current_traceback()
last_frame = next(frame for frame in reversed(tb.frames) if not frame.is_library)
last_frame_args = inspect.getargs(last_frame.code)
return (
flask.render_template(
"show_error.html",
tb=tb,
last_frame=last_frame,
last_frame_args=last_frame_args,
),
500,
)
def parse_articles_with_dab_links(root: lxml.html.Element) -> list[tuple[str, int]]:
"""Parse Articles With Multiple Dablinks."""
articles = []
table = root.find(".//table")
for tr in table:
title = tr[0][0].text
count_text = tr[1][0].text
assert count_text.endswith(" links")
count = int(count_text[:-6])
articles.append((title, count))
return articles
2022-08-13 13:16:49 +01:00
@app.route("/")
def index():
2023-09-29 14:17:56 +01:00
title = flask.request.args.get("title")
exists = None
if title:
title = title.strip()
exists = mediawiki_api.article_exists(title)
if exists:
return flask.redirect(
flask.url_for("article_page", enwiki=title.replace(" ", "_"))
)
2022-08-17 13:34:55 +01:00
r = requests.get(awdl_url, params={"limit": 100})
root = lxml.html.fromstring(r.content)
articles = parse_articles_with_dab_links(root)
# articles = [line[:-1] for line in open("article_list")]
2022-08-13 13:16:49 +01:00
2023-09-29 14:17:56 +01:00
return flask.render_template(
"index.html", title=title, exists=exists, articles=articles,
)
2022-08-13 13:16:49 +01:00
2022-08-21 11:31:21 +01:00
class Edit(TypedDict):
"""Edit to an article."""
num: int
link_to: str
title: str
2022-08-14 17:44:07 +01:00
2023-09-29 14:17:56 +01:00
def old_apply_edits(article_text: str, edits: list[Edit]) -> str:
2022-08-14 17:44:07 +01:00
"""Apply edits to article text."""
def escape(s: str) -> str:
return re.escape(s).replace("_", "[ _]").replace(r"\ ", "[ _]")
2022-08-21 11:31:21 +01:00
for edit in edits:
# print(rf"\[\[{escape(link_from)}\]\]")
2022-08-14 17:44:07 +01:00
article_text = re.sub(
rf"\[\[{escape(link_from)}\]\]",
f"[[{link_to}|{link_from}]]",
article_text,
)
return article_text
2022-08-21 11:31:21 +01:00
def make_disamb_link(edit: Edit) -> str:
"""Given an edit return the appropriate link."""
return f"[[{edit['title']}|{edit['link_to']}]]"
2022-08-14 17:44:07 +01:00
2022-08-21 11:31:21 +01:00
def build_edit_summary(edits: list[Edit]) -> str:
"""Given a list of edits return an edit summary."""
2022-08-14 17:44:07 +01:00
titles = ", ".join(make_disamb_link(edit) for edit in edits[:-1])
if len(titles) > 1:
titles += " and "
titles += make_disamb_link(edits[-1])
2022-08-21 11:31:21 +01:00
return f"Disambiguate {titles} using [[User:Edward/Dab mechanic]]"
2023-09-29 14:17:56 +01:00
def get_links(wikicode, dab_links):
edits = [edit for edit in dab_links if edit.get("title")]
2022-08-21 11:31:21 +01:00
dab_titles = {dab["link_to"] for dab in edits}
return [
link for link in wikicode.filter_wikilinks() if str(link.title) in dab_titles
]
2023-09-29 14:17:56 +01:00
def apply_edits(text, dab_links):
2022-08-21 11:31:21 +01:00
wikicode = mwparserfromhell.parse(text)
links = get_links(wikicode, dab_links)
2023-09-29 14:17:56 +01:00
if len(links) != len(dab_links):
print("links:", len(links))
print("dab_links:", len(dab_links))
print("dab_links:", dab_links)
2022-08-21 11:31:21 +01:00
assert len(links) == len(dab_links)
for wikilink, edit in zip(links, dab_links):
if not edit.get("title"):
continue
if not wikilink.text:
wikilink.text = wikilink.title
wikilink.title = edit["title"]
2022-08-14 17:44:07 +01:00
2023-09-29 14:17:56 +01:00
return str(wikicode)
@app.route("/preview/<path:enwiki>", methods=["POST"])
def preview(enwiki: str) -> Response | str:
"""Preview article edits."""
enwiki = enwiki.replace("_", " ")
dab_links = json.loads(flask.request.form["edits"])
dab_links = [link for link in dab_links if "title" in link]
cur_text, baserevid = mediawiki_api.get_content(enwiki)
text = apply_edits(cur_text, dab_links)
diff = mediawiki_api.compare(enwiki, text)
2022-08-14 17:44:07 +01:00
return flask.render_template(
2023-09-29 14:17:56 +01:00
"preview.html",
edit_summary=build_edit_summary(dab_links),
2022-08-14 17:44:07 +01:00
title=enwiki,
2022-08-21 11:31:21 +01:00
edits=dab_links,
diff=diff,
2022-08-14 17:44:07 +01:00
)
2023-09-29 14:17:56 +01:00
def do_save(enwiki: str):
"""Update page on Wikipedia."""
dab_links = json.loads(flask.request.form["edits"])
dab_links = [link for link in dab_links if "title" in link]
cur_text, baserevid = mediawiki_api.get_content(enwiki)
new_text = apply_edits(cur_text, dab_links)
token = wikidata_oauth.get_token()
summary = build_edit_summary(dab_links)
print(summary)
edit = mediawiki_api.edit_page(
title=enwiki,
text=new_text,
summary=summary,
baserevid=baserevid,
token=token,
)
return edit
@app.route("/save/<path:enwiki>", methods=["GET", "POST"])
def save(enwiki: str) -> Response | str:
"""Save edits to article."""
enwiki_norm = enwiki.replace("_", " ")
if flask.request.method == "GET":
return flask.render_template("edit_saved.html", title=enwiki_norm)
do_save(enwiki_norm)
return flask.redirect(flask.url_for(flask.request.endpoint, enwiki=enwiki))
2022-08-17 14:48:20 +01:00
def redirect_if_needed(enwiki: str) -> Optional[Response]:
"""Check if there are spaces in the article name and redirect."""
return (
flask.redirect(
flask.url_for(flask.request.endpoint, enwiki=enwiki.replace(" ", "_"))
)
if " " in enwiki
else None
)
2022-08-13 13:16:49 +01:00
@app.route("/enwiki/<path:enwiki>")
2022-08-14 17:44:07 +01:00
def article_page(enwiki: str) -> Response:
2022-08-13 13:16:49 +01:00
"""Article Page."""
2022-08-17 14:48:20 +01:00
redirect = redirect_if_needed(enwiki)
if redirect:
return redirect
2022-08-13 13:16:49 +01:00
2023-09-29 14:17:56 +01:00
if "owner_key" not in flask.session:
return flask.render_template("login_needed.html")
2022-08-17 13:35:26 +01:00
article = wikipedia.Article(enwiki)
2022-08-14 17:44:07 +01:00
article.load()
article.process_links()
2022-08-13 13:16:49 +01:00
assert article.parse
2022-08-14 17:48:19 +01:00
return flask.render_template("article.html", article=article)
2022-08-13 13:16:49 +01:00
2022-08-15 11:43:22 +01:00
@app.route("/oauth/start")
def start_oauth():
next_page = flask.request.args.get("next")
if next_page:
flask.session["after_login"] = next_page
client_key = app.config["CLIENT_KEY"]
client_secret = app.config["CLIENT_SECRET"]
2022-08-16 12:43:03 +01:00
request_token_url = wiki_index_php + "?title=Special%3aOAuth%2finitiate"
2022-08-15 11:43:22 +01:00
oauth = OAuth1Session(client_key, client_secret=client_secret, callback_uri="oob")
fetch_response = oauth.fetch_request_token(request_token_url)
flask.session["owner_key"] = fetch_response.get("oauth_token")
flask.session["owner_secret"] = fetch_response.get("oauth_token_secret")
2022-08-16 12:43:03 +01:00
base_authorization_url = f"https://{wiki_hostname}/wiki/Special:OAuth/authorize"
2022-08-15 11:43:22 +01:00
authorization_url = oauth.authorization_url(
base_authorization_url, oauth_consumer_key=client_key
)
return flask.redirect(authorization_url)
@app.route("/oauth/callback", methods=["GET"])
def oauth_callback():
client_key = app.config["CLIENT_KEY"]
client_secret = app.config["CLIENT_SECRET"]
oauth = OAuth1Session(
client_key,
client_secret=client_secret,
resource_owner_key=flask.session["owner_key"],
resource_owner_secret=flask.session["owner_secret"],
)
oauth_response = oauth.parse_authorization_response(flask.request.url)
verifier = oauth_response.get("oauth_verifier")
2022-08-16 12:43:03 +01:00
access_token_url = wiki_index_php + "?title=Special%3aOAuth%2ftoken"
2022-08-15 11:43:22 +01:00
oauth = OAuth1Session(
client_key,
client_secret=client_secret,
resource_owner_key=flask.session["owner_key"],
resource_owner_secret=flask.session["owner_secret"],
verifier=verifier,
)
oauth_tokens = oauth.fetch_access_token(access_token_url)
flask.session["owner_key"] = oauth_tokens.get("oauth_token")
flask.session["owner_secret"] = oauth_tokens.get("oauth_token_secret")
next_page = flask.session.get("after_login")
return flask.redirect(next_page) if next_page else flask.url_for("index")
@app.route("/oauth/disconnect")
def oauth_disconnect():
for key in "owner_key", "owner_secret", "username", "after_login":
if key in flask.session:
del flask.session[key]
return flask.redirect(flask.url_for("index"))
2022-08-13 13:16:49 +01:00
if __name__ == "__main__":
app.run(host="0.0.0.0")