goodreads-backup/get.py

113 lines
2.7 KiB
Python
Raw Normal View History

#!/usr/bin/python3
2024-04-17 10:02:26 +01:00
"""Download shelves from goodreads."""
import os
import re
2024-04-17 10:02:26 +01:00
from http.cookiejar import LWPCookieJar
from random import shuffle
from time import sleep
2024-04-17 10:02:26 +01:00
import lxml.html
import requests
2024-04-17 09:12:11 +01:00
re_recommend = re.compile(
' <a class="actionLinkLite " href="(/recommendations/([^/]*?)/([^/]*?))">'
)
s = requests.Session()
2024-04-17 09:12:11 +01:00
cookie_dir = "/home/edward/lib/cookies"
cookie_file = os.path.join(cookie_dir, "goodreads")
cj = LWPCookieJar(cookie_file)
if os.path.exists(cookie_file):
cj.load()
2024-04-17 10:02:26 +01:00
s.cookies = cj # type: ignore
2024-04-17 09:12:11 +01:00
2024-04-17 10:02:26 +01:00
def login() -> None:
"""Login."""
2024-04-17 09:12:11 +01:00
sign_in_page = "https://www.goodreads.com/user/sign_in"
page = s.get(sign_in_page).text
2024-04-17 09:12:11 +01:00
open("sign_in.html", "w").write(page)
if '"name":"Edward Betts"' in page:
return # already signed in
2024-04-17 09:12:11 +01:00
re_token = re.compile(
'<input type="hidden" name="authenticity_token" value="([^"]*?)" />'
)
2024-04-17 10:02:26 +01:00
re_n = re.compile(r"<input name='n' type='hidden' value='(\d+)'>")
m_n = re_n.search(page)
m_token = re_token.search(page)
2024-04-17 10:02:26 +01:00
assert m_token and m_n
token = m_token.group(1)
data = {
2024-04-17 09:12:11 +01:00
"utf8": "\u2713",
"authenticity_token": token,
"user[email]": "edward@4angle.com",
"user[password]": "8V8~9:3~U!Ly",
"remember_me": 1,
"next": "Sign in",
2024-04-17 10:02:26 +01:00
"n": m_n.group(1),
}
print(token)
2024-04-17 09:12:11 +01:00
print(data["n"])
2024-04-17 09:12:11 +01:00
r = s.post(sign_in_page, data=data, headers={"referer": sign_in_page})
2024-04-17 09:12:11 +01:00
open("signed_in.html", "w").write(r.text)
root = lxml.html.fromstring(r.content)
2024-04-17 09:12:11 +01:00
flash = root.find_class("flash")
if flash:
2024-04-17 09:12:11 +01:00
print("flash:", flash[0].text)
cj.save(ignore_discard=True)
2024-04-17 09:12:11 +01:00
2024-04-17 10:02:26 +01:00
def get_index() -> None:
"""Get index."""
# url = 'https://www.goodreads.com/recommendations'
2024-04-17 09:12:11 +01:00
url = "https://www.goodreads.com/recommendations/?recs_current_view=list"
r = s.get(url)
2024-04-17 09:12:11 +01:00
open("recommendations.html", "w").write(r.text)
def get_individual():
2024-04-17 10:02:26 +01:00
"""Get individual page."""
2024-04-17 09:12:11 +01:00
for line in open("recommendations.html"):
if "actionLinkLite" not in line:
continue
m = re_recommend.match(line)
if m:
yield m.groups()
2024-04-17 10:02:26 +01:00
def main() -> None:
"""Login and download shelves."""
# art = 'https://www.goodreads.com/recommendations/genre/art'
login()
get_index()
recommend_list = list(get_individual())
shuffle(recommend_list)
headers = {"Accept": "text/html"}
for a, b, c in recommend_list:
print((b, c))
url = "https://www.goodreads.com" + a
2024-04-17 10:02:26 +01:00
r = s.get(url, headers=headers)
filename = os.path.join(b, c + ".html")
open(filename, "w").write(r.text)
sleep(0.5)
2024-04-17 10:02:26 +01:00
if __name__ == "__main__":
main()