Compare commits

..

2 commits

Author SHA1 Message Date
Edward Betts 99844005d6 Update code style 2024-04-17 10:02:26 +01:00
Edward Betts c1cbfd5f79 Split code into smaller functions 2024-04-17 09:55:24 +01:00
2 changed files with 144 additions and 106 deletions

47
get.py
View file

@ -1,13 +1,16 @@
#!/usr/bin/python3 #!/usr/bin/python3
import requests """Download shelves from goodreads."""
from http.cookiejar import LWPCookieJar
import os import os
import re import re
import lxml.html from http.cookiejar import LWPCookieJar
from random import shuffle from random import shuffle
from time import sleep from time import sleep
import lxml.html
import requests
re_recommend = re.compile( re_recommend = re.compile(
' <a class="actionLinkLite " href="(/recommendations/([^/]*?)/([^/]*?))">' ' <a class="actionLinkLite " href="(/recommendations/([^/]*?)/([^/]*?))">'
) )
@ -20,10 +23,11 @@ cookie_file = os.path.join(cookie_dir, "goodreads")
cj = LWPCookieJar(cookie_file) cj = LWPCookieJar(cookie_file)
if os.path.exists(cookie_file): if os.path.exists(cookie_file):
cj.load() cj.load()
s.cookies = cj s.cookies = cj # type: ignore
def login(): def login() -> None:
"""Login."""
sign_in_page = "https://www.goodreads.com/user/sign_in" sign_in_page = "https://www.goodreads.com/user/sign_in"
page = s.get(sign_in_page).text page = s.get(sign_in_page).text
open("sign_in.html", "w").write(page) open("sign_in.html", "w").write(page)
@ -33,9 +37,12 @@ def login():
re_token = re.compile( re_token = re.compile(
'<input type="hidden" name="authenticity_token" value="([^"]*?)" />' '<input type="hidden" name="authenticity_token" value="([^"]*?)" />'
) )
re_n = re.compile("<input name='n' type='hidden' value='(\d+)'>") re_n = re.compile(r"<input name='n' type='hidden' value='(\d+)'>")
m_n = re_n.search(page)
m_token = re_token.search(page)
token = re_token.search(page).group(1) assert m_token and m_n
token = m_token.group(1)
data = { data = {
"utf8": "\u2713", "utf8": "\u2713",
@ -44,7 +51,7 @@ def login():
"user[password]": "8V8~9:3~U!Ly", "user[password]": "8V8~9:3~U!Ly",
"remember_me": 1, "remember_me": 1,
"next": "Sign in", "next": "Sign in",
"n": re_n.search(page).group(1), "n": m_n.group(1),
} }
print(token) print(token)
@ -62,7 +69,8 @@ def login():
cj.save(ignore_discard=True) cj.save(ignore_discard=True)
def get_index(): def get_index() -> None:
"""Get index."""
# url = 'https://www.goodreads.com/recommendations' # url = 'https://www.goodreads.com/recommendations'
url = "https://www.goodreads.com/recommendations/?recs_current_view=list" url = "https://www.goodreads.com/recommendations/?recs_current_view=list"
@ -71,6 +79,7 @@ def get_index():
def get_individual(): def get_individual():
"""Get individual page."""
for line in open("recommendations.html"): for line in open("recommendations.html"):
if "actionLinkLite" not in line: if "actionLinkLite" not in line:
continue continue
@ -79,15 +88,17 @@ def get_individual():
yield m.groups() yield m.groups()
# art = 'https://www.goodreads.com/recommendations/genre/art' def main() -> None:
login() """Login and download shelves."""
get_index() # art = 'https://www.goodreads.com/recommendations/genre/art'
recommend_list = list(get_individual()) login()
shuffle(recommend_list) get_index()
recommend_list = list(get_individual())
shuffle(recommend_list)
headers = {"Accept": "text/html"} headers = {"Accept": "text/html"}
for a, b, c in recommend_list: for a, b, c in recommend_list:
print((b, c)) print((b, c))
url = "https://www.goodreads.com" + a url = "https://www.goodreads.com" + a
@ -95,3 +106,7 @@ for a, b, c in recommend_list:
filename = os.path.join(b, c + ".html") filename = os.path.join(b, c + ".html")
open(filename, "w").write(r.text) open(filename, "w").write(r.text)
sleep(0.5) sleep(0.5)
if __name__ == "__main__":
main()

View file

@ -1,26 +1,28 @@
#!/usr/bin/python3 #!/usr/bin/python3
import json
import os import os
import re import re
import lxml.html
import jinja2
import json
import sys import sys
import requests import typing
from time import sleep from time import sleep
from pprint import pprint
import jinja2
import lxml.html
import requests
parser = lxml.html.HTMLParser(encoding="utf-8") parser = lxml.html.HTMLParser(encoding="utf-8")
env = jinja2.Environment(loader=jinja2.FileSystemLoader("templates")) env = jinja2.Environment(loader=jinja2.FileSystemLoader("templates"))
re_book = re.compile( re_book = re.compile(
"function refreshGroupBox(group_id, book_id) \{(.*?)\n *\}", re.DOTALL r"function refreshGroupBox(group_id, book_id) \{(.*?)\n *\}", re.DOTALL
) )
re_book = re.compile( re_book = re.compile(
"function refreshGroupBox\(group_id, book_id\) \{(.*?)\n *\}", re.DOTALL r"function refreshGroupBox\(group_id, book_id\) \{(.*?)\n *\}", re.DOTALL
) )
re_tip = re.compile(rb'var newTip = new Tip\(\$\(\'[^\']+\'\), "(.*?)", {') re_tip = re.compile(r'var newTip = new Tip\(\$\(\'[^\']+\'\), "(.*?)", {')
dirs = ["shelf", "genre"] dirs = ["shelf", "genre"]
start = "https://www.goodreads.com/book/show/" start = "https://www.goodreads.com/book/show/"
@ -28,30 +30,44 @@ start = "https://www.goodreads.com/book/show/"
existing = {book["title"] for book in json.load(open("calibre_book_list.json"))} existing = {book["title"] for book in json.load(open("calibre_book_list.json"))}
def parse_book_div(div: lxml.html.HtmlElement) -> dict[str, typing.Any]:
"""Parse book div."""
link = div.find(".//a")
assert link is not None
rating = div.find('.//span[@class="minirating"]')
description = div.find('./div[@class="bookDescription"]')
assert rating is not None and rating[0] is not None
r = rating[0].tail
assert r
div_next = div.getnext()
assert div_next is not None
cover_img = div_next.find(".//img")
assert cover_img is not None
cover = cover_img.get("src")
book = {
"title": link.text,
"url": link.get("href"),
"rating": r,
"r": float(r[:3].strip()),
"cover": cover,
"authors": [a.text for a in div.find_class("authorName")],
}
if description is not None:
index = 1 if len(description) == 3 else 0
book["description"] = description[index].text
return book
def iter_books(): def iter_books():
for d in dirs: for d in dirs:
for f in sorted(os.listdir(d)): for f in sorted(os.listdir(d)):
filename = os.path.join(d, f) filename = os.path.join(d, f)
root = lxml.html.parse(filename, parser=parser).getroot() root = lxml.html.parse(filename, parser=parser).getroot()
for div in root.find_class("bookInformation"): for div in root.find_class("bookInformation"):
link = div.find(".//a") book = parse_book_div(div)
rating = div.find('.//span[@class="minirating"]')
description = div.find('./div[@class="bookDescription"]')
r = rating[0].tail.strip()
cover = div.getnext().find(".//img").get("src")
book = {
"title": link.text,
"url": link.get("href"),
"rating": r,
"r": float(r[:3]),
"cover": cover,
"authors": [a.text for a in div.find_class("authorName")],
}
if description is not None:
index = 1 if len(description) == 3 else 0
book["description"] = description[index].text
yield d, f, book yield d, f, book
continue continue
@ -61,7 +77,7 @@ def iter_books():
continue continue
print(line) print(line)
m = re_tip.search(line) m = re_tip.search(line)
tip = m.group(1).decode("unicode_escape").replace("\/", "/") tip = m.group(1).decode("unicode_escape").replace(r"\/", "/")
# tip = m.group(1) # .replace('\/', '/') # tip = m.group(1) # .replace('\/', '/')
# print(tip) # print(tip)
if '<ul class="formatting_tips recommendation_tip">' in tip: if '<ul class="formatting_tips recommendation_tip">' in tip:
@ -74,11 +90,13 @@ def iter_books():
yield (d, f, lxml.html.fromstring(tip)) yield (d, f, lxml.html.fromstring(tip))
template = env.get_template("books.html") def main() -> None:
seen = set() """Download books."""
books = [] template = env.get_template("books.html")
first_authors = set() seen = set()
for d, f, book in sorted(iter_books(), key=lambda i: i[2]["r"], reverse=True): books = []
first_authors = set()
for d, f, book in sorted(iter_books(), key=lambda i: i[2]["r"], reverse=True):
# pprint(book) # pprint(book)
# print(repr(book.get('description'))) # print(repr(book.get('description')))
# continue # continue
@ -127,12 +145,17 @@ for d, f, book in sorted(iter_books(), key=lambda i: i[2]["r"], reverse=True):
} }
) )
page = template.render(books=books) page = template.render(books=books)
open("book_list.html", "w").write(page) open("book_list.html", "w").write(page)
sys.exit(0) sys.exit(0)
for a in sorted(first_authors): for a in sorted(first_authors):
print(a) print(a)
# authors = u' OR '.join(u'"{}"'.format(a) for a in sorted(first_authors) if a not in {'Hugh Howey', 'Elizabeth Moon', 'Max Hastings'}) # authors = u' OR '.join(u'"{}"'.format(a) for a in sorted(first_authors)
# print(authors.encode('utf-8')) # if a not in {'Hugh Howey', 'Elizabeth Moon', 'Max Hastings'})
# print(authors.encode('utf-8'))
if __name__ == "__main__":
main()