edward/goodreads-backup
1
0
Fork 0
Code Issues 1 Pull requests Packages Projects Releases Wiki Activity

Split code into smaller functions

Browse source
This commit is contained in:
Edward Betts 2024-04-17 09:55:24 +01:00
parent 4ac90301d5
commit c1cbfd5f79
1 changed files with 107 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

191
parse.py
View file

@ -1,26 +1,28 @@
#!/usr/bin/python3 #!/usr/bin/python3
import json
import os import os
import re import re
import lxml.html
import jinja2
import json
import sys import sys
import requests import typing
from time import sleep from time import sleep
from pprint import pprint
import jinja2
import lxml.html
import requests
parser = lxml.html.HTMLParser(encoding="utf-8") parser = lxml.html.HTMLParser(encoding="utf-8")
env = jinja2.Environment(loader=jinja2.FileSystemLoader("templates")) env = jinja2.Environment(loader=jinja2.FileSystemLoader("templates"))
re_book = re.compile( re_book = re.compile(
"function refreshGroupBox(group_id, book_id) \{(.*?)\n *\}", re.DOTALL r"function refreshGroupBox(group_id, book_id) \{(.*?)\n *\}", re.DOTALL
) )
re_book = re.compile( re_book = re.compile(
"function refreshGroupBox\(group_id, book_id\) \{(.*?)\n *\}", re.DOTALL r"function refreshGroupBox\(group_id, book_id\) \{(.*?)\n *\}", re.DOTALL
) )
re_tip = re.compile(rb'var newTip = new Tip\(\$\(\'[^\']+\'\), "(.*?)", {') re_tip = re.compile(r'var newTip = new Tip\(\$\(\'[^\']+\'\), "(.*?)", {')
dirs = ["shelf", "genre"] dirs = ["shelf", "genre"]
start = "https://www.goodreads.com/book/show/" start = "https://www.goodreads.com/book/show/"
@ -28,30 +30,44 @@ start = "https://www.goodreads.com/book/show/"
existing = {book["title"] for book in json.load(open("calibre_book_list.json"))} existing = {book["title"] for book in json.load(open("calibre_book_list.json"))}
def parse_book_div(div: lxml.html.HtmlElement) -> dict[str, typing.Any]:
"""Parse book div."""
link = div.find(".//a")
assert link is not None
rating = div.find('.//span[@class="minirating"]')
description = div.find('./div[@class="bookDescription"]')
assert rating is not None and rating[0] is not None
r = rating[0].tail
assert r
div_next = div.getnext()
assert div_next is not None
cover_img = div_next.find(".//img")
assert cover_img is not None
cover = cover_img.get("src")
book = {
"title": link.text,
"url": link.get("href"),
"rating": r,
"r": float(r[:3].strip()),
"cover": cover,
"authors": [a.text for a in div.find_class("authorName")],
}
if description is not None:
index = 1 if len(description) == 3 else 0
book["description"] = description[index].text
return book
def iter_books(): def iter_books():
for d in dirs: for d in dirs:
for f in sorted(os.listdir(d)): for f in sorted(os.listdir(d)):
filename = os.path.join(d, f) filename = os.path.join(d, f)
root = lxml.html.parse(filename, parser=parser).getroot() root = lxml.html.parse(filename, parser=parser).getroot()
for div in root.find_class("bookInformation"): for div in root.find_class("bookInformation"):
link = div.find(".//a") book = parse_book_div(div)
rating = div.find('.//span[@class="minirating"]')
description = div.find('./div[@class="bookDescription"]')
r = rating[0].tail.strip()
cover = div.getnext().find(".//img").get("src")
book = {
"title": link.text,
"url": link.get("href"),
"rating": r,
"r": float(r[:3]),
"cover": cover,
"authors": [a.text for a in div.find_class("authorName")],
}
if description is not None:
index = 1 if len(description) == 3 else 0
book["description"] = description[index].text
yield d, f, book yield d, f, book
continue continue
@ -61,7 +77,7 @@ def iter_books():
continue continue
print(line) print(line)
m = re_tip.search(line) m = re_tip.search(line)
tip = m.group(1).decode("unicode_escape").replace("\/", "/") tip = m.group(1).decode("unicode_escape").replace(r"\/", "/")
# tip = m.group(1) # .replace('\/', '/') # tip = m.group(1) # .replace('\/', '/')
# print(tip) # print(tip)
if '<ul class="formatting_tips recommendation_tip">' in tip: if '<ul class="formatting_tips recommendation_tip">' in tip:
@ -74,65 +90,72 @@ def iter_books():
yield (d, f, lxml.html.fromstring(tip)) yield (d, f, lxml.html.fromstring(tip))
template = env.get_template("books.html") def main() -> None:
seen = set() """Download books."""
books = [] template = env.get_template("books.html")
first_authors = set() seen = set()
for d, f, book in sorted(iter_books(), key=lambda i: i[2]["r"], reverse=True): books = []
# pprint(book) first_authors = set()
# print(repr(book.get('description'))) for d, f, book in sorted(iter_books(), key=lambda i: i[2]["r"], reverse=True):
# continue # pprint(book)
# print(repr(book.get('description')))
# continue
# title_link = book.find_class('bookTitle')[0] # title_link = book.find_class('bookTitle')[0]
url = book["url"] url = book["url"]
url = url[: url.find("?")] url = url[: url.find("?")]
if url in seen: if url in seen:
continue continue
seen.add(url) seen.add(url)
title = book["title"] title = book["title"]
authors = book["authors"] authors = book["authors"]
first_authors.add(authors[0]) first_authors.add(authors[0])
main_title = title main_title = title
# for sep in ['(']: # for sep in ['(']:
for sep in ":", " - ", "(", ",": for sep in ":", " - ", "(", ",":
if sep in title: if sep in title:
main_title = title[: title.find(sep)] main_title = title[: title.find(sep)]
break break
# print((main_title + ' by ' + u', '.join(authors)).encode('utf-8')) # print((main_title + ' by ' + u', '.join(authors)).encode('utf-8'))
if len(main_title) < 10: if len(main_title) < 10:
continue continue
if main_title in existing: if main_title in existing:
continue continue
# print(u'{} by {}'.format(main_title, authors[0]).encode('utf-8')) # print(u'{} by {}'.format(main_title, authors[0]).encode('utf-8'))
print("{}".format(main_title)) print("{}".format(main_title))
# print(main_title.encode('utf-8')) # print(main_title.encode('utf-8'))
assert url.startswith(start) assert url.startswith(start)
filename = "books/" + url[len(start) :] + ".html" filename = "books/" + url[len(start) :] + ".html"
# print(filename) # print(filename)
if False and not os.path.exists(filename): if False and not os.path.exists(filename):
open(filename, "w").write(requests.get(url).content) open(filename, "w").write(requests.get(url).content)
sleep(1) sleep(1)
books.append( books.append(
{ {
"dir": d, "dir": d,
"file": f[:-5], "file": f[:-5],
"title": title, "title": title,
"main_title": main_title, "main_title": main_title,
"authors": authors, "authors": authors,
"url": url, "url": url,
"rating": book["rating"], "rating": book["rating"],
"cover": book["cover"], "cover": book["cover"],
"description": book.get("description"), "description": book.get("description"),
} }
) )
page = template.render(books=books) page = template.render(books=books)
open("book_list.html", "w").write(page) open("book_list.html", "w").write(page)
sys.exit(0) sys.exit(0)
for a in sorted(first_authors): for a in sorted(first_authors):
print(a) print(a)
# authors = u' OR '.join(u'"{}"'.format(a) for a in sorted(first_authors) if a not in {'Hugh Howey', 'Elizabeth Moon', 'Max Hastings'}) # authors = u' OR '.join(u'"{}"'.format(a) for a in sorted(first_authors)
# print(authors.encode('utf-8')) # if a not in {'Hugh Howey', 'Elizabeth Moon', 'Max Hastings'})
# print(authors.encode('utf-8'))
if __name__ == "__main__":
main()