Compare commits
No commits in common. "4ac90301d5103c0ded1c8b04c3c0578811d9f052" and "271251c1f4a1b77c8711f0bbc79ef51aadf2ae34" have entirely different histories.
4ac90301d5
...
271251c1f4
|
|
@ -68,7 +68,7 @@ def run_backup(page: Page) -> None:
|
||||||
|
|
||||||
def run(playwright: Playwright) -> None:
|
def run(playwright: Playwright) -> None:
|
||||||
"""Download export."""
|
"""Download export."""
|
||||||
browser = playwright.chromium.launch(headless=True)
|
browser = playwright.chromium.launch(headless=False)
|
||||||
auth_json = os.path.join(script_dir, "auth.json")
|
auth_json = os.path.join(script_dir, "auth.json")
|
||||||
context = browser.new_context(storage_state=auth_json)
|
context = browser.new_context(storage_state=auth_json)
|
||||||
page = context.new_page()
|
page = context.new_page()
|
||||||
|
|
|
||||||
97
get.py
97
get.py
|
|
@ -1,97 +0,0 @@
|
||||||
#!/usr/bin/python3
|
|
||||||
|
|
||||||
import requests
|
|
||||||
from http.cookiejar import LWPCookieJar
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import lxml.html
|
|
||||||
from random import shuffle
|
|
||||||
from time import sleep
|
|
||||||
|
|
||||||
re_recommend = re.compile(
|
|
||||||
' <a class="actionLinkLite " href="(/recommendations/([^/]*?)/([^/]*?))">'
|
|
||||||
)
|
|
||||||
|
|
||||||
s = requests.Session()
|
|
||||||
|
|
||||||
cookie_dir = "/home/edward/lib/cookies"
|
|
||||||
cookie_file = os.path.join(cookie_dir, "goodreads")
|
|
||||||
|
|
||||||
cj = LWPCookieJar(cookie_file)
|
|
||||||
if os.path.exists(cookie_file):
|
|
||||||
cj.load()
|
|
||||||
s.cookies = cj
|
|
||||||
|
|
||||||
|
|
||||||
def login():
|
|
||||||
sign_in_page = "https://www.goodreads.com/user/sign_in"
|
|
||||||
page = s.get(sign_in_page).text
|
|
||||||
open("sign_in.html", "w").write(page)
|
|
||||||
if '"name":"Edward Betts"' in page:
|
|
||||||
return # already signed in
|
|
||||||
|
|
||||||
re_token = re.compile(
|
|
||||||
'<input type="hidden" name="authenticity_token" value="([^"]*?)" />'
|
|
||||||
)
|
|
||||||
re_n = re.compile("<input name='n' type='hidden' value='(\d+)'>")
|
|
||||||
|
|
||||||
token = re_token.search(page).group(1)
|
|
||||||
|
|
||||||
data = {
|
|
||||||
"utf8": "\u2713",
|
|
||||||
"authenticity_token": token,
|
|
||||||
"user[email]": "edward@4angle.com",
|
|
||||||
"user[password]": "8V8~9:3~U!Ly",
|
|
||||||
"remember_me": 1,
|
|
||||||
"next": "Sign in",
|
|
||||||
"n": re_n.search(page).group(1),
|
|
||||||
}
|
|
||||||
|
|
||||||
print(token)
|
|
||||||
print(data["n"])
|
|
||||||
|
|
||||||
r = s.post(sign_in_page, data=data, headers={"referer": sign_in_page})
|
|
||||||
|
|
||||||
open("signed_in.html", "w").write(r.text)
|
|
||||||
|
|
||||||
root = lxml.html.fromstring(r.content)
|
|
||||||
flash = root.find_class("flash")
|
|
||||||
if flash:
|
|
||||||
print("flash:", flash[0].text)
|
|
||||||
|
|
||||||
cj.save(ignore_discard=True)
|
|
||||||
|
|
||||||
|
|
||||||
def get_index():
|
|
||||||
# url = 'https://www.goodreads.com/recommendations'
|
|
||||||
url = "https://www.goodreads.com/recommendations/?recs_current_view=list"
|
|
||||||
|
|
||||||
r = s.get(url)
|
|
||||||
open("recommendations.html", "w").write(r.text)
|
|
||||||
|
|
||||||
|
|
||||||
def get_individual():
|
|
||||||
for line in open("recommendations.html"):
|
|
||||||
if "actionLinkLite" not in line:
|
|
||||||
continue
|
|
||||||
m = re_recommend.match(line)
|
|
||||||
if m:
|
|
||||||
yield m.groups()
|
|
||||||
|
|
||||||
|
|
||||||
# art = 'https://www.goodreads.com/recommendations/genre/art'
|
|
||||||
login()
|
|
||||||
get_index()
|
|
||||||
recommend_list = list(get_individual())
|
|
||||||
shuffle(recommend_list)
|
|
||||||
|
|
||||||
headers = {"Accept": "text/html"}
|
|
||||||
|
|
||||||
for a, b, c in recommend_list:
|
|
||||||
print((b, c))
|
|
||||||
url = "https://www.goodreads.com" + a
|
|
||||||
|
|
||||||
r = s.get(url, headers=headers)
|
|
||||||
filename = os.path.join(b, c + ".html")
|
|
||||||
open(filename, "w").write(r.text)
|
|
||||||
sleep(0.5)
|
|
||||||
138
parse.py
138
parse.py
|
|
@ -1,138 +0,0 @@
|
||||||
#!/usr/bin/python3
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import lxml.html
|
|
||||||
import jinja2
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
import requests
|
|
||||||
from time import sleep
|
|
||||||
from pprint import pprint
|
|
||||||
|
|
||||||
parser = lxml.html.HTMLParser(encoding="utf-8")
|
|
||||||
|
|
||||||
env = jinja2.Environment(loader=jinja2.FileSystemLoader("templates"))
|
|
||||||
|
|
||||||
re_book = re.compile(
|
|
||||||
"function refreshGroupBox(group_id, book_id) \{(.*?)\n *\}", re.DOTALL
|
|
||||||
)
|
|
||||||
re_book = re.compile(
|
|
||||||
"function refreshGroupBox\(group_id, book_id\) \{(.*?)\n *\}", re.DOTALL
|
|
||||||
)
|
|
||||||
|
|
||||||
re_tip = re.compile(rb'var newTip = new Tip\(\$\(\'[^\']+\'\), "(.*?)", {')
|
|
||||||
|
|
||||||
dirs = ["shelf", "genre"]
|
|
||||||
start = "https://www.goodreads.com/book/show/"
|
|
||||||
|
|
||||||
existing = {book["title"] for book in json.load(open("calibre_book_list.json"))}
|
|
||||||
|
|
||||||
|
|
||||||
def iter_books():
|
|
||||||
for d in dirs:
|
|
||||||
for f in sorted(os.listdir(d)):
|
|
||||||
filename = os.path.join(d, f)
|
|
||||||
root = lxml.html.parse(filename, parser=parser).getroot()
|
|
||||||
for div in root.find_class("bookInformation"):
|
|
||||||
link = div.find(".//a")
|
|
||||||
rating = div.find('.//span[@class="minirating"]')
|
|
||||||
description = div.find('./div[@class="bookDescription"]')
|
|
||||||
r = rating[0].tail.strip()
|
|
||||||
|
|
||||||
cover = div.getnext().find(".//img").get("src")
|
|
||||||
|
|
||||||
book = {
|
|
||||||
"title": link.text,
|
|
||||||
"url": link.get("href"),
|
|
||||||
"rating": r,
|
|
||||||
"r": float(r[:3]),
|
|
||||||
"cover": cover,
|
|
||||||
"authors": [a.text for a in div.find_class("authorName")],
|
|
||||||
}
|
|
||||||
if description is not None:
|
|
||||||
index = 1 if len(description) == 3 else 0
|
|
||||||
book["description"] = description[index].text
|
|
||||||
yield d, f, book
|
|
||||||
continue
|
|
||||||
|
|
||||||
# print(filename)
|
|
||||||
for line in open(filename, "rb"):
|
|
||||||
if b"var newTip" not in line:
|
|
||||||
continue
|
|
||||||
print(line)
|
|
||||||
m = re_tip.search(line)
|
|
||||||
tip = m.group(1).decode("unicode_escape").replace("\/", "/")
|
|
||||||
# tip = m.group(1) # .replace('\/', '/')
|
|
||||||
# print(tip)
|
|
||||||
if '<ul class="formatting_tips recommendation_tip">' in tip:
|
|
||||||
continue
|
|
||||||
if "Recommendations are disabled for that shelf." in tip:
|
|
||||||
continue
|
|
||||||
if "Customize by selecting your" in tip:
|
|
||||||
continue
|
|
||||||
print(tip)
|
|
||||||
yield (d, f, lxml.html.fromstring(tip))
|
|
||||||
|
|
||||||
|
|
||||||
template = env.get_template("books.html")
|
|
||||||
seen = set()
|
|
||||||
books = []
|
|
||||||
first_authors = set()
|
|
||||||
for d, f, book in sorted(iter_books(), key=lambda i: i[2]["r"], reverse=True):
|
|
||||||
# pprint(book)
|
|
||||||
# print(repr(book.get('description')))
|
|
||||||
# continue
|
|
||||||
|
|
||||||
# title_link = book.find_class('bookTitle')[0]
|
|
||||||
url = book["url"]
|
|
||||||
url = url[: url.find("?")]
|
|
||||||
if url in seen:
|
|
||||||
continue
|
|
||||||
seen.add(url)
|
|
||||||
title = book["title"]
|
|
||||||
authors = book["authors"]
|
|
||||||
first_authors.add(authors[0])
|
|
||||||
main_title = title
|
|
||||||
# for sep in ['(']:
|
|
||||||
for sep in ":", " - ", "(", ",":
|
|
||||||
if sep in title:
|
|
||||||
main_title = title[: title.find(sep)]
|
|
||||||
break
|
|
||||||
# print((main_title + ' by ' + u', '.join(authors)).encode('utf-8'))
|
|
||||||
if len(main_title) < 10:
|
|
||||||
continue
|
|
||||||
if main_title in existing:
|
|
||||||
continue
|
|
||||||
# print(u'{} by {}'.format(main_title, authors[0]).encode('utf-8'))
|
|
||||||
print("{}".format(main_title))
|
|
||||||
# print(main_title.encode('utf-8'))
|
|
||||||
assert url.startswith(start)
|
|
||||||
|
|
||||||
filename = "books/" + url[len(start) :] + ".html"
|
|
||||||
# print(filename)
|
|
||||||
if False and not os.path.exists(filename):
|
|
||||||
open(filename, "w").write(requests.get(url).content)
|
|
||||||
sleep(1)
|
|
||||||
books.append(
|
|
||||||
{
|
|
||||||
"dir": d,
|
|
||||||
"file": f[:-5],
|
|
||||||
"title": title,
|
|
||||||
"main_title": main_title,
|
|
||||||
"authors": authors,
|
|
||||||
"url": url,
|
|
||||||
"rating": book["rating"],
|
|
||||||
"cover": book["cover"],
|
|
||||||
"description": book.get("description"),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
page = template.render(books=books)
|
|
||||||
open("book_list.html", "w").write(page)
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
for a in sorted(first_authors):
|
|
||||||
print(a)
|
|
||||||
|
|
||||||
# authors = u' OR '.join(u'"{}"'.format(a) for a in sorted(first_authors) if a not in {'Hugh Howey', 'Elizabeth Moon', 'Max Hastings'})
|
|
||||||
# print(authors.encode('utf-8'))
|
|
||||||
|
|
@ -1,39 +0,0 @@
|
||||||
<html>
|
|
||||||
<meta charset="utf-8">
|
|
||||||
<table>
|
|
||||||
{% for book in books %}
|
|
||||||
{% set url_title = ('"' + book.main_title.replace('.', ' ') + '"') | urlencode %}
|
|
||||||
{% set title_and_author = ('"' + book.main_title + '" ' + book.authors.0) | urlencode %}
|
|
||||||
<tr>
|
|
||||||
<td rowspan="3"><img src="{{ book.cover }}" /></td>
|
|
||||||
<td nowrap="nowrap">
|
|
||||||
<a href="http://1337x.to/search/{{- title_and_author -}}/1/">1337x</a>
|
|
||||||
{# | <a href="https://www.google.co.uk/search?q=site%3Aforum.mobilism.org+{{- title_and_author -}}">mobilism</a> #}
|
|
||||||
| <a href="https://forum.mobilism.org/search.php?sr=topics&keywords={{- title_and_author -}}">mobilism</a>
|
|
||||||
| <a href="http://en.booksee.org/s/?q={{- title_and_author -}}">booksee</a>
|
|
||||||
| <a href="http://gen.lib.rus.ec/search.php?req={{- title_and_author -}}">Library Genesis</a>
|
|
||||||
| <a href="http://libgen.io/foreignfiction/index.php?s={{- book.authors.0 | urlencode -}}">Library Genesis Fiction</a>
|
|
||||||
</td>
|
|
||||||
<td nowrap="nowrap">
|
|
||||||
{{ book.dir }} / {{ book.file }}
|
|
||||||
</td>
|
|
||||||
<td nowrap="nowrap">
|
|
||||||
{{ book.rating }}
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td colspan="3"><a href="{{ book.url }}">{{ book.title }}</a>
|
|
||||||
by {% for a in book.authors %}{{ a -}}{% if not loop.last %}, {% endif %}{% endfor %}<br>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td colspan="3">
|
|
||||||
{% if book.description %}
|
|
||||||
{% for p in book.description.split('\xa0') %}
|
|
||||||
<p>{{ p }}</p>
|
|
||||||
{% endfor %}
|
|
||||||
{% endif %}
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
{% endfor %}
|
|
||||||
</html>
|
|
||||||
Loading…
Reference in a new issue