goodreads-backup/get.py

#!/usr/bin/python3

"""Download shelves from goodreads."""

import os
import re
from http.cookiejar import LWPCookieJar
from random import shuffle
from time import sleep

import lxml.html
import requests

re_recommend = re.compile(
    '  <a class="actionLinkLite " href="(/recommendations/([^/]*?)/([^/]*?))">'
)

s = requests.Session()

cookie_dir = "/home/edward/lib/cookies"
cookie_file = os.path.join(cookie_dir, "goodreads")

cj = LWPCookieJar(cookie_file)
if os.path.exists(cookie_file):
    cj.load()
s.cookies = cj  # type: ignore


def login() -> None:
    """Login."""
    sign_in_page = "https://www.goodreads.com/user/sign_in"
    page = s.get(sign_in_page).text
    open("sign_in.html", "w").write(page)
    if '"name":"Edward Betts"' in page:
        return  # already signed in

    re_token = re.compile(
        '<input type="hidden" name="authenticity_token" value="([^"]*?)" />'
    )
    re_n = re.compile(r"<input name='n' type='hidden' value='(\d+)'>")
    m_n = re_n.search(page)
    m_token = re_token.search(page)

    assert m_token and m_n
    token = m_token.group(1)

    data = {
        "utf8": "\u2713",
        "authenticity_token": token,
        "user[email]": "edward@4angle.com",
        "user[password]": "8V8~9:3~U!Ly",
        "remember_me": 1,
        "next": "Sign in",
        "n": m_n.group(1),
    }

    print(token)
    print(data["n"])

    r = s.post(sign_in_page, data=data, headers={"referer": sign_in_page})

    open("signed_in.html", "w").write(r.text)

    root = lxml.html.fromstring(r.content)
    flash = root.find_class("flash")
    if flash:
        print("flash:", flash[0].text)

    cj.save(ignore_discard=True)


def get_index() -> None:
    """Get index."""
    # url = 'https://www.goodreads.com/recommendations'
    url = "https://www.goodreads.com/recommendations/?recs_current_view=list"

    r = s.get(url)
    open("recommendations.html", "w").write(r.text)


def get_individual():
    """Get individual page."""
    for line in open("recommendations.html"):
        if "actionLinkLite" not in line:
            continue
        m = re_recommend.match(line)
        if m:
            yield m.groups()


def main() -> None:
    """Login and download shelves."""
    # art = 'https://www.goodreads.com/recommendations/genre/art'
    login()
    get_index()
    recommend_list = list(get_individual())
    shuffle(recommend_list)

    headers = {"Accept": "text/html"}

    for a, b, c in recommend_list:
        print((b, c))
        url = "https://www.goodreads.com" + a

        r = s.get(url, headers=headers)
        filename = os.path.join(b, c + ".html")
        open(filename, "w").write(r.text)
        sleep(0.5)


if __name__ == "__main__":
    main()