diff --git a/crawl.py b/crawl.py index 9e24331..e397a9f 100755 --- a/crawl.py +++ b/crawl.py @@ -4,6 +4,7 @@ import collections import configparser import decimal +import sqlite3 import logging import os import random @@ -333,6 +334,46 @@ def group_items( yield {"name": name, "label": label, "items": items} +db_path = os.path.join(data_root, "prices.db") +sqlite3.register_adapter(Decimal, str) + + +def init_db(conn: sqlite3.Connection) -> None: + """Create tables if they don't exist.""" + conn.execute(""" + CREATE TABLE IF NOT EXISTS price ( + item_number TEXT NOT NULL, + title TEXT NOT NULL, + size_gb NUMERIC NOT NULL, + price NUMERIC NOT NULL, + category TEXT NOT NULL, + seen_at DATE NOT NULL, + PRIMARY KEY (item_number, seen_at) + ) + """) + conn.commit() + + +def record_prices(data: list[Grouped], today: date) -> None: + """Record today's prices to the database.""" + conn = sqlite3.connect(db_path) + init_db(conn) + for cat in data: + for item in cat["items"]: + conn.execute( + """ + INSERT OR REPLACE INTO price + (item_number, title, size_gb, price, category, seen_at) + VALUES (?, ?, ?, ?, ?, ?) + """, + (item["number"], item["title"], item["size_gb"], + item["price"], cat["name"], today), + ) + conn.commit() + conn.close() + logger.info("prices recorded", db=db_path, date=today) + + def get_build_root() -> str: """Read build_dir from ~/.config/newegg-hdd/config, fall back to output/.""" config_path = os.path.expanduser("~/.config/newegg-hdd/config") @@ -353,6 +394,7 @@ def build() -> None: env = Environment(loader=FileSystemLoader(templates_dir)) data = list(group_items(today)) + record_prices(data, today) index = os.path.join(build_root, "index.html") index_template = env.get_template("index.html") page = index_template.render(best=data, today=today)