Record prices to SQLite database for historical tracking, closes #3
Stores item_number, title, size_gb, price, category, seen_at in data/prices.db with (item_number, seen_at) as the primary key — one row per item per day. Uses INSERT OR REPLACE so re-running for the same day is safe. Called automatically from build(). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
323a30d9e0
commit
d161e82865
1 changed files with 42 additions and 0 deletions
42
crawl.py
42
crawl.py
|
|
@ -4,6 +4,7 @@
|
|||
import collections
|
||||
import configparser
|
||||
import decimal
|
||||
import sqlite3
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
|
|
@ -333,6 +334,46 @@ def group_items(
|
|||
yield {"name": name, "label": label, "items": items}
|
||||
|
||||
|
||||
db_path = os.path.join(data_root, "prices.db")
|
||||
sqlite3.register_adapter(Decimal, str)
|
||||
|
||||
|
||||
def init_db(conn: sqlite3.Connection) -> None:
|
||||
"""Create tables if they don't exist."""
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS price (
|
||||
item_number TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
size_gb NUMERIC NOT NULL,
|
||||
price NUMERIC NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
seen_at DATE NOT NULL,
|
||||
PRIMARY KEY (item_number, seen_at)
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def record_prices(data: list[Grouped], today: date) -> None:
|
||||
"""Record today's prices to the database."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
init_db(conn)
|
||||
for cat in data:
|
||||
for item in cat["items"]:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO price
|
||||
(item_number, title, size_gb, price, category, seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(item["number"], item["title"], item["size_gb"],
|
||||
item["price"], cat["name"], today),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.info("prices recorded", db=db_path, date=today)
|
||||
|
||||
|
||||
def get_build_root() -> str:
|
||||
"""Read build_dir from ~/.config/newegg-hdd/config, fall back to output/."""
|
||||
config_path = os.path.expanduser("~/.config/newegg-hdd/config")
|
||||
|
|
@ -353,6 +394,7 @@ def build() -> None:
|
|||
env = Environment(loader=FileSystemLoader(templates_dir))
|
||||
|
||||
data = list(group_items(today))
|
||||
record_prices(data, today)
|
||||
index = os.path.join(build_root, "index.html")
|
||||
index_template = env.get_template("index.html")
|
||||
page = index_template.render(best=data, today=today)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue