Record prices to SQLite database for historical tracking, closes #3
Stores item_number, title, size_gb, price, category, seen_at in data/prices.db with (item_number, seen_at) as the primary key — one row per item per day. Uses INSERT OR REPLACE so re-running for the same day is safe. Called automatically from build(). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
323a30d9e0
commit
d161e82865
1 changed files with 42 additions and 0 deletions
42
crawl.py
42
crawl.py
|
|
@ -4,6 +4,7 @@
|
||||||
import collections
|
import collections
|
||||||
import configparser
|
import configparser
|
||||||
import decimal
|
import decimal
|
||||||
|
import sqlite3
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
|
|
@ -333,6 +334,46 @@ def group_items(
|
||||||
yield {"name": name, "label": label, "items": items}
|
yield {"name": name, "label": label, "items": items}
|
||||||
|
|
||||||
|
|
||||||
|
db_path = os.path.join(data_root, "prices.db")
|
||||||
|
sqlite3.register_adapter(Decimal, str)
|
||||||
|
|
||||||
|
|
||||||
|
def init_db(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Create tables if they don't exist."""
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS price (
|
||||||
|
item_number TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
size_gb NUMERIC NOT NULL,
|
||||||
|
price NUMERIC NOT NULL,
|
||||||
|
category TEXT NOT NULL,
|
||||||
|
seen_at DATE NOT NULL,
|
||||||
|
PRIMARY KEY (item_number, seen_at)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def record_prices(data: list[Grouped], today: date) -> None:
|
||||||
|
"""Record today's prices to the database."""
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
init_db(conn)
|
||||||
|
for cat in data:
|
||||||
|
for item in cat["items"]:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT OR REPLACE INTO price
|
||||||
|
(item_number, title, size_gb, price, category, seen_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(item["number"], item["title"], item["size_gb"],
|
||||||
|
item["price"], cat["name"], today),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
logger.info("prices recorded", db=db_path, date=today)
|
||||||
|
|
||||||
|
|
||||||
def get_build_root() -> str:
|
def get_build_root() -> str:
|
||||||
"""Read build_dir from ~/.config/newegg-hdd/config, fall back to output/."""
|
"""Read build_dir from ~/.config/newegg-hdd/config, fall back to output/."""
|
||||||
config_path = os.path.expanduser("~/.config/newegg-hdd/config")
|
config_path = os.path.expanduser("~/.config/newegg-hdd/config")
|
||||||
|
|
@ -353,6 +394,7 @@ def build() -> None:
|
||||||
env = Environment(loader=FileSystemLoader(templates_dir))
|
env = Environment(loader=FileSystemLoader(templates_dir))
|
||||||
|
|
||||||
data = list(group_items(today))
|
data = list(group_items(today))
|
||||||
|
record_prices(data, today)
|
||||||
index = os.path.join(build_root, "index.html")
|
index = os.path.join(build_root, "index.html")
|
||||||
index_template = env.get_template("index.html")
|
index_template = env.get_template("index.html")
|
||||||
page = index_template.render(best=data, today=today)
|
page = index_template.render(best=data, today=today)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue