Track upstream discovery state

debian_todo/__init__.py

@@ -11,6 +11,7 @@ import re
 import glob
 import os
 import shutil
+import sqlite3
 import subprocess
 import sys
 import urllib.error
@@ -34,8 +35,12 @@ PRERELEASE_RE = re.compile(r"(?i)(?:^|[0-9.\-])(?:alpha|beta|rc|a|b)\d*")
 CURRENTLY_RE = re.compile(
     r"^(?P<new>.+?)\s*\(currently in unstable:\s*(?P<current>.+?)\)\s*$"
 )
-CACHE_PATH = Path(".vcs_git_cache.json")
+CACHE_PATH = Path(".debian_todo.sqlite3")
 CACHE_VERSION = 4
+DISCOVERY_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+DISCOVERY_DISPLAY_CUTOFF = datetime.datetime(
+    2026, 5, 5, 12, 0, tzinfo=datetime.timezone.utc
+)
 SourceInfo = dict[str, str]
 HIDE_UPLOADER = "Edward Betts <edward@4angle.com>"
 DEBIAN_SRC_BASE = Path.home() / "src" / "debian"
@@ -192,75 +197,178 @@ def normalize_uploaders(uploaders: str) -> str:
     return "\n".join(cleaned)
 
 
-def load_cache(source_paths: list[str]) -> Optional[dict[str, SourceInfo]]:
-    """Load cached Vcs-Git and uploader info if still valid.
+def get_db_connection() -> sqlite3.Connection:
+    """Open the SQLite database and ensure the schema exists."""
+    conn = sqlite3.connect(CACHE_PATH)
+    conn.row_factory = sqlite3.Row
+    conn.executescript(
+        """
+        CREATE TABLE IF NOT EXISTS metadata (
+            key TEXT PRIMARY KEY,
+            value TEXT NOT NULL
+        );
 
-    Returns None if cache is missing, corrupted, or stale (based on
-    Sources file mtimes or cache version mismatch).
+        CREATE TABLE IF NOT EXISTS source_file_mtimes (
+            path TEXT PRIMARY KEY,
+            mtime REAL NOT NULL
+        );
+
+        CREATE TABLE IF NOT EXISTS source_info_cache (
+            source TEXT PRIMARY KEY,
+            team TEXT NOT NULL DEFAULT '',
+            uploaders TEXT NOT NULL DEFAULT '',
+            vcs_git TEXT
+        );
+
+        CREATE TABLE IF NOT EXISTS package_state (
+            source TEXT PRIMARY KEY,
+            shortname TEXT,
+            todo_type TEXT,
+            details TEXT,
+            current_new_version TEXT,
+            current_version TEXT,
+            is_upstream INTEGER NOT NULL DEFAULT 0,
+            is_prerelease INTEGER NOT NULL DEFAULT 0,
+            discovered_version TEXT,
+            discovered_at TEXT,
+            discovered_by_refresh INTEGER NOT NULL DEFAULT 0,
+            watch_content TEXT,
+            last_seen_at TEXT,
+            active INTEGER NOT NULL DEFAULT 1
+        );
+        """
+    )
+    package_state_columns = {
+        row["name"]
+        for row in conn.execute("PRAGMA table_info(package_state)").fetchall()
+    }
+    if "discovered_by_refresh" not in package_state_columns:
+        conn.execute(
+            """
+            ALTER TABLE package_state
+            ADD COLUMN discovered_by_refresh INTEGER NOT NULL DEFAULT 0
+            """
+        )
+    return conn
+
+
+def format_discovery_timestamp(
+    when: Optional[datetime.datetime] = None,
+) -> str:
+    """Return a UTC timestamp string for persisted discovery times."""
+    if when is None:
+        when = datetime.datetime.now(datetime.timezone.utc)
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=datetime.timezone.utc)
+    else:
+        when = when.astimezone(datetime.timezone.utc)
+    return when.replace(microsecond=0).strftime(DISCOVERY_TIMESTAMP_FORMAT)
+
+
+def parse_discovery_timestamp(timestamp: str) -> Optional[datetime.datetime]:
+    """Parse a stored discovery timestamp."""
+    try:
+        return datetime.datetime.strptime(
+            timestamp, DISCOVERY_TIMESTAMP_FORMAT
+        ).replace(tzinfo=datetime.timezone.utc)
+    except ValueError:
+        return None
+
+
+def load_cache(source_paths: list[str]) -> Optional[dict[str, SourceInfo]]:
+    """Load cached source info from SQLite if still valid.
+
+    Returns None if cache is missing, unavailable, or stale.
     """
     try:
-        with CACHE_PATH.open("r", encoding="utf-8") as handle:
-            data = json.load(handle)
-    except (FileNotFoundError, json.JSONDecodeError, OSError):
-        return None
-    if not isinstance(data, dict):
+        conn = get_db_connection()
+    except sqlite3.Error:
         return None
 
-    if data.get("cache_version") != CACHE_VERSION:
-        return None
-    cached_mtimes = data.get("sources_mtimes", {})
-    if not isinstance(cached_mtimes, dict):
-        return None
-    for path in source_paths:
-        try:
-            mtime = os.path.getmtime(path)
-        except OSError:
-            return None
-        if str(mtime) != str(cached_mtimes.get(path)):
+    try:
+        row = conn.execute(
+            "SELECT value FROM metadata WHERE key = 'source_cache_version'"
+        ).fetchone()
+        if row is None or row["value"] != str(CACHE_VERSION):
             return None
 
-    vcs_by_source = data.get("vcs_by_source")
-    if not isinstance(vcs_by_source, dict):
+        cached_mtimes = {
+            row["path"]: row["mtime"]
+            for row in conn.execute(
+                "SELECT path, mtime FROM source_file_mtimes"
+            ).fetchall()
+        }
+        for path in source_paths:
+            try:
+                mtime = os.path.getmtime(path)
+            except OSError:
+                return None
+            if str(mtime) != str(cached_mtimes.get(path)):
+                return None
+
+        normalized: dict[str, SourceInfo] = {}
+        for row in conn.execute(
+            "SELECT source, team, uploaders, vcs_git FROM source_info_cache"
+        ).fetchall():
+            normalized[row["source"]] = {
+                "team": row["team"],
+                "uploaders": row["uploaders"],
+                "vcs_git": row["vcs_git"],
+            }
+        return normalized
+    except sqlite3.Error:
         return None
-    normalized: dict[str, SourceInfo] = {}
-    for key, value in vcs_by_source.items():
-        if not isinstance(key, str):
-            return None
-        if isinstance(value, str):
-            normalized[key] = {"team": value, "uploaders": ""}
-            continue
-        if not isinstance(value, dict):
-            return None
-        team = value.get("team")
-        uploaders = value.get("uploaders")
-        vcs_git = value.get("vcs_git")
-        if not isinstance(team, str) or not isinstance(uploaders, str):
-            return None
-        normalized[key] = {"team": team, "uploaders": uploaders, "vcs_git": vcs_git}
-    return normalized
+    finally:
+        conn.close()
 
 
 def save_cache(source_paths: list[str], vcs_by_source: dict[str, SourceInfo]) -> None:
-    """Save Vcs-Git and uploader info to cache file.
-
-    Stores current mtimes of Sources files for cache invalidation.
-    """
+    """Save source info cache to SQLite."""
     sources_mtimes: dict[str, float] = {}
     for path in source_paths:
         try:
             sources_mtimes[path] = os.path.getmtime(path)
         except OSError:
-            return
-    data = {
-        "cache_version": CACHE_VERSION,
-        "sources_mtimes": sources_mtimes,
-        "vcs_by_source": vcs_by_source,
-    }
+            return None
     try:
-        with CACHE_PATH.open("w", encoding="utf-8") as handle:
-            json.dump(data, handle, sort_keys=True)
-    except OSError:
+        conn = get_db_connection()
+    except sqlite3.Error:
         return
+    try:
+        with conn:
+            conn.execute(
+                """
+                INSERT INTO metadata (key, value)
+                VALUES ('source_cache_version', ?)
+                ON CONFLICT(key) DO UPDATE SET value = excluded.value
+                """,
+                (str(CACHE_VERSION),),
+            )
+            conn.execute("DELETE FROM source_file_mtimes")
+            conn.executemany(
+                "INSERT INTO source_file_mtimes (path, mtime) VALUES (?, ?)",
+                sources_mtimes.items(),
+            )
+            conn.execute("DELETE FROM source_info_cache")
+            conn.executemany(
+                """
+                INSERT INTO source_info_cache (source, team, uploaders, vcs_git)
+                VALUES (?, ?, ?, ?)
+                """,
+                [
+                    (
+                        source,
+                        info.get("team", ""),
+                        info.get("uploaders", ""),
+                        info.get("vcs_git"),
+                    )
+                    for source, info in vcs_by_source.items()
+                ],
+            )
+    except sqlite3.Error:
+        return
+    finally:
+        conn.close()
 
 
 def load_source_info_map() -> dict[str, SourceInfo]:
@@ -356,6 +464,253 @@ def save_todo_list(todo_list: TodoList) -> None:
         handle.write("\n")
 
 
+def sync_package_state(
+    todo_list: TodoList,
+    seen_at: Optional[datetime.datetime] = None,
+    record_discovery: bool = True,
+) -> None:
+    """Sync package rows from todo.json into SQLite state."""
+    timestamp = format_discovery_timestamp(seen_at)
+    try:
+        conn = get_db_connection()
+    except sqlite3.Error:
+        return
+
+    present_sources: set[str] = set()
+    try:
+        with conn:
+            for item in todo_list:
+                source = item.get(":source")
+                if not isinstance(source, str):
+                    continue
+                present_sources.add(source)
+
+                shortname = item.get(":shortname") if isinstance(
+                    item.get(":shortname"), str
+                ) else None
+                todo_type = item.get(":type") if isinstance(item.get(":type"), str) else None
+                details = item.get(":details") if isinstance(item.get(":details"), str) else None
+                is_upstream = bool(shortname and shortname.startswith("newupstream_"))
+
+                new_version = None
+                current_version = None
+                is_prerelease = 0
+                if details is not None:
+                    new_version, current_version = parse_details(details)
+                    is_prerelease = int(is_prerelease_version(details))
+
+                existing = conn.execute(
+                    """
+                    SELECT
+                        discovered_version,
+                        discovered_at,
+                        discovered_by_refresh,
+                        watch_content
+                    FROM package_state WHERE source = ?
+                    """,
+                    (source,),
+                ).fetchone()
+                discovered_version = (
+                    existing["discovered_version"] if existing is not None else None
+                )
+                discovered_at = existing["discovered_at"] if existing is not None else None
+                discovered_by_refresh = (
+                    existing["discovered_by_refresh"] if existing is not None else 0
+                )
+                watch_content = existing["watch_content"] if existing is not None else None
+
+                if record_discovery and is_upstream and new_version:
+                    if discovered_version != new_version or not discovered_at:
+                        discovered_version = new_version
+                        discovered_at = timestamp
+                    discovered_by_refresh = 1
+
+                conn.execute(
+                    """
+                    INSERT INTO package_state (
+                        source,
+                        shortname,
+                        todo_type,
+                        details,
+                        current_new_version,
+                        current_version,
+                        is_upstream,
+                        is_prerelease,
+                        discovered_version,
+                        discovered_at,
+                        discovered_by_refresh,
+                        watch_content,
+                        last_seen_at,
+                        active
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
+                    ON CONFLICT(source) DO UPDATE SET
+                        shortname = excluded.shortname,
+                        todo_type = excluded.todo_type,
+                        details = excluded.details,
+                        current_new_version = excluded.current_new_version,
+                        current_version = excluded.current_version,
+                        is_upstream = excluded.is_upstream,
+                        is_prerelease = excluded.is_prerelease,
+                        discovered_version = excluded.discovered_version,
+                        discovered_at = excluded.discovered_at,
+                        discovered_by_refresh = excluded.discovered_by_refresh,
+                        watch_content = excluded.watch_content,
+                        last_seen_at = excluded.last_seen_at,
+                        active = excluded.active
+                    """,
+                    (
+                        source,
+                        shortname,
+                        todo_type,
+                        details,
+                        new_version,
+                        current_version,
+                        int(is_upstream),
+                        is_prerelease,
+                        discovered_version,
+                        discovered_at,
+                        discovered_by_refresh,
+                        watch_content,
+                        timestamp,
+                    ),
+                )
+
+            if present_sources:
+                placeholders = ",".join("?" for _ in present_sources)
+                conn.execute(
+                    f"UPDATE package_state SET active = 0 WHERE source NOT IN ({placeholders})",
+                    tuple(sorted(present_sources)),
+                )
+            else:
+                conn.execute("UPDATE package_state SET active = 0")
+    except sqlite3.Error:
+        return
+    finally:
+        conn.close()
+
+
+def load_package_discovery_map() -> dict[str, tuple[Optional[str], Optional[str], bool]]:
+    """Load per-package discovery data keyed by source package."""
+    try:
+        conn = get_db_connection()
+    except sqlite3.Error:
+        return {}
+    try:
+        return {
+            row["source"]: (
+                row["discovered_version"],
+                row["discovered_at"],
+                bool(row["discovered_by_refresh"]),
+            )
+            for row in conn.execute(
+                """
+                SELECT
+                    source,
+                    discovered_version,
+                    discovered_at,
+                    discovered_by_refresh
+                FROM package_state
+                """
+            ).fetchall()
+        }
+    except sqlite3.Error:
+        return {}
+    finally:
+        conn.close()
+
+
+def sort_todo_list_by_discovery(todo_list: TodoList) -> TodoList:
+    """Sort TODO items by first-seen upstream version date, oldest first."""
+    discovery_map = load_package_discovery_map()
+
+    def sort_key(item: TodoItem) -> tuple[int, str, str]:
+        source = item.get(":source")
+        details = item.get(":details")
+        if not isinstance(source, str) or not isinstance(details, str):
+            return (1, "", "")
+        discovered_version, discovered_at, discovered_by_refresh = discovery_map.get(
+            source, (None, None, False)
+        )
+        new_version, _ = parse_details(details)
+        parsed = (
+            parse_discovery_timestamp(discovered_at) if discovered_at is not None else None
+        )
+        if (
+            discovered_version == new_version
+            and discovered_at
+            and parsed is not None
+            and parsed >= DISCOVERY_DISPLAY_CUTOFF
+        ):
+            return (0, discovered_at, source)
+        return (1, "", source)
+
+    return sorted(todo_list, key=sort_key)
+
+
+def get_todo_discovery_timestamp(
+    todo: TodoItem,
+    discovery_map: dict[str, tuple[Optional[str], Optional[str], bool]],
+) -> Optional[str]:
+    """Return the recorded discovery timestamp for the todo's current version."""
+    source = todo.get(":source")
+    details = todo.get(":details")
+    if not isinstance(source, str) or not isinstance(details, str):
+        return None
+    discovered_version, discovered_at, discovered_by_refresh = discovery_map.get(
+        source, (None, None, False)
+    )
+    new_version, _ = parse_details(details)
+    parsed = parse_discovery_timestamp(discovered_at) if discovered_at else None
+    if (
+        discovered_version == new_version
+        and discovered_at
+        and parsed is not None
+        and parsed >= DISCOVERY_DISPLAY_CUTOFF
+    ):
+        return discovered_at
+    return None
+
+
+def format_display_discovery_timestamp(timestamp: Optional[str]) -> str:
+    """Format a stored discovery timestamp for list output."""
+    if not timestamp:
+        return "-"
+    parsed = parse_discovery_timestamp(timestamp)
+    if parsed is None:
+        return timestamp
+    return parsed.strftime("%Y-%m-%d %H:%M UTC")
+
+
+def save_watch_snapshot(package: str, repo_dir: Path) -> None:
+    """Persist a copy of debian/watch for a package when available."""
+    watch_path = repo_dir / "debian" / "watch"
+    watch_content = None
+    if watch_path.exists():
+        try:
+            watch_content = watch_path.read_text(encoding="utf-8")
+        except OSError:
+            watch_content = None
+
+    try:
+        conn = get_db_connection()
+    except sqlite3.Error:
+        return
+    try:
+        with conn:
+            conn.execute(
+                """
+                INSERT INTO package_state (source, watch_content, active)
+                VALUES (?, ?, 1)
+                ON CONFLICT(source) DO UPDATE SET watch_content = excluded.watch_content
+                """,
+                (package, watch_content),
+            )
+    except sqlite3.Error:
+        return
+    finally:
+        conn.close()
+
+
 def summarize_sources(todo_list: TodoList) -> set[str]:
     """Extract set of source package names from TODO list."""
     sources: set[str] = set()
@@ -489,10 +844,13 @@ def list_todos(include_prerelease: bool) -> None:
         with TODO_PATH.open("r", encoding="utf-8") as handle:
             todo_list = cast(TodoList, json.load(handle))
 
+    sync_package_state(todo_list, record_discovery=False)
     source_info_map = load_source_info_map()
     notes_by_source = load_notes()
+    discovery_map = load_package_discovery_map()
     console = Console()
     filtered = filter_todo_list(todo_list, include_prerelease=include_prerelease)
+    filtered = sort_todo_list_by_discovery(filtered)
     is_narrow = console.width < 100
     if is_narrow:
         console.print("Debian New Upstream TODOs")
@@ -501,11 +859,14 @@ def list_todos(include_prerelease: bool) -> None:
         table.add_column("Source", style="bold")
         table.add_column("New", style="green", justify="right")
         table.add_column("Current", style="dim", justify="right")
+        table.add_column("Seen", style="cyan")
         table.add_column("Team", justify="right")
         table.add_column("Note/Uploaders", overflow="fold")
 
     for todo in filtered:
         new_version, current_version = parse_details(todo[":details"])
+        discovered_at = get_todo_discovery_timestamp(todo, discovery_map)
+        display_discovered_at = format_display_discovery_timestamp(discovered_at)
         source_info = source_info_map.get(todo[":source"], {})
         team = source_info.get("team")
         uploaders = source_info.get("uploaders", "")
@@ -527,6 +888,7 @@ def list_todos(include_prerelease: bool) -> None:
                 parts.append(f"[dim]{display_team}[/dim]")
             parts.append(f"N: {display_new}")
             parts.append(f"C: {current_version or '-'}")
+            parts.append(f"D: {display_discovered_at}")
             console.print("  ".join(parts))
             if display_note != "-":
                 console.print("  " + display_note)
@@ -535,6 +897,7 @@ def list_todos(include_prerelease: bool) -> None:
                 source,
                 display_new,
                 current_version or "-",
+                display_discovered_at,
                 display_team,
                 display_note,
             )
@@ -558,6 +921,7 @@ def refresh_todos() -> None:
         sys.exit(1)
 
     save_todo_list(todo_list)
+    sync_package_state(todo_list, record_discovery=True)
     print_changes(filter_todo_list(old_list), filter_todo_list(todo_list))
 
 
@@ -1117,6 +1481,8 @@ def run_package_updates(repo_dir: Path) -> None:
         ExternalCommandError: If any update command fails
     """
     print("Updating package files...")
+    subprocess.run(["update-debian-package"], check=True, text=True, cwd=repo_dir)
+    return
     update_debian_control(repo_dir)
     update_debian_copyright_year(repo_dir)
     update_debian_watch(repo_dir)
@@ -1181,9 +1547,12 @@ def update_package(package: str) -> None:
 
     # Run package updates
     run_package_updates(repo_dir)
+    save_watch_snapshot(package, repo_dir)
 
     run_command(["debuild", "-S"], repo_dir, "debuild -S")
-    os.remove(repo_dir / "debian/files")
+    debian_files = repo_dir / "debian/files"
+    if debian_files.exists():
+        os.remove(debian_files)
 
     print(f"Successfully updated {package}")
     print(pkg_dir)