From 427a4913807621d195074572de93f068a26434a2 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 16:02:46 +0200 Subject: [PATCH 1/9] M1-T01: add app-chain revision creating location + poo_records tables Add alembic_app revision 20260611_06_merge_location_poo_tables that builds empty location and poo_records tables (REAL float columns, matching the production schema and the adopt scripts' EXPECTED_*_TABLE_INFO constants). Update APP_BASELINE_REVISION to the new head. Schema-only; data migration is handled separately by scripts/migrate_legacy_data.py (M1-T02). --- .../20260611_06_merge_location_poo_tables.py | 43 +++++++++++++++++++ scripts/app_db_adopt.py | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 alembic_app/versions/20260611_06_merge_location_poo_tables.py diff --git a/alembic_app/versions/20260611_06_merge_location_poo_tables.py b/alembic_app/versions/20260611_06_merge_location_poo_tables.py new file mode 100644 index 0000000..a0d7997 --- /dev/null +++ b/alembic_app/versions/20260611_06_merge_location_poo_tables.py @@ -0,0 +1,43 @@ +"""merge location and poo_records tables into app chain + +Revision ID: 20260611_06_merge_location_poo_tables +Revises: 20260429_05_public_ip_monitor +Create Date: 2026-06-11 00:00:01.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260611_06_merge_location_poo_tables" +down_revision: Union[str, None] = "20260429_05_public_ip_monitor" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "location", + sa.Column("person", sa.Text(), nullable=False), + sa.Column("datetime", sa.Text(), nullable=False), + sa.Column("latitude", sa.REAL(), nullable=False), + sa.Column("longitude", sa.REAL(), nullable=False), + sa.Column("altitude", sa.REAL(), nullable=True), + sa.PrimaryKeyConstraint("person", "datetime"), + ) + + op.create_table( + "poo_records", + sa.Column("timestamp", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("latitude", sa.REAL(), nullable=False), + sa.Column("longitude", sa.REAL(), nullable=False), + sa.PrimaryKeyConstraint("timestamp"), + ) + + +def downgrade() -> None: + op.drop_table("poo_records") + op.drop_table("location") diff --git a/scripts/app_db_adopt.py b/scripts/app_db_adopt.py index 4a1b30d..acd0074 100644 --- a/scripts/app_db_adopt.py +++ b/scripts/app_db_adopt.py @@ -15,7 +15,7 @@ if str(PROJECT_ROOT) not in sys.path: from app.config import get_settings -APP_BASELINE_REVISION = "20260429_05_public_ip_monitor" +APP_BASELINE_REVISION = "20260611_06_merge_location_poo_tables" class AppDatabaseAdoptionError(RuntimeError): From bc8dd062d5a937b6a658744c70edfb99053646ae Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 16:13:55 +0200 Subject: [PATCH 2/9] M1-T02: add idempotent legacy data migration script scripts/migrate_legacy_data.py copies rows from the legacy locationRecorder.db / pooRecorder.db into the unified app DB's location / poo_records tables using ATTACH + INSERT OR IGNORE (idempotent via PK-conflict skip; explicit columns, never SELECT *). After copy it reconciles every source row against the target and raises / exits non-zero on any shortfall. Missing legacy files are a safe no-op (skipped); --dry-run writes nothing. Not part of the Alembic chain; run manually once at cut-over. Never deletes or overwrites any file. Validated end-to-end on copies of the real production DBs: dry-run reported 75103 location + 874 poo rows and wrote nothing; the real run copied all rows with reconciliation passing; a second run copied 0 (idempotent). --- scripts/migrate_legacy_data.py | 267 ++++++++++++++++++++ tests/test_migrate_legacy_data.py | 403 ++++++++++++++++++++++++++++++ 2 files changed, 670 insertions(+) create mode 100644 scripts/migrate_legacy_data.py create mode 100644 tests/test_migrate_legacy_data.py diff --git a/scripts/migrate_legacy_data.py b/scripts/migrate_legacy_data.py new file mode 100644 index 0000000..d36bfae --- /dev/null +++ b/scripts/migrate_legacy_data.py @@ -0,0 +1,267 @@ +"""One-time idempotent data migration: copy rows from legacy locationRecorder.db / +pooRecorder.db into the unified app DB's location / poo_records tables. + +NOT part of the Alembic chain. Run manually, once, during production cut-over: + + python -m scripts.migrate_legacy_data \\ + --app-db sqlite:///./data/app.db \\ + --location-db sqlite:///./data/locationRecorder.db \\ + --poo-db sqlite:///./data/pooRecorder.db + +Or rely on environment variables: + APP_DATABASE_URL, LOCATION_DATABASE_URL, POO_DATABASE_URL + +Add --dry-run to preview row counts without writing anything. + +Return value of migrate_legacy_data(): a dict shaped like: + { + "location": {"source": N, "copied": C, "skipped": bool, "final": F}, + "poo_records": {"source": N, "copied": C, "skipped": bool, "final": F}, + } +where: + source - rows in the legacy DB (0 when skipped) + copied - rows inserted by this run (0 when dry_run or skipped) + skipped - True when the legacy file was absent + final - rows present in the app table after the run (0 when dry_run) +""" + +from __future__ import annotations + +import argparse +import os +import sqlite3 +import sys +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _sqlite_path_from_url(url: str) -> Path: + """Extract the filesystem path from a sqlite:///... URL. + + If *url* does not start with 'sqlite:///', it is treated as a plain path. + """ + prefix = "sqlite:///" + if url.startswith(prefix): + return Path(url[len(prefix):]) + return Path(url) + + +def _reconcile( + conn: sqlite3.Connection, + table: str, + pk_cols: list[str], + source_count: int, +) -> int: + """Verify every legacy source row is present in the main (app) table. + + Returns the count of source rows present in main. + Raises RuntimeError if any rows are missing. + """ + join_cond = " AND ".join( + f"m.{col} = l.{col}" for col in pk_cols + ) + sql = ( + f"SELECT COUNT(*) FROM legacy.{table} l " + f"WHERE EXISTS (SELECT 1 FROM main.{table} m WHERE {join_cond})" + ) + (present,) = conn.execute(sql).fetchone() + if present < source_count: + missing = source_count - present + raise RuntimeError( + f"Reconciliation failed for table '{table}': " + f"{missing} of {source_count} source rows are missing from the app DB." + ) + return present + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def migrate_legacy_data( + app_url: str, + location_url: str | None, + poo_url: str | None, + *, + dry_run: bool = False, +) -> dict: + """Copy rows from legacy DBs into the app DB's location / poo_records tables. + + Parameters + ---------- + app_url: sqlite:///... URL (or plain path) for the unified app DB. + location_url: sqlite:///... URL (or plain path) for the legacy location DB, + or None to skip that table. + poo_url: sqlite:///... URL (or plain path) for the legacy poo DB, + or None to skip that table. + dry_run: When True, gather counts only; perform no writes. + + Returns a dict with per-table stats (see module docstring). + Raises RuntimeError on reconciliation failure (non-zero rows missing). + """ + app_path = _sqlite_path_from_url(app_url) + + results: dict[str, dict] = {} + + # --- location table --- + results["location"] = _migrate_table( + app_path=app_path, + legacy_url=location_url, + table="location", + columns=["person", "datetime", "latitude", "longitude", "altitude"], + pk_cols=["person", "datetime"], + dry_run=dry_run, + ) + + # --- poo_records table --- + results["poo_records"] = _migrate_table( + app_path=app_path, + legacy_url=poo_url, + table="poo_records", + columns=["timestamp", "status", "latitude", "longitude"], + pk_cols=["timestamp"], + dry_run=dry_run, + ) + + return results + + +def _migrate_table( + *, + app_path: Path, + legacy_url: str | None, + table: str, + columns: list[str], + pk_cols: list[str], + dry_run: bool, +) -> dict: + """Migrate a single table from a legacy DB into the app DB. + + Returns a per-table stats dict. + """ + # If the caller passed None → treat as absent + if legacy_url is None: + return {"source": 0, "copied": 0, "skipped": True, "final": 0} + + legacy_path = _sqlite_path_from_url(legacy_url) + + # If the file doesn't exist → safe no-op + if not legacy_path.exists(): + return {"source": 0, "copied": 0, "skipped": True, "final": 0} + + col_list = ", ".join(columns) + + conn = sqlite3.connect(app_path) + try: + conn.execute("ATTACH DATABASE ? AS legacy", (str(legacy_path),)) + + # Count source rows + (source_count,) = conn.execute(f"SELECT COUNT(*) FROM legacy.{table}").fetchone() + + if dry_run: + conn.execute("DETACH DATABASE legacy") + return { + "source": source_count, + "copied": 0, + "skipped": False, + "final": 0, + } + + # Count rows already in the target before this run + (before_count,) = conn.execute(f"SELECT COUNT(*) FROM main.{table}").fetchone() + + # Idempotent insert — PK conflict → skip + conn.execute( + f"INSERT OR IGNORE INTO main.{table} ({col_list}) " + f"SELECT {col_list} FROM legacy.{table}" + ) + conn.commit() + + # Count rows now + (after_count,) = conn.execute(f"SELECT COUNT(*) FROM main.{table}").fetchone() + copied = after_count - before_count + + # Reconciliation: every source row must be present + _reconcile(conn, table, pk_cols, source_count) + + conn.execute("DETACH DATABASE legacy") + finally: + conn.close() + + return { + "source": source_count, + "copied": copied, + "skipped": False, + "final": after_count, + } + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Migrate legacy location/poo data into the unified app DB." + ) + parser.add_argument( + "--app-db", + default=os.environ.get("APP_DATABASE_URL"), + help="sqlite:///... URL or path for the app DB " + "(default: $APP_DATABASE_URL)", + ) + parser.add_argument( + "--location-db", + default=os.environ.get("LOCATION_DATABASE_URL"), + help="sqlite:///... URL or path for the legacy location DB " + "(default: $LOCATION_DATABASE_URL). Omit to skip location table.", + ) + parser.add_argument( + "--poo-db", + default=os.environ.get("POO_DATABASE_URL"), + help="sqlite:///... URL or path for the legacy poo DB " + "(default: $POO_DATABASE_URL). Omit to skip poo_records table.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Report counts only; do not write any rows.", + ) + args = parser.parse_args() + + if not args.app_db: + parser.error( + "App DB not specified. Pass --app-db or set APP_DATABASE_URL." + ) + + try: + results = migrate_legacy_data( + app_url=args.app_db, + location_url=args.location_db, + poo_url=args.poo_db, + dry_run=args.dry_run, + ) + except RuntimeError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + + prefix = "[DRY RUN] " if args.dry_run else "" + print(f"{prefix}Migration results:") + for table_name, stats in results.items(): + if stats["skipped"]: + print(f" {table_name}: SKIPPED (legacy file absent or not provided)") + else: + print( + f" {table_name}: source={stats['source']}, " + f"copied={stats['copied']}, final={stats['final']}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_migrate_legacy_data.py b/tests/test_migrate_legacy_data.py new file mode 100644 index 0000000..bd6fa47 --- /dev/null +++ b/tests/test_migrate_legacy_data.py @@ -0,0 +1,403 @@ +"""Tests for scripts/migrate_legacy_data.py (M1-T02). + +Uses pytest tmp_path for all temp files. The app DB is brought to head via +alembic_app.ini (the same approach used by conftest._make_app_alembic_config), +so it has the location and poo_records tables created in T01. + +Legacy DBs are built by hand with real columns matching the legacy baseline schema. +""" + +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path + +import pytest +from alembic import command +from alembic.config import Config + +from scripts.migrate_legacy_data import ( + _reconcile, + _sqlite_path_from_url, + migrate_legacy_data, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_app_alembic_config(database_url: str) -> Config: + cfg = Config("alembic_app.ini") + cfg.set_main_option("sqlalchemy.url", database_url) + return cfg + + +def _upgraded_app_db(tmp_path: Path, name: str = "app_test.db") -> tuple[Path, str]: + """Create and upgrade an app DB to head; return (path, url).""" + db_path = tmp_path / name + db_url = f"sqlite:///{db_path}" + command.upgrade(_make_app_alembic_config(db_url), "head") + return db_path, db_url + + +def _make_legacy_location_db(db_path: Path, rows: list[tuple]) -> None: + """Create a legacy location DB and insert given rows. + + Each row is a tuple: (person, datetime, latitude, longitude, altitude). + altitude may be None. + """ + conn = sqlite3.connect(db_path) + conn.execute( + """ + CREATE TABLE location ( + person TEXT NOT NULL, + datetime TEXT NOT NULL, + latitude REAL NOT NULL, + longitude REAL NOT NULL, + altitude REAL, + PRIMARY KEY (person, datetime) + ) + """ + ) + conn.executemany( + "INSERT INTO location (person, datetime, latitude, longitude, altitude) " + "VALUES (?, ?, ?, ?, ?)", + rows, + ) + conn.commit() + conn.close() + + +def _make_legacy_poo_db(db_path: Path, rows: list[tuple]) -> None: + """Create a legacy poo DB and insert given rows. + + Each row is a tuple: (timestamp, status, latitude, longitude). + """ + conn = sqlite3.connect(db_path) + conn.execute( + """ + CREATE TABLE poo_records ( + timestamp TEXT NOT NULL, + status TEXT NOT NULL, + latitude REAL NOT NULL, + longitude REAL NOT NULL, + PRIMARY KEY (timestamp) + ) + """ + ) + conn.executemany( + "INSERT INTO poo_records (timestamp, status, latitude, longitude) " + "VALUES (?, ?, ?, ?)", + rows, + ) + conn.commit() + conn.close() + + +def _count_rows(db_path: Path, table: str) -> int: + conn = sqlite3.connect(db_path) + try: + (count,) = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone() + return count + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# Sample data +# --------------------------------------------------------------------------- + +LOCATION_ROWS = [ + ("alice", "2026-01-01T10:00:00Z", 1.23, 4.56, 7.89), + ("bob", "2026-01-02T10:00:00Z", 2.34, 5.67, None), + ("alice", "2026-01-03T10:00:00Z", 3.45, 6.78, 9.01), +] + +POO_ROWS = [ + ("2026-01-01T08:00:00Z", "complete", 10.0, 20.0), + ("2026-01-02T08:00:00Z", "urgent", 11.0, 21.0), +] + + +# --------------------------------------------------------------------------- +# Test 1: Idempotency +# --------------------------------------------------------------------------- + + +def test_location_migration_is_idempotent(tmp_path: Path) -> None: + """N source rows → app table has N rows; run again → still N rows.""" + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(legacy_path, LOCATION_ROWS) + legacy_url = f"sqlite:///{legacy_path}" + + # First run + result1 = migrate_legacy_data(app_url, legacy_url, None) + assert result1["location"]["source"] == len(LOCATION_ROWS) + assert result1["location"]["copied"] == len(LOCATION_ROWS) + assert result1["location"]["skipped"] is False + assert result1["location"]["final"] == len(LOCATION_ROWS) + assert _count_rows(app_path, "location") == len(LOCATION_ROWS) + + # Second run — idempotent, no dupes, no error + result2 = migrate_legacy_data(app_url, legacy_url, None) + assert result2["location"]["source"] == len(LOCATION_ROWS) + assert result2["location"]["copied"] == 0 # nothing new + assert result2["location"]["skipped"] is False + assert result2["location"]["final"] == len(LOCATION_ROWS) + assert _count_rows(app_path, "location") == len(LOCATION_ROWS) + + +def test_poo_migration_is_idempotent(tmp_path: Path) -> None: + """N poo source rows → app table has N rows; run again → still N rows.""" + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "pooRecorder.db" + _make_legacy_poo_db(legacy_path, POO_ROWS) + legacy_url = f"sqlite:///{legacy_path}" + + result1 = migrate_legacy_data(app_url, None, legacy_url) + assert result1["poo_records"]["source"] == len(POO_ROWS) + assert result1["poo_records"]["copied"] == len(POO_ROWS) + assert result1["poo_records"]["skipped"] is False + assert result1["poo_records"]["final"] == len(POO_ROWS) + assert _count_rows(app_path, "poo_records") == len(POO_ROWS) + + result2 = migrate_legacy_data(app_url, None, legacy_url) + assert result2["poo_records"]["copied"] == 0 + assert result2["poo_records"]["final"] == len(POO_ROWS) + assert _count_rows(app_path, "poo_records") == len(POO_ROWS) + + +def test_both_tables_migration_is_idempotent(tmp_path: Path) -> None: + """Migrating both tables at once is idempotent.""" + app_path, app_url = _upgraded_app_db(tmp_path) + + loc_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(loc_path, LOCATION_ROWS) + loc_url = f"sqlite:///{loc_path}" + + poo_path = tmp_path / "pooRecorder.db" + _make_legacy_poo_db(poo_path, POO_ROWS) + poo_url = f"sqlite:///{poo_path}" + + result1 = migrate_legacy_data(app_url, loc_url, poo_url) + assert result1["location"]["final"] == len(LOCATION_ROWS) + assert result1["poo_records"]["final"] == len(POO_ROWS) + + result2 = migrate_legacy_data(app_url, loc_url, poo_url) + assert result2["location"]["copied"] == 0 + assert result2["poo_records"]["copied"] == 0 + assert _count_rows(app_path, "location") == len(LOCATION_ROWS) + assert _count_rows(app_path, "poo_records") == len(POO_ROWS) + + +# --------------------------------------------------------------------------- +# Test 2: Missing legacy file +# --------------------------------------------------------------------------- + + +def test_missing_location_file_is_skipped(tmp_path: Path) -> None: + """Absent location DB → table result is skipped, no exception, app table empty.""" + app_path, app_url = _upgraded_app_db(tmp_path) + nonexistent = f"sqlite:///{tmp_path / 'does_not_exist_location.db'}" + + result = migrate_legacy_data(app_url, nonexistent, None) + + assert result["location"]["skipped"] is True + assert result["location"]["source"] == 0 + assert result["location"]["copied"] == 0 + assert _count_rows(app_path, "location") == 0 + + +def test_missing_poo_file_is_skipped(tmp_path: Path) -> None: + """Absent poo DB → table result is skipped, no exception, app table empty.""" + app_path, app_url = _upgraded_app_db(tmp_path) + nonexistent = f"sqlite:///{tmp_path / 'does_not_exist_poo.db'}" + + result = migrate_legacy_data(app_url, None, nonexistent) + + assert result["poo_records"]["skipped"] is True + assert result["poo_records"]["source"] == 0 + assert result["poo_records"]["copied"] == 0 + assert _count_rows(app_path, "poo_records") == 0 + + +def test_none_location_url_is_skipped(tmp_path: Path) -> None: + """Passing None for location_url → skipped, no exception.""" + _, app_url = _upgraded_app_db(tmp_path) + + result = migrate_legacy_data(app_url, None, None) + + assert result["location"]["skipped"] is True + assert result["poo_records"]["skipped"] is True + + +# --------------------------------------------------------------------------- +# Test 3: Reconciliation failure +# --------------------------------------------------------------------------- + + +def test_reconcile_raises_on_missing_rows(tmp_path: Path) -> None: + """_reconcile() raises RuntimeError when source rows are missing from target.""" + # Build an app DB and a legacy DB with 3 rows + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(legacy_path, LOCATION_ROWS) + + # Only insert 1 row into the app DB manually (simulate partial migration) + conn = sqlite3.connect(app_path) + conn.execute( + "INSERT INTO location (person, datetime, latitude, longitude, altitude) " + "VALUES (?, ?, ?, ?, ?)", + LOCATION_ROWS[0], + ) + conn.commit() + # ATTACH legacy to run _reconcile + conn.execute(f"ATTACH DATABASE '{legacy_path}' AS legacy") + with pytest.raises(RuntimeError, match="Reconciliation failed"): + _reconcile( + conn, + table="location", + pk_cols=["person", "datetime"], + source_count=len(LOCATION_ROWS), + ) + conn.execute("DETACH DATABASE legacy") + conn.close() + + +def test_migrate_reconciliation_failure_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """If a row goes missing after INSERT, migrate_legacy_data raises RuntimeError.""" + import scripts.migrate_legacy_data as mod + + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(legacy_path, LOCATION_ROWS) + legacy_url = f"sqlite:///{legacy_path}" + + def _always_fail(conn, table, pk_cols, source_count): + # Simulate a scenario where reconciliation finds rows missing + raise RuntimeError( + f"Reconciliation failed for table '{table}': " + f"1 of {source_count} source rows are missing from the app DB." + ) + + monkeypatch.setattr(mod, "_reconcile", _always_fail) + + with pytest.raises(RuntimeError, match="Reconciliation failed"): + migrate_legacy_data(app_url, legacy_url, None) + + +def test_cli_exits_nonzero_on_reconciliation_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """CLI main() exits non-zero when reconciliation raises.""" + import scripts.migrate_legacy_data as mod + + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(legacy_path, LOCATION_ROWS) + legacy_url = f"sqlite:///{legacy_path}" + + # Patch _reconcile to always raise + def _always_fail(conn, table, pk_cols, source_count): + raise RuntimeError( + f"Reconciliation failed for table '{table}': 1 row missing." + ) + + monkeypatch.setattr(mod, "_reconcile", _always_fail) + + # Patch sys.argv so main() picks up the right args + monkeypatch.setattr( + sys, + "argv", + [ + "migrate_legacy_data", + "--app-db", app_url, + "--location-db", legacy_url, + ], + ) + + with pytest.raises(SystemExit) as exc_info: + mod.main() + + assert exc_info.value.code != 0 + + +# --------------------------------------------------------------------------- +# Test 4: dry_run +# --------------------------------------------------------------------------- + + +def test_dry_run_does_not_write_location_rows(tmp_path: Path) -> None: + """dry_run=True reports source counts but writes nothing.""" + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(legacy_path, LOCATION_ROWS) + legacy_url = f"sqlite:///{legacy_path}" + + result = migrate_legacy_data(app_url, legacy_url, None, dry_run=True) + + assert result["location"]["source"] == len(LOCATION_ROWS) + assert result["location"]["copied"] == 0 + assert result["location"]["skipped"] is False + # dry_run returns final=0 (no actual query on app side) + assert result["location"]["final"] == 0 + # App table must still be empty + assert _count_rows(app_path, "location") == 0 + + +def test_dry_run_does_not_write_poo_rows(tmp_path: Path) -> None: + """dry_run=True for poo_records: source reported, nothing written.""" + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "pooRecorder.db" + _make_legacy_poo_db(legacy_path, POO_ROWS) + legacy_url = f"sqlite:///{legacy_path}" + + result = migrate_legacy_data(app_url, None, legacy_url, dry_run=True) + + assert result["poo_records"]["source"] == len(POO_ROWS) + assert result["poo_records"]["copied"] == 0 + assert result["poo_records"]["skipped"] is False + assert result["poo_records"]["final"] == 0 + assert _count_rows(app_path, "poo_records") == 0 + + +def test_dry_run_both_tables(tmp_path: Path) -> None: + """dry_run=True for both tables: both reported, nothing written.""" + app_path, app_url = _upgraded_app_db(tmp_path) + + loc_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(loc_path, LOCATION_ROWS) + loc_url = f"sqlite:///{loc_path}" + + poo_path = tmp_path / "pooRecorder.db" + _make_legacy_poo_db(poo_path, POO_ROWS) + poo_url = f"sqlite:///{poo_path}" + + result = migrate_legacy_data(app_url, loc_url, poo_url, dry_run=True) + + assert result["location"]["source"] == len(LOCATION_ROWS) + assert result["location"]["copied"] == 0 + assert result["poo_records"]["source"] == len(POO_ROWS) + assert result["poo_records"]["copied"] == 0 + assert _count_rows(app_path, "location") == 0 + assert _count_rows(app_path, "poo_records") == 0 + + +# --------------------------------------------------------------------------- +# Test: _sqlite_path_from_url helper +# --------------------------------------------------------------------------- + + +def test_sqlite_path_from_url_parses_url() -> None: + path = _sqlite_path_from_url("sqlite:///./data/app.db") + # Path normalises './' away, but the tail should remain + assert path == Path("data/app.db") + + +def test_sqlite_path_from_url_treats_plain_path_as_path() -> None: + path = _sqlite_path_from_url("/tmp/some.db") + assert str(path) == "/tmp/some.db" From 3d3c2bcc5744ac00d758f6c26278ace0b39c682c Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 16:35:07 +0200 Subject: [PATCH 3/9] M1-T03: unify data layer, models, deps and routes onto single app DB Collapse the three data layers into one. app/db.py now exposes a single Base, a cached engine bound to app_database_url with SQLite WAL enabled, and get_engine/get_session_local/reset_db_caches/get_db_session. Delete app/auth_db.py, app/poo_db.py and app/models/base.py. All models (auth, config, public_ip, location, poo) inherit the one Base and register on a single metadata. Dependencies converge to a single get_db; all routes use it. Also update the alembic env.py files (app/location/poo) and tests that imported the removed modules so the suite stays green, and drop the obsolete test_legacy_style_location_db test whose flow (app reading a separate location DB) no longer exists. Location/poo Alembic chains, adopt scripts and adoption tests remain for M1-T04; config fields remain for M1-T05. pytest 109 passed; ruff clean (pre-existing only); WAL verified; single Base.metadata holds all seven tables. --- alembic_app/env.py | 6 ++- alembic_location/env.py | 2 +- alembic_poo/env.py | 4 +- app/api/routes/auth.py | 8 ++-- app/api/routes/homeassistant.py | 4 +- app/api/routes/pages.py | 8 ++-- app/api/routes/poo.py | 6 +-- app/api/routes/public_ip.py | 4 +- app/api/routes/ticktick.py | 4 +- app/auth_db.py | 53 ---------------------- app/db.py | 49 ++++++++++++++++---- app/dependencies.py | 16 ++----- app/main.py | 6 +-- app/models/__init__.py | 2 + app/models/auth.py | 6 +-- app/models/base.py | 4 -- app/models/config.py | 4 +- app/models/poo.py | 4 +- app/models/public_ip.py | 6 +-- app/poo_db.py | 28 ------------ app/services/config_page.py | 8 ++-- tests/conftest.py | 50 +++++---------------- tests/test_app.py | 26 +++++------ tests/test_auth.py | 4 +- tests/test_deployment.py | 10 ++--- tests/test_homeassistant_inbound.py | 69 +++++------------------------ tests/test_location.py | 66 +-------------------------- tests/test_ticktick.py | 12 ++--- 28 files changed, 134 insertions(+), 335 deletions(-) delete mode 100644 app/auth_db.py delete mode 100644 app/models/base.py delete mode 100644 app/poo_db.py diff --git a/alembic_app/env.py b/alembic_app/env.py index 00d6bea..f198417 100644 --- a/alembic_app/env.py +++ b/alembic_app/env.py @@ -3,11 +3,13 @@ from logging.config import fileConfig from alembic import context from sqlalchemy import engine_from_config, pool -from app.auth_db import AuthBase from app.config import get_settings +from app.db import Base from app.models.config import AppConfigEntry # noqa: F401 from app.models.auth import AuthSession, AuthUser # noqa: F401 from app.models.public_ip import PublicIPHistory, PublicIPState # noqa: F401 +from app.models.location import Location # noqa: F401 +from app.models.poo import PooRecord # noqa: F401 config = context.config @@ -19,7 +21,7 @@ configured_url = config.get_main_option("sqlalchemy.url") if not configured_url or configured_url == "sqlite:///./data/app.db": config.set_main_option("sqlalchemy.url", settings.app_database_url) -target_metadata = AuthBase.metadata +target_metadata = Base.metadata def run_migrations_offline() -> None: diff --git a/alembic_location/env.py b/alembic_location/env.py index 5b2d901..05b9217 100644 --- a/alembic_location/env.py +++ b/alembic_location/env.py @@ -5,7 +5,7 @@ from sqlalchemy import engine_from_config, pool from app.config import get_settings from app.models import Location # noqa: F401 -from app.models.base import Base +from app.db import Base config = context.config diff --git a/alembic_poo/env.py b/alembic_poo/env.py index 44cb0b9..98b528f 100644 --- a/alembic_poo/env.py +++ b/alembic_poo/env.py @@ -5,7 +5,7 @@ from sqlalchemy import engine_from_config, pool from app.config import get_settings from app.models.poo import PooRecord # noqa: F401 -from app.poo_db import PooBase +from app.db import Base config = context.config @@ -17,7 +17,7 @@ configured_url = config.get_main_option("sqlalchemy.url") if not configured_url or configured_url == "sqlite:///./data/pooRecorder.db": config.set_main_option("sqlalchemy.url", settings.poo_database_url) -target_metadata = PooBase.metadata +target_metadata = Base.metadata def run_migrations_offline() -> None: diff --git a/app/api/routes/auth.py b/app/api/routes/auth.py index d9603e2..d80846f 100644 --- a/app/api/routes/auth.py +++ b/app/api/routes/auth.py @@ -7,7 +7,7 @@ from fastapi.templating import Jinja2Templates from sqlalchemy.orm import Session from app.config import Settings -from app.dependencies import get_app_settings, get_auth_db, get_current_auth_session +from app.dependencies import get_app_settings, get_db, get_current_auth_session from app.services.auth import ( AuthenticatedSession, authenticate_user, @@ -57,7 +57,7 @@ def login_submit( username: str = Form(), password: str = Form(), csrf_token: str = Form(), - session: Session = Depends(get_auth_db), + session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), ) -> Response: cookie_csrf_token = request.cookies.get(LOGIN_CSRF_COOKIE_NAME) @@ -102,7 +102,7 @@ def change_password_submit( new_password: str = Form(), confirm_password: str = Form(), csrf_token: str = Form(), - session: Session = Depends(get_auth_db), + session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), current_auth: AuthenticatedSession | None = Depends(get_current_auth_session), ) -> Response: @@ -151,7 +151,7 @@ def change_password_submit( def logout( request: Request, csrf_token: str = Form(), - session: Session = Depends(get_auth_db), + session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), current_auth: AuthenticatedSession | None = Depends(get_current_auth_session), ) -> RedirectResponse: diff --git a/app/api/routes/homeassistant.py b/app/api/routes/homeassistant.py index ccee1f8..703df37 100644 --- a/app/api/routes/homeassistant.py +++ b/app/api/routes/homeassistant.py @@ -11,7 +11,6 @@ from app.dependencies import ( get_app_settings, get_db, get_homeassistant_client, - get_poo_db, get_ticktick_client, ) from app.integrations.homeassistant import ( @@ -36,7 +35,6 @@ INTERNAL_SERVER_ERROR_MESSAGE = "internal server error" async def publish_from_homeassistant( request: Request, db: Session = Depends(get_db), - poo_db: Session = Depends(get_poo_db), settings: Settings = Depends(get_app_settings), homeassistant_client: HomeAssistantClient = Depends(get_homeassistant_client), ticktick_client: TickTickClient = Depends(get_ticktick_client), @@ -49,7 +47,7 @@ async def publish_from_homeassistant( db, envelope, ticktick_client=ticktick_client, - poo_session=poo_db, + poo_session=db, settings=settings, homeassistant_client=homeassistant_client, ) diff --git a/app/api/routes/pages.py b/app/api/routes/pages.py index 4b474cb..bbd2594 100644 --- a/app/api/routes/pages.py +++ b/app/api/routes/pages.py @@ -6,7 +6,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse, Response from fastapi.templating import Jinja2Templates from app.config import Settings, get_settings -from app.dependencies import get_app_settings, get_auth_db, get_current_auth_session +from app.dependencies import get_app_settings, get_db, get_current_auth_session from app.services.auth import AuthenticatedSession from app.services.config_page import ( ConfigSaveError, @@ -100,7 +100,7 @@ def admin_redirect( @router.get("/config", response_class=HTMLResponse) def config_page( request: Request, - auth_db_session: Session = Depends(get_auth_db), + auth_db_session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), current_auth: AuthenticatedSession | None = Depends(get_current_auth_session), ) -> Response: @@ -129,7 +129,7 @@ def config_page( @router.post("/config", response_class=HTMLResponse) async def config_submit( request: Request, - auth_db_session: Session = Depends(get_auth_db), + auth_db_session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), current_auth: AuthenticatedSession | None = Depends(get_current_auth_session), ) -> Response: @@ -189,7 +189,7 @@ async def config_submit( @router.post("/config/smtp/test", response_class=HTMLResponse) async def smtp_test_submit( request: Request, - auth_db_session: Session = Depends(get_auth_db), + auth_db_session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), current_auth: AuthenticatedSession | None = Depends(get_current_auth_session), ) -> Response: diff --git a/app/api/routes/poo.py b/app/api/routes/poo.py index 451741d..4473e30 100644 --- a/app/api/routes/poo.py +++ b/app/api/routes/poo.py @@ -7,7 +7,7 @@ from pydantic import ValidationError from sqlalchemy.orm import Session from app.config import Settings -from app.dependencies import get_app_settings, get_homeassistant_client, get_poo_db +from app.dependencies import get_app_settings, get_homeassistant_client, get_db from app.integrations.homeassistant import HomeAssistantClient from app.schemas.poo import PooRecordRequest from app.services.poo import publish_latest_poo_status, record_poo @@ -21,7 +21,7 @@ INTERNAL_SERVER_ERROR_MESSAGE = "internal server error" @router.post("/poo/record") async def create_poo_record( request: Request, - db: Session = Depends(get_poo_db), + db: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), homeassistant_client: HomeAssistantClient = Depends(get_homeassistant_client), ) -> Response: @@ -56,7 +56,7 @@ async def create_poo_record( @router.get("/poo/latest") def notify_latest_poo( - db: Session = Depends(get_poo_db), + db: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), homeassistant_client: HomeAssistantClient = Depends(get_homeassistant_client), ) -> Response: diff --git a/app/api/routes/public_ip.py b/app/api/routes/public_ip.py index 766525f..6ebc0b4 100644 --- a/app/api/routes/public_ip.py +++ b/app/api/routes/public_ip.py @@ -1,7 +1,7 @@ from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session -from app.dependencies import get_auth_db, get_current_auth_session +from app.dependencies import get_db, get_current_auth_session from app.schemas.public_ip import PublicIPCheckResponse from app.config import get_settings from app.services.auth import AuthenticatedSession @@ -12,7 +12,7 @@ router = APIRouter(tags=["public-ip"]) @router.get("/public-ip/check", response_model=PublicIPCheckResponse) def run_public_ip_check( - session: Session = Depends(get_auth_db), + session: Session = Depends(get_db), current_auth: AuthenticatedSession | None = Depends(get_current_auth_session), ) -> PublicIPCheckResponse: if current_auth is None: diff --git a/app/api/routes/ticktick.py b/app/api/routes/ticktick.py index b728108..9a1a417 100644 --- a/app/api/routes/ticktick.py +++ b/app/api/routes/ticktick.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import Session from app.config import Settings from app.dependencies import ( get_app_settings, - get_auth_db, + get_db, get_current_auth_session, get_ticktick_client, ) @@ -39,7 +39,7 @@ def start_ticktick_auth( @router.get("/ticktick/auth/code") def handle_ticktick_auth_code( request: Request, - auth_db_session: Session = Depends(get_auth_db), + auth_db_session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), ticktick_client: TickTickClient = Depends(get_ticktick_client), ) -> Response: diff --git a/app/auth_db.py b/app/auth_db.py deleted file mode 100644 index 41dcd1f..0000000 --- a/app/auth_db.py +++ /dev/null @@ -1,53 +0,0 @@ -from collections.abc import Generator -from functools import lru_cache - -from sqlalchemy import create_engine -from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker - -from app.config import get_settings - - -class AuthBase(DeclarativeBase): - pass - - -def _build_connect_args(database_url: str) -> dict[str, object]: - connect_args: dict[str, object] = {} - if database_url.startswith("sqlite"): - connect_args["check_same_thread"] = False - return connect_args - - -@lru_cache -def _get_auth_engine(database_url: str): - return create_engine(database_url, connect_args=_build_connect_args(database_url)) - - -@lru_cache -def _get_auth_session_local(database_url: str): - engine = _get_auth_engine(database_url) - return sessionmaker(bind=engine, autoflush=False, autocommit=False, class_=Session) - - -def get_auth_engine(): - settings = get_settings() - return _get_auth_engine(settings.app_database_url) - - -def get_auth_session_local(): - settings = get_settings() - return _get_auth_session_local(settings.app_database_url) - - -def reset_auth_db_caches() -> None: - _get_auth_session_local.cache_clear() - _get_auth_engine.cache_clear() - - -def get_auth_db_session() -> Generator[Session, None, None]: - session_local = get_auth_session_local() - session = session_local() - try: - yield session - finally: - session.close() diff --git a/app/db.py b/app/db.py index c8d94a9..ed8068d 100644 --- a/app/db.py +++ b/app/db.py @@ -1,6 +1,8 @@ from collections.abc import Generator +from functools import lru_cache -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event +from sqlalchemy.engine import Engine from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker from app.config import get_settings @@ -10,18 +12,49 @@ class Base(DeclarativeBase): pass -settings = get_settings() +def _build_connect_args(database_url: str) -> dict[str, object]: + connect_args: dict[str, object] = {} + if database_url.startswith("sqlite"): + connect_args["check_same_thread"] = False + return connect_args -connect_args: dict[str, object] = {} -if settings.location_database_url.startswith("sqlite"): - connect_args["check_same_thread"] = False -engine = create_engine(settings.location_database_url, connect_args=connect_args) -SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, class_=Session) +@lru_cache +def _get_engine(database_url: str) -> Engine: + engine = create_engine(database_url, connect_args=_build_connect_args(database_url)) + if database_url.startswith("sqlite"): + + @event.listens_for(engine, "connect") + def _enable_sqlite_wal(dbapi_connection, _connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.close() + + return engine + + +@lru_cache +def _get_session_local(database_url: str) -> sessionmaker: + engine = _get_engine(database_url) + return sessionmaker(bind=engine, autoflush=False, autocommit=False, class_=Session) + + +def get_engine() -> Engine: + return _get_engine(get_settings().app_database_url) + + +def get_session_local() -> sessionmaker: + return _get_session_local(get_settings().app_database_url) + + +def reset_db_caches() -> None: + _get_session_local.cache_clear() + _get_engine.cache_clear() def get_db_session() -> Generator[Session, None, None]: - session = SessionLocal() + session_local = get_session_local() + session = session_local() try: yield session finally: diff --git a/app/dependencies.py b/app/dependencies.py index ed4f3f0..570fb2b 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -3,30 +3,20 @@ from collections.abc import Generator from fastapi import Depends, Request from sqlalchemy.orm import Session -from app.auth_db import get_auth_db_session from app.config import Settings, get_settings from app.db import get_db_session from app.integrations.homeassistant import HomeAssistantClient from app.integrations.ticktick import TickTickClient -from app.poo_db import get_poo_db_session from app.services.auth import AuthenticatedSession, get_authenticated_session from app.services.config_page import build_runtime_settings -def get_auth_db() -> Generator[Session, None, None]: - yield from get_auth_db_session() - - -def get_app_settings(session: Session = Depends(get_auth_db)) -> Settings: - return build_runtime_settings(session, get_settings()) - - def get_db() -> Generator[Session, None, None]: yield from get_db_session() -def get_poo_db() -> Generator[Session, None, None]: - yield from get_poo_db_session() +def get_app_settings(session: Session = Depends(get_db)) -> Settings: + return build_runtime_settings(session, get_settings()) def get_homeassistant_client(settings: Settings = Depends(get_app_settings)) -> HomeAssistantClient: @@ -39,7 +29,7 @@ def get_ticktick_client(settings: Settings = Depends(get_app_settings)) -> TickT def get_current_auth_session( request: Request, - session: Session = Depends(get_auth_db), + session: Session = Depends(get_db), settings: Settings = Depends(get_app_settings), ) -> AuthenticatedSession | None: raw_token = request.cookies.get(settings.auth_session_cookie_name) diff --git a/app/main.py b/app/main.py index dd8a9ec..3e0647e 100644 --- a/app/main.py +++ b/app/main.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import Session from app import models # noqa: F401 from app.api.routes.auth import router as auth_router from app.api.routes import pages, status -import app.auth_db as auth_db +from app.db import get_session_local from app.api.routes.homeassistant import router as homeassistant_router from app.api.routes.location import router as location_router from app.api.routes.poo import router as poo_router @@ -26,7 +26,7 @@ from scripts.poo_db_adopt import PooDatabaseAdoptionError, validate_poo_runtime_ def _run_scheduled_public_ip_check() -> None: - session_local = auth_db.get_auth_session_local() + session_local = get_session_local() session: Session = session_local() try: check_public_ipv4_and_notify(session, bootstrap_settings=get_settings()) @@ -35,7 +35,7 @@ def _run_scheduled_public_ip_check() -> None: def ensure_auth_db_ready() -> None: - session_local = auth_db.get_auth_session_local() + session_local = get_session_local() session: Session = session_local() try: validate_app_runtime_db(get_settings().app_database_url) diff --git a/app/models/__init__.py b/app/models/__init__.py index 24d4862..93e8e82 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -3,6 +3,7 @@ from app.models.auth import AuthSession, AuthUser from app.models.config import AppConfigEntry from app.models.location import Location +from app.models.poo import PooRecord from app.models.public_ip import PublicIPHistory, PublicIPState __all__ = [ @@ -10,6 +11,7 @@ __all__ = [ "AuthSession", "AuthUser", "Location", + "PooRecord", "PublicIPHistory", "PublicIPState", ] diff --git a/app/models/auth.py b/app/models/auth.py index 3284913..08da8e5 100644 --- a/app/models/auth.py +++ b/app/models/auth.py @@ -3,10 +3,10 @@ from datetime import datetime from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String from sqlalchemy.orm import Mapped, mapped_column, relationship -from app.auth_db import AuthBase +from app.db import Base -class AuthUser(AuthBase): +class AuthUser(Base): __tablename__ = "auth_users" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) @@ -19,7 +19,7 @@ class AuthUser(AuthBase): sessions: Mapped[list["AuthSession"]] = relationship(back_populates="user") -class AuthSession(AuthBase): +class AuthSession(Base): __tablename__ = "auth_sessions" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) diff --git a/app/models/base.py b/app/models/base.py deleted file mode 100644 index b852be4..0000000 --- a/app/models/base.py +++ /dev/null @@ -1,4 +0,0 @@ -from app.db import Base - -__all__ = ["Base"] - diff --git a/app/models/config.py b/app/models/config.py index 31c0dff..ec7d504 100644 --- a/app/models/config.py +++ b/app/models/config.py @@ -3,10 +3,10 @@ from datetime import datetime from sqlalchemy import DateTime, Integer, String from sqlalchemy.orm import Mapped, mapped_column -from app.auth_db import AuthBase +from app.db import Base -class AppConfigEntry(AuthBase): +class AppConfigEntry(Base): __tablename__ = "app_config" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) diff --git a/app/models/poo.py b/app/models/poo.py index 6476fd8..2083a64 100644 --- a/app/models/poo.py +++ b/app/models/poo.py @@ -1,10 +1,10 @@ from sqlalchemy import Float, String from sqlalchemy.orm import Mapped, mapped_column -from app.poo_db import PooBase +from app.db import Base -class PooRecord(PooBase): +class PooRecord(Base): __tablename__ = "poo_records" timestamp: Mapped[str] = mapped_column(String, primary_key=True) diff --git a/app/models/public_ip.py b/app/models/public_ip.py index a88fd4e..589a237 100644 --- a/app/models/public_ip.py +++ b/app/models/public_ip.py @@ -3,10 +3,10 @@ from datetime import datetime from sqlalchemy import DateTime, Integer, String from sqlalchemy.orm import Mapped, mapped_column -from app.auth_db import AuthBase +from app.db import Base -class PublicIPState(AuthBase): +class PublicIPState(Base): __tablename__ = "public_ip_state" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) @@ -20,7 +20,7 @@ class PublicIPState(AuthBase): last_provider: Mapped[str | None] = mapped_column(String(64), nullable=True) -class PublicIPHistory(AuthBase): +class PublicIPHistory(Base): __tablename__ = "public_ip_history" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) diff --git a/app/poo_db.py b/app/poo_db.py deleted file mode 100644 index 3fdda48..0000000 --- a/app/poo_db.py +++ /dev/null @@ -1,28 +0,0 @@ -from collections.abc import Generator - -from sqlalchemy import create_engine -from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker - -from app.config import get_settings - - -class PooBase(DeclarativeBase): - pass - - -settings = get_settings() - -connect_args: dict[str, object] = {} -if settings.poo_database_url.startswith("sqlite"): - connect_args["check_same_thread"] = False - -poo_engine = create_engine(settings.poo_database_url, connect_args=connect_args) -PooSessionLocal = sessionmaker(bind=poo_engine, autoflush=False, autocommit=False, class_=Session) - - -def get_poo_db_session() -> Generator[Session, None, None]: - session = PooSessionLocal() - try: - yield session - finally: - session.close() diff --git a/app/services/config_page.py b/app/services/config_page.py index db7a450..956417f 100644 --- a/app/services/config_page.py +++ b/app/services/config_page.py @@ -7,7 +7,7 @@ from typing import Any from sqlalchemy import select from sqlalchemy.orm import Session -from app.auth_db import reset_auth_db_caches +from app.db import reset_db_caches from app.config import Settings, get_settings from app.models.config import AppConfigEntry @@ -127,7 +127,7 @@ def sync_app_hostname_from_bootstrap(session: Session, bootstrap_settings: Setti current_values["APP_HOSTNAME"] = bootstrap_hostname _persist_config_values(session, current_values) get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def build_runtime_settings(session: Session, bootstrap_settings: Settings) -> Settings: @@ -184,7 +184,7 @@ def save_config_updates(session: Session, form_data: dict[str, str], bootstrap_s _validate_config_values(merged_values, bootstrap_settings) _persist_config_values(session, merged_values) get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def save_config_value( @@ -199,7 +199,7 @@ def save_config_value( _validate_config_values(current_values, bootstrap_settings) _persist_config_values(session, current_values) get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def is_ticktick_oauth_ready(settings: Settings) -> bool: diff --git a/tests/conftest.py b/tests/conftest.py index 948661f..270284b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,8 @@ from alembic import command from alembic.config import Config from fastapi.testclient import TestClient from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from app.auth_db import reset_auth_db_caches -import app.db as app_db +from app.db import reset_db_caches from app.config import get_settings from app.main import create_app @@ -47,7 +45,7 @@ def test_database_urls(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") monkeypatch.setenv("AUTH_COOKIE_SECURE_OVERRIDE", "false") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() try: yield { @@ -60,7 +58,7 @@ def test_database_urls(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): } finally: get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() @pytest.fixture @@ -79,10 +77,10 @@ def ready_poo_database(test_database_urls): def auth_database(test_database_urls, monkeypatch: pytest.MonkeyPatch): database_url = test_database_urls["app_url"] command.upgrade(_make_app_alembic_config(database_url), "head") - reset_auth_db_caches() + reset_db_caches() yield test_database_urls - reset_auth_db_caches() + reset_db_caches() @pytest.fixture @@ -97,46 +95,20 @@ def client(app): @pytest.fixture -def location_client( - ready_location_database, - ready_poo_database, - auth_database, - monkeypatch: pytest.MonkeyPatch, -): - database_url = ready_location_database["location_url"] - - engine = create_engine(database_url, connect_args={"check_same_thread": False}) - session_local = sessionmaker(bind=engine, autoflush=False, autocommit=False) - - monkeypatch.setattr(app_db, "engine", engine) - monkeypatch.setattr(app_db, "SessionLocal", session_local) - +def location_client(ready_location_database, ready_poo_database, auth_database): + app_url = auth_database["app_url"] + engine = create_engine(app_url, connect_args={"check_same_thread": False}) fastapi_app = create_app() with TestClient(fastapi_app) as client: yield client, engine - engine.dispose() @pytest.fixture -def poo_client( - ready_location_database, - ready_poo_database, - auth_database, - monkeypatch: pytest.MonkeyPatch, -): - database_url = ready_poo_database["poo_url"] - - engine = create_engine(database_url, connect_args={"check_same_thread": False}) - session_local = sessionmaker(bind=engine, autoflush=False, autocommit=False) - - import app.poo_db as poo_db - - monkeypatch.setattr(poo_db, "poo_engine", engine) - monkeypatch.setattr(poo_db, "PooSessionLocal", session_local) - +def poo_client(ready_location_database, ready_poo_database, auth_database): + app_url = auth_database["app_url"] + engine = create_engine(app_url, connect_args={"check_same_thread": False}) fastapi_app = create_app() with TestClient(fastapi_app) as client: yield client, engine - engine.dispose() diff --git a/tests/test_app.py b/tests/test_app.py index 05a57dc..2857032 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -5,7 +5,7 @@ import pytest from alembic import command from fastapi.testclient import TestClient -from app.auth_db import reset_auth_db_caches +from app.db import reset_db_caches from app.config import get_settings from app.main import create_app from scripts.app_db_adopt import APP_BASELINE_REVISION, adopt_or_initialize_app_db @@ -49,7 +49,7 @@ def test_app_start_fails_when_app_db_missing(tmp_path, monkeypatch: pytest.Monke monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}") monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() app = create_app() with pytest.raises(RuntimeError, match="Run 'python scripts/app_db_adopt.py' first"): @@ -58,7 +58,7 @@ def test_app_start_fails_when_app_db_missing(tmp_path, monkeypatch: pytest.Monke assert not missing_app_path.exists() get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def test_app_db_adoption_initializes_new_database(tmp_path) -> None: @@ -108,7 +108,7 @@ def test_app_start_seeds_missing_config_from_env_without_overwriting_existing_va monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}") monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() app = create_app() anyio.run(_run_lifespan, app) @@ -124,7 +124,7 @@ def test_app_start_seeds_missing_config_from_env_without_overwriting_existing_va assert rows["AUTH_SESSION_COOKIE_NAME"] == "home_automation_session" get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def test_app_start_syncs_app_hostname_from_env_even_when_db_has_old_value( @@ -152,7 +152,7 @@ def test_app_start_syncs_app_hostname_from_env_even_when_db_has_old_value( monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}") monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() app = create_app() anyio.run(_run_lifespan, app) @@ -166,7 +166,7 @@ def test_app_start_syncs_app_hostname_from_env_even_when_db_has_old_value( assert rows["APP_HOSTNAME"] == "new.example.com" get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def test_app_start_fails_when_location_db_missing( @@ -182,14 +182,14 @@ def test_app_start_fails_when_location_db_missing( monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{tmp_path / 'missing.db'}") monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() app = create_app() with pytest.raises(RuntimeError, match="Run 'python scripts/location_db_adopt.py' first"): anyio.run(_run_lifespan, app) get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def test_app_start_fails_when_location_db_exists_but_is_not_adopted( @@ -223,14 +223,14 @@ def test_app_start_fails_when_location_db_exists_but_is_not_adopted( monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{database_path}") monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() app = create_app() with pytest.raises(RuntimeError, match="is not yet Alembic-managed"): anyio.run(_run_lifespan, app) get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def test_app_start_fails_when_location_db_revision_mismatches( @@ -254,11 +254,11 @@ def test_app_start_fails_when_location_db_revision_mismatches( monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{database_path}") monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() app = create_app() with pytest.raises(RuntimeError, match="Location DB revision mismatch"): anyio.run(_run_lifespan, app) get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() diff --git a/tests/test_auth.py b/tests/test_auth.py index b0d8c56..2558a5e 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -4,7 +4,7 @@ from pathlib import Path from fastapi.testclient import TestClient -from app.auth_db import reset_auth_db_caches +from app.db import reset_db_caches from app.config import get_settings from app.main import create_app @@ -205,7 +205,7 @@ def test_config_page_shows_ticktick_oauth_link_when_ticktick_is_configured( monkeypatch.setenv("TICKTICK_CLIENT_ID", "ticktick-client-id") monkeypatch.setenv("TICKTICK_CLIENT_SECRET", "ticktick-client-secret") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() with TestClient(create_app()) as client: login_page = client.get("/login") diff --git a/tests/test_deployment.py b/tests/test_deployment.py index d46691f..f2b4f28 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -6,7 +6,7 @@ import pytest import yaml from alembic import command -from app.auth_db import reset_auth_db_caches +from app.db import reset_db_caches from app.config import get_settings from app.main import create_app from scripts.app_db_adopt import APP_BASELINE_REVISION @@ -41,7 +41,7 @@ def _configure_database_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") monkeypatch.setenv("AUTH_COOKIE_SECURE_OVERRIDE", "false") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() return { "app_path": app_path, @@ -165,7 +165,7 @@ def test_migration_runner_initializes_and_is_idempotent( conn.close() get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def test_migration_runner_adopts_legacy_sqlite_without_data_loss( @@ -194,7 +194,7 @@ def test_migration_runner_adopts_legacy_sqlite_without_data_loss( conn.close() get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() def test_app_startup_still_fails_closed_without_running_adoption( @@ -212,4 +212,4 @@ def test_app_startup_still_fails_closed_without_running_adoption( assert not Path(missing_app_path).exists() get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() diff --git a/tests/test_homeassistant_inbound.py b/tests/test_homeassistant_inbound.py index a3f9753..47407f7 100644 --- a/tests/test_homeassistant_inbound.py +++ b/tests/test_homeassistant_inbound.py @@ -1,7 +1,5 @@ from sqlalchemy import text -import app.db as app_db -import app.poo_db as poo_db from app.config import Settings, get_settings from app.dependencies import get_app_settings, get_homeassistant_client from app.main import create_app @@ -161,42 +159,24 @@ def test_homeassistant_publish_poo_get_latest_publishes_latest_status( ready_location_database, ready_poo_database, auth_database, - monkeypatch, ) -> None: - location_engine = app_db.create_engine( - ready_location_database["location_url"], - connect_args={"check_same_thread": False}, - ) - location_session_local = app_db.sessionmaker( - bind=location_engine, - autoflush=False, - autocommit=False, - ) - poo_engine = poo_db.create_engine( - ready_poo_database["poo_url"], - connect_args={"check_same_thread": False}, - ) - poo_session_local = poo_db.sessionmaker( - bind=poo_engine, - autoflush=False, - autocommit=False, - ) + from fastapi.testclient import TestClient + from sqlalchemy import create_engine + + app_url = auth_database["app_url"] + engine = create_engine(app_url, connect_args={"check_same_thread": False}) + fake_ha = _FakeHomeAssistantClient() settings = Settings( poo_sensor_entity_name="sensor.test_poo_status", poo_sensor_friendly_name="Poo Status", ) - monkeypatch.setattr(app_db, "engine", location_engine) - monkeypatch.setattr(app_db, "SessionLocal", location_session_local) - monkeypatch.setattr(poo_db, "poo_engine", poo_engine) - monkeypatch.setattr(poo_db, "PooSessionLocal", poo_session_local) - test_app = create_app() test_app.dependency_overrides[get_homeassistant_client] = lambda: fake_ha test_app.dependency_overrides[get_app_settings] = lambda: settings - with poo_engine.begin() as conn: + with engine.begin() as conn: conn.execute( text( "INSERT INTO poo_records (timestamp, status, latitude, longitude) " @@ -211,8 +191,6 @@ def test_homeassistant_publish_poo_get_latest_publishes_latest_status( ) try: - from fastapi.testclient import TestClient - with TestClient(test_app) as client: response = client.post( "/homeassistant/publish", @@ -233,52 +211,27 @@ def test_homeassistant_publish_poo_get_latest_publishes_latest_status( finally: test_app.dependency_overrides.clear() get_settings.cache_clear() - location_engine.dispose() - poo_engine.dispose() + engine.dispose() def test_homeassistant_publish_returns_internal_error_for_unknown_poo_action( ready_location_database, ready_poo_database, auth_database, - monkeypatch, ) -> None: - location_engine = app_db.create_engine( - ready_location_database["location_url"], - connect_args={"check_same_thread": False}, - ) - location_session_local = app_db.sessionmaker( - bind=location_engine, - autoflush=False, - autocommit=False, - ) - poo_engine = poo_db.create_engine( - ready_poo_database["poo_url"], - connect_args={"check_same_thread": False}, - ) - poo_session_local = poo_db.sessionmaker( - bind=poo_engine, - autoflush=False, - autocommit=False, - ) + from fastapi.testclient import TestClient + fake_ha = _FakeHomeAssistantClient() settings = Settings( poo_sensor_entity_name="sensor.test_poo_status", poo_sensor_friendly_name="Poo Status", ) - monkeypatch.setattr(app_db, "engine", location_engine) - monkeypatch.setattr(app_db, "SessionLocal", location_session_local) - monkeypatch.setattr(poo_db, "poo_engine", poo_engine) - monkeypatch.setattr(poo_db, "PooSessionLocal", poo_session_local) - test_app = create_app() test_app.dependency_overrides[get_homeassistant_client] = lambda: fake_ha test_app.dependency_overrides[get_app_settings] = lambda: settings try: - from fastapi.testclient import TestClient - with TestClient(test_app) as client: response = client.post( "/homeassistant/publish", @@ -295,8 +248,6 @@ def test_homeassistant_publish_returns_internal_error_for_unknown_poo_action( finally: test_app.dependency_overrides.clear() get_settings.cache_clear() - location_engine.dispose() - poo_engine.dispose() def test_homeassistant_publish_returns_not_implemented_for_unknown_location_action( diff --git a/tests/test_location.py b/tests/test_location.py index 3af7c6e..f510245 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -5,18 +5,14 @@ import sqlite3 import pytest from alembic import command from alembic.config import Config -from sqlalchemy import create_engine, text -from sqlalchemy.orm import sessionmaker +from sqlalchemy import text -import app.db as app_db -from app.main import create_app from scripts.location_db_adopt import ( EXPECTED_USER_VERSION, LOCATION_BASELINE_REVISION, LocationDatabaseAdoptionError, adopt_or_initialize_location_db, ) -from tests.conftest import _make_app_alembic_config, _make_poo_alembic_config def _make_alembic_config(database_url: str) -> Config: @@ -197,66 +193,6 @@ def test_location_record_endpoint_defaults_invalid_altitude_to_zero(location_cli assert row.altitude == pytest.approx(0.0) -def test_legacy_style_location_db_can_be_stamped_and_adopted( - test_database_urls, monkeypatch: pytest.MonkeyPatch -) -> None: - app_database_url = test_database_urls["app_url"] - database_path = test_database_urls["location_path"] - database_url = test_database_urls["location_url"] - poo_database_url = test_database_urls["poo_url"] - - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE location ( - person TEXT NOT NULL, - datetime TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - altitude REAL, - PRIMARY KEY (person, datetime) - ) - """ - ) - conn.execute("PRAGMA user_version = 2") - conn.commit() - conn.close() - - command.upgrade(_make_app_alembic_config(app_database_url), "head") - command.stamp(_make_alembic_config(database_url), LOCATION_BASELINE_REVISION) - command.upgrade(_make_poo_alembic_config(poo_database_url), "head") - - engine = create_engine(database_url, connect_args={"check_same_thread": False}) - session_local = sessionmaker(bind=engine, autoflush=False, autocommit=False) - monkeypatch.setattr(app_db, "engine", engine) - monkeypatch.setattr(app_db, "SessionLocal", session_local) - - from fastapi.testclient import TestClient - - fastapi_app = create_app() - with TestClient(fastapi_app) as client: - response = client.post( - "/location/record", - json={ - "person": "legacy-user", - "latitude": "12.3", - "longitude": "45.6", - "altitude": "7.8", - }, - ) - - assert response.status_code == 200 - - with engine.connect() as db_conn: - revision = db_conn.execute(text("SELECT version_num FROM alembic_version")).scalar_one() - row_count = db_conn.execute(text("SELECT COUNT(*) FROM location")).scalar_one() - - assert revision == LOCATION_BASELINE_REVISION - assert row_count == 1 - - engine.dispose() - - def test_location_db_adoption_initializes_new_db(tmp_path: Path) -> None: database_path = tmp_path / "new_location.db" result = adopt_or_initialize_location_db(f"sqlite:///{database_path}") diff --git a/tests/test_ticktick.py b/tests/test_ticktick.py index 32ea1a9..036a486 100644 --- a/tests/test_ticktick.py +++ b/tests/test_ticktick.py @@ -6,7 +6,7 @@ from urllib.parse import parse_qs, urlparse import pytest from fastapi.testclient import TestClient -from app.auth_db import reset_auth_db_caches +from app.db import reset_db_caches from app.config import Settings, get_settings from app.integrations.ticktick import ( AUTH_SCOPE, @@ -221,7 +221,7 @@ def test_homeassistant_publish_creates_ticktick_action_task( monkeypatch.setenv("TICKTICK_TOKEN", "ticktick-access-token") monkeypatch.setenv("HOME_ASSISTANT_ACTION_TASK_PROJECT_ID", "project-123") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() captured = {"calls": []} @@ -265,7 +265,7 @@ def test_ticktick_auth_start_redirects_authenticated_user( monkeypatch.setenv("TICKTICK_CLIENT_ID", "ticktick-client-id") monkeypatch.setenv("TICKTICK_CLIENT_SECRET", "ticktick-client-secret") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() monkeypatch.setattr("app.integrations.ticktick.secrets.token_hex", lambda _: "state-redirect") with TestClient(create_app()) as client: @@ -301,7 +301,7 @@ def test_ticktick_auth_callback_persists_token( monkeypatch.setenv("TICKTICK_CLIENT_ID", "ticktick-client-id") monkeypatch.setenv("TICKTICK_CLIENT_SECRET", "ticktick-client-secret") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() default_auth_state_store.pending_state = "callback-state" def fake_urlopen(req, timeout): @@ -342,7 +342,7 @@ def test_ticktick_auth_callback_redirects_on_invalid_state( monkeypatch.setenv("TICKTICK_CLIENT_ID", "ticktick-client-id") monkeypatch.setenv("TICKTICK_CLIENT_SECRET", "ticktick-client-secret") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() default_auth_state_store.pending_state = "expected-state" with TestClient(create_app()) as client: @@ -366,7 +366,7 @@ def test_ticktick_auth_callback_redirects_when_token_exchange_fails( monkeypatch.setenv("TICKTICK_CLIENT_ID", "ticktick-client-id") monkeypatch.setenv("TICKTICK_CLIENT_SECRET", "ticktick-client-secret") get_settings.cache_clear() - reset_auth_db_caches() + reset_db_caches() default_auth_state_store.pending_state = "callback-state" def fake_urlopen(req, timeout): From 0d898e09f2567126bb13e76859b72966b99aac5a Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 16:50:05 +0200 Subject: [PATCH 4/9] M1-T04: converge startup chain onto the single app DB run_all_migrations() now adopts/initializes only the app DB and returns {'app': ...}. app/main.py drops the location/poo readiness checks (ensure_location_db_ready / ensure_poo_db_ready) and their imports; ensure_runtime_dirs only provisions the app DB path; lifespan still fail-closes on a missing/unmanaged app DB. Delete the retired location/poo adopt scripts and the alembic_location / alembic_poo chains. Update tests to single-DB expectations and drop the obsolete location/poo adoption + readiness tests. pytest 95 passed; ruff clean (pre-existing only); a fresh app DB initialized via scripts.run_migrations contains location + poo_records. --- alembic_location.ini | 37 ---- alembic_location/README | 2 - alembic_location/env.py | 48 ---- alembic_location/script.py.mako | 26 --- alembic_location/versions/.gitkeep | 1 - .../versions/20260419_01_location_baseline.py | 33 --- alembic_poo.ini | 37 ---- alembic_poo/env.py | 48 ---- .../versions/20260420_01_poo_baseline.py | 32 --- app/main.py | 31 +-- scripts/location_db_adopt.py | 205 ------------------ scripts/poo_db_adopt.py | 200 ----------------- scripts/run_migrations.py | 4 - tests/conftest.py | 30 +-- tests/test_app.py | 115 +--------- tests/test_auth.py | 3 - tests/test_deployment.py | 114 +--------- tests/test_homeassistant_inbound.py | 4 - tests/test_location.py | 149 ------------- tests/test_poo.py | 102 --------- tests/test_ticktick.py | 17 +- 21 files changed, 12 insertions(+), 1226 deletions(-) delete mode 100644 alembic_location.ini delete mode 100644 alembic_location/README delete mode 100644 alembic_location/env.py delete mode 100644 alembic_location/script.py.mako delete mode 100644 alembic_location/versions/.gitkeep delete mode 100644 alembic_location/versions/20260419_01_location_baseline.py delete mode 100644 alembic_poo.ini delete mode 100644 alembic_poo/env.py delete mode 100644 alembic_poo/versions/20260420_01_poo_baseline.py delete mode 100644 scripts/location_db_adopt.py delete mode 100644 scripts/poo_db_adopt.py diff --git a/alembic_location.ini b/alembic_location.ini deleted file mode 100644 index 9ee39f0..0000000 --- a/alembic_location.ini +++ /dev/null @@ -1,37 +0,0 @@ -[alembic] -script_location = alembic_location -prepend_sys_path = . -path_separator = os -sqlalchemy.url = sqlite:///./data/locationRecorder.db - -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/alembic_location/README b/alembic_location/README deleted file mode 100644 index 0eb160c..0000000 --- a/alembic_location/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains the Alembic migration environment for the Python rewrite skeleton. - diff --git a/alembic_location/env.py b/alembic_location/env.py deleted file mode 100644 index 05b9217..0000000 --- a/alembic_location/env.py +++ /dev/null @@ -1,48 +0,0 @@ -from logging.config import fileConfig - -from alembic import context -from sqlalchemy import engine_from_config, pool - -from app.config import get_settings -from app.models import Location # noqa: F401 -from app.db import Base - -config = context.config - -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -settings = get_settings() -configured_url = config.get_main_option("sqlalchemy.url") -if not configured_url or configured_url == "sqlite:///./data/locationRecorder.db": - config.set_main_option("sqlalchemy.url", settings.location_database_url) - -target_metadata = Base.metadata - - -def run_migrations_offline() -> None: - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=target_metadata, literal_binds=True) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/alembic_location/script.py.mako b/alembic_location/script.py.mako deleted file mode 100644 index 2e8960a..0000000 --- a/alembic_location/script.py.mako +++ /dev/null @@ -1,26 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} - diff --git a/alembic_location/versions/.gitkeep b/alembic_location/versions/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/alembic_location/versions/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/alembic_location/versions/20260419_01_location_baseline.py b/alembic_location/versions/20260419_01_location_baseline.py deleted file mode 100644 index cc94da5..0000000 --- a/alembic_location/versions/20260419_01_location_baseline.py +++ /dev/null @@ -1,33 +0,0 @@ -"""location baseline - -Revision ID: 20260419_01_location_baseline -Revises: -Create Date: 2026-04-19 00:00:00.000000 -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -revision: str = "20260419_01_location_baseline" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "location", - sa.Column("person", sa.Text(), nullable=False), - sa.Column("datetime", sa.Text(), nullable=False), - sa.Column("latitude", sa.Float(), nullable=False), - sa.Column("longitude", sa.Float(), nullable=False), - sa.Column("altitude", sa.Float(), nullable=True), - sa.PrimaryKeyConstraint("person", "datetime"), - ) - - -def downgrade() -> None: - op.drop_table("location") diff --git a/alembic_poo.ini b/alembic_poo.ini deleted file mode 100644 index cfe1727..0000000 --- a/alembic_poo.ini +++ /dev/null @@ -1,37 +0,0 @@ -[alembic] -script_location = alembic_poo -prepend_sys_path = . -path_separator = os -sqlalchemy.url = sqlite:///./data/pooRecorder.db - -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = console -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/alembic_poo/env.py b/alembic_poo/env.py deleted file mode 100644 index 98b528f..0000000 --- a/alembic_poo/env.py +++ /dev/null @@ -1,48 +0,0 @@ -from logging.config import fileConfig - -from alembic import context -from sqlalchemy import engine_from_config, pool - -from app.config import get_settings -from app.models.poo import PooRecord # noqa: F401 -from app.db import Base - -config = context.config - -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -settings = get_settings() -configured_url = config.get_main_option("sqlalchemy.url") -if not configured_url or configured_url == "sqlite:///./data/pooRecorder.db": - config.set_main_option("sqlalchemy.url", settings.poo_database_url) - -target_metadata = Base.metadata - - -def run_migrations_offline() -> None: - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=target_metadata, literal_binds=True) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/alembic_poo/versions/20260420_01_poo_baseline.py b/alembic_poo/versions/20260420_01_poo_baseline.py deleted file mode 100644 index 7abeb90..0000000 --- a/alembic_poo/versions/20260420_01_poo_baseline.py +++ /dev/null @@ -1,32 +0,0 @@ -"""poo baseline - -Revision ID: 20260420_01_poo_baseline -Revises: -Create Date: 2026-04-20 00:00:00.000000 -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -revision: str = "20260420_01_poo_baseline" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "poo_records", - sa.Column("timestamp", sa.Text(), nullable=False), - sa.Column("status", sa.Text(), nullable=False), - sa.Column("latitude", sa.Float(), nullable=False), - sa.Column("longitude", sa.Float(), nullable=False), - sa.PrimaryKeyConstraint("timestamp"), - ) - - -def downgrade() -> None: - op.drop_table("poo_records") diff --git a/app/main.py b/app/main.py index 3e0647e..6b42820 100644 --- a/app/main.py +++ b/app/main.py @@ -21,8 +21,6 @@ from app.services.auth import AuthBootstrapError, initialize_auth_schema from app.services.config_page import seed_missing_config_from_bootstrap, sync_app_hostname_from_bootstrap from app.services.public_ip import check_public_ipv4_and_notify from scripts.app_db_adopt import AppDatabaseAdoptionError, validate_app_runtime_db -from scripts.location_db_adopt import LocationDatabaseAdoptionError, validate_location_runtime_db -from scripts.poo_db_adopt import PooDatabaseAdoptionError, validate_poo_runtime_db def _run_scheduled_public_ip_check() -> None: @@ -50,41 +48,16 @@ def ensure_auth_db_ready() -> None: session.close() -def ensure_location_db_ready() -> None: - settings = get_settings() - if settings.location_sqlite_path is None: - return - - try: - validate_location_runtime_db(settings.location_database_url) - except LocationDatabaseAdoptionError as exc: - raise RuntimeError(str(exc)) from exc - - -def ensure_poo_db_ready() -> None: - settings = get_settings() - if settings.poo_sqlite_path is None: - return - - try: - validate_poo_runtime_db(settings.poo_database_url) - except PooDatabaseAdoptionError as exc: - raise RuntimeError(str(exc)) from exc - - def ensure_runtime_dirs() -> None: settings = get_settings() - for path in (settings.app_sqlite_path, settings.location_sqlite_path, settings.poo_sqlite_path): - if path is not None: - path.parent.mkdir(parents=True, exist_ok=True) + if settings.app_sqlite_path is not None: + settings.app_sqlite_path.parent.mkdir(parents=True, exist_ok=True) @asynccontextmanager async def lifespan(_: FastAPI): ensure_runtime_dirs() ensure_auth_db_ready() - ensure_location_db_ready() - ensure_poo_db_ready() scheduler = BackgroundScheduler(timezone="UTC") scheduler.add_job( _run_scheduled_public_ip_check, diff --git a/scripts/location_db_adopt.py b/scripts/location_db_adopt.py deleted file mode 100644 index 4424364..0000000 --- a/scripts/location_db_adopt.py +++ /dev/null @@ -1,205 +0,0 @@ -from __future__ import annotations - -import sqlite3 -import sys -from pathlib import Path - -from alembic import command -from alembic.config import Config -from alembic.script import ScriptDirectory -from alembic.util.exc import CommandError - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from app.config import get_settings - -LOCATION_BASELINE_REVISION = "20260419_01_location_baseline" -EXPECTED_USER_VERSION = 2 -EXPECTED_LOCATION_TABLE_INFO = [ - (0, "person", "TEXT", 1, None, 1), - (1, "datetime", "TEXT", 1, None, 2), - (2, "latitude", "REAL", 1, None, 0), - (3, "longitude", "REAL", 1, None, 0), - (4, "altitude", "REAL", 0, None, 0), -] - - -class LocationDatabaseAdoptionError(RuntimeError): - """Raised when a legacy location database does not match the expected baseline.""" - - -def _database_path_from_url(database_url: str) -> Path: - prefix = "sqlite:///" - if not database_url.startswith(prefix): - raise LocationDatabaseAdoptionError( - f"Only sqlite URLs are supported for location DB adoption, got: {database_url}" - ) - return Path(database_url[len(prefix) :]) - - -def _make_alembic_config(database_url: str) -> Config: - config = Config("alembic_location.ini") - config.set_main_option("sqlalchemy.url", database_url) - return config - - -def _expected_head_revision(alembic_config: Config) -> str: - script = ScriptDirectory.from_config(alembic_config) - heads = script.get_heads() - if len(heads) != 1: - raise LocationDatabaseAdoptionError( - f"Expected exactly one Alembic head for location DB, got {len(heads)}" - ) - return heads[0] - - -def _is_known_revision(alembic_config: Config, revision: str) -> bool: - script = ScriptDirectory.from_config(alembic_config) - try: - return script.get_revision(revision) is not None - except CommandError: - return False - - -def _location_table_exists(database_path: Path) -> bool: - conn = sqlite3.connect(database_path) - try: - row = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'location'" - ).fetchone() - return row is not None - finally: - conn.close() - - -def _alembic_version_table_exists(database_path: Path) -> bool: - conn = sqlite3.connect(database_path) - try: - row = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'alembic_version'" - ).fetchone() - return row is not None - finally: - conn.close() - - -def _fetch_alembic_revision(database_path: Path) -> str: - conn = sqlite3.connect(database_path) - try: - row = conn.execute("SELECT version_num FROM alembic_version").fetchone() - if row is None: - raise LocationDatabaseAdoptionError( - "Alembic version table exists but contains no revision" - ) - return row[0] - finally: - conn.close() - - -def _fetch_location_table_info(database_path: Path) -> list[tuple]: - conn = sqlite3.connect(database_path) - try: - return list(conn.execute("PRAGMA table_info(location)")) - finally: - conn.close() - - -def _fetch_user_version(database_path: Path) -> int: - conn = sqlite3.connect(database_path) - try: - return conn.execute("PRAGMA user_version").fetchone()[0] - finally: - conn.close() - - -def validate_legacy_location_db(database_url: str) -> None: - database_path = _database_path_from_url(database_url) - if not database_path.exists(): - raise LocationDatabaseAdoptionError(f"Location DB file does not exist: {database_path}") - - if not _location_table_exists(database_path): - raise LocationDatabaseAdoptionError("Expected table 'location' was not found in the DB") - - table_info = _fetch_location_table_info(database_path) - if table_info != EXPECTED_LOCATION_TABLE_INFO: - raise LocationDatabaseAdoptionError( - "Location table schema does not match the expected baseline schema" - ) - - user_version = _fetch_user_version(database_path) - if user_version != EXPECTED_USER_VERSION: - raise LocationDatabaseAdoptionError( - f"Expected PRAGMA user_version = {EXPECTED_USER_VERSION}, got {user_version}" - ) - - -def validate_location_runtime_db(database_url: str) -> None: - database_path = _database_path_from_url(database_url) - alembic_config = _make_alembic_config(database_url) - expected_revision = _expected_head_revision(alembic_config) - if not database_path.exists(): - raise LocationDatabaseAdoptionError( - "Location DB file was not found. Run 'python scripts/location_db_adopt.py' " - "first to initialize or adopt the location DB before starting the app." - ) - - if not _alembic_version_table_exists(database_path): - raise LocationDatabaseAdoptionError( - "Location DB exists but is not yet Alembic-managed. Run " - "'python scripts/location_db_adopt.py' first to adopt the legacy DB " - "before starting the app." - ) - - current_revision = _fetch_alembic_revision(database_path) - if current_revision != expected_revision: - raise LocationDatabaseAdoptionError( - "Location DB revision mismatch. Refusing to start the app: " - f"expected {expected_revision}, got {current_revision}" - ) - - -def adopt_or_initialize_location_db(database_url: str) -> str: - database_path = _database_path_from_url(database_url) - alembic_config = _make_alembic_config(database_url) - expected_revision = _expected_head_revision(alembic_config) - - if database_path.exists(): - if _alembic_version_table_exists(database_path): - current_revision = _fetch_alembic_revision(database_path) - if current_revision == expected_revision: - return "already_managed" - if not _is_known_revision(alembic_config, current_revision): - raise LocationDatabaseAdoptionError( - "Location DB is already Alembic-managed but revision does not match " - f"a known migration revision: got {current_revision}" - ) - command.upgrade(alembic_config, "head") - return "upgraded" - - validate_legacy_location_db(database_url) - command.stamp(alembic_config, LOCATION_BASELINE_REVISION) - if LOCATION_BASELINE_REVISION != expected_revision: - command.upgrade(alembic_config, "head") - return "upgraded" - return "adopted" - - database_path.parent.mkdir(parents=True, exist_ok=True) - command.upgrade(alembic_config, "head") - return "initialized" - - -def main() -> None: - settings = get_settings() - result = adopt_or_initialize_location_db(settings.location_database_url) - if result == "initialized": - print("Initialized a new location DB via Alembic upgrade head.") - elif result == "already_managed": - print("Location DB is already Alembic-managed at the expected baseline revision.") - else: - print("Validated legacy location DB and stamped Alembic baseline successfully.") - - -if __name__ == "__main__": - main() diff --git a/scripts/poo_db_adopt.py b/scripts/poo_db_adopt.py deleted file mode 100644 index 7540dce..0000000 --- a/scripts/poo_db_adopt.py +++ /dev/null @@ -1,200 +0,0 @@ -from __future__ import annotations - -import sqlite3 -import sys -from pathlib import Path - -from alembic import command -from alembic.config import Config -from alembic.script import ScriptDirectory -from alembic.util.exc import CommandError - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from app.config import get_settings - -POO_BASELINE_REVISION = "20260420_01_poo_baseline" -EXPECTED_USER_VERSION = 1 -EXPECTED_POO_TABLE_INFO = [ - (0, "timestamp", "TEXT", 1, None, 1), - (1, "status", "TEXT", 1, None, 0), - (2, "latitude", "REAL", 1, None, 0), - (3, "longitude", "REAL", 1, None, 0), -] - - -class PooDatabaseAdoptionError(RuntimeError): - """Raised when a legacy poo database does not match the expected baseline.""" - - -def _database_path_from_url(database_url: str) -> Path: - prefix = "sqlite:///" - if not database_url.startswith(prefix): - raise PooDatabaseAdoptionError( - f"Only sqlite URLs are supported for poo DB adoption, got: {database_url}" - ) - return Path(database_url[len(prefix) :]) - - -def _make_alembic_config(database_url: str) -> Config: - config = Config("alembic_poo.ini") - config.set_main_option("sqlalchemy.url", database_url) - return config - - -def _expected_head_revision(alembic_config: Config) -> str: - script = ScriptDirectory.from_config(alembic_config) - heads = script.get_heads() - if len(heads) != 1: - raise PooDatabaseAdoptionError( - f"Expected exactly one Alembic head for poo DB, got {len(heads)}" - ) - return heads[0] - - -def _is_known_revision(alembic_config: Config, revision: str) -> bool: - script = ScriptDirectory.from_config(alembic_config) - try: - return script.get_revision(revision) is not None - except CommandError: - return False - - -def _poo_table_exists(database_path: Path) -> bool: - conn = sqlite3.connect(database_path) - try: - row = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'poo_records'" - ).fetchone() - return row is not None - finally: - conn.close() - - -def _alembic_version_table_exists(database_path: Path) -> bool: - conn = sqlite3.connect(database_path) - try: - row = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'alembic_version'" - ).fetchone() - return row is not None - finally: - conn.close() - - -def _fetch_alembic_revision(database_path: Path) -> str: - conn = sqlite3.connect(database_path) - try: - row = conn.execute("SELECT version_num FROM alembic_version").fetchone() - if row is None: - raise PooDatabaseAdoptionError("Alembic version table exists but contains no revision") - return row[0] - finally: - conn.close() - - -def _fetch_poo_table_info(database_path: Path) -> list[tuple]: - conn = sqlite3.connect(database_path) - try: - return list(conn.execute("PRAGMA table_info(poo_records)")) - finally: - conn.close() - - -def _fetch_user_version(database_path: Path) -> int: - conn = sqlite3.connect(database_path) - try: - return conn.execute("PRAGMA user_version").fetchone()[0] - finally: - conn.close() - - -def validate_legacy_poo_db(database_url: str) -> None: - database_path = _database_path_from_url(database_url) - if not database_path.exists(): - raise PooDatabaseAdoptionError(f"Poo DB file does not exist: {database_path}") - - if not _poo_table_exists(database_path): - raise PooDatabaseAdoptionError("Expected table 'poo_records' was not found in the DB") - - table_info = _fetch_poo_table_info(database_path) - if table_info != EXPECTED_POO_TABLE_INFO: - raise PooDatabaseAdoptionError("Poo table schema does not match the expected baseline") - - user_version = _fetch_user_version(database_path) - if user_version != EXPECTED_USER_VERSION: - raise PooDatabaseAdoptionError( - f"Expected PRAGMA user_version = {EXPECTED_USER_VERSION}, got {user_version}" - ) - - -def validate_poo_runtime_db(database_url: str) -> None: - database_path = _database_path_from_url(database_url) - alembic_config = _make_alembic_config(database_url) - expected_revision = _expected_head_revision(alembic_config) - if not database_path.exists(): - raise PooDatabaseAdoptionError( - "Poo DB file was not found. Run 'python scripts/poo_db_adopt.py' first to " - "initialize or adopt the poo DB before starting the app." - ) - - if not _alembic_version_table_exists(database_path): - raise PooDatabaseAdoptionError( - "Poo DB exists but is not yet Alembic-managed. Run " - "'python scripts/poo_db_adopt.py' first to adopt the legacy DB " - "before starting the app." - ) - - current_revision = _fetch_alembic_revision(database_path) - if current_revision != expected_revision: - raise PooDatabaseAdoptionError( - "Poo DB revision mismatch. Refusing to start the app: " - f"expected {expected_revision}, got {current_revision}" - ) - - -def adopt_or_initialize_poo_db(database_url: str) -> str: - database_path = _database_path_from_url(database_url) - alembic_config = _make_alembic_config(database_url) - expected_revision = _expected_head_revision(alembic_config) - - if database_path.exists(): - if _alembic_version_table_exists(database_path): - current_revision = _fetch_alembic_revision(database_path) - if current_revision == expected_revision: - return "already_managed" - if not _is_known_revision(alembic_config, current_revision): - raise PooDatabaseAdoptionError( - "Poo DB is already Alembic-managed but revision does not match " - f"a known migration revision: got {current_revision}" - ) - command.upgrade(alembic_config, "head") - return "upgraded" - - validate_legacy_poo_db(database_url) - command.stamp(alembic_config, POO_BASELINE_REVISION) - if POO_BASELINE_REVISION != expected_revision: - command.upgrade(alembic_config, "head") - return "upgraded" - return "adopted" - - database_path.parent.mkdir(parents=True, exist_ok=True) - command.upgrade(alembic_config, "head") - return "initialized" - - -def main() -> None: - settings = get_settings() - result = adopt_or_initialize_poo_db(settings.poo_database_url) - if result == "initialized": - print("Initialized a new poo DB via Alembic upgrade head.") - elif result == "already_managed": - print("Poo DB is already Alembic-managed at the expected baseline revision.") - else: - print("Validated legacy poo DB and stamped Alembic baseline successfully.") - - -if __name__ == "__main__": - main() diff --git a/scripts/run_migrations.py b/scripts/run_migrations.py index ba3af4e..b213886 100644 --- a/scripts/run_migrations.py +++ b/scripts/run_migrations.py @@ -2,16 +2,12 @@ from __future__ import annotations from app.config import get_settings from scripts.app_db_adopt import adopt_or_initialize_app_db -from scripts.location_db_adopt import adopt_or_initialize_location_db -from scripts.poo_db_adopt import adopt_or_initialize_poo_db def run_all_migrations() -> dict[str, str]: settings = get_settings() return { "app": adopt_or_initialize_app_db(settings.app_database_url), - "location": adopt_or_initialize_location_db(settings.location_database_url), - "poo": adopt_or_initialize_poo_db(settings.poo_database_url), } diff --git a/tests/conftest.py b/tests/conftest.py index 270284b..da18463 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,18 +17,6 @@ def _make_app_alembic_config(database_url: str) -> Config: return config -def _make_alembic_config(database_url: str) -> Config: - config = Config("alembic_location.ini") - config.set_main_option("sqlalchemy.url", database_url) - return config - - -def _make_poo_alembic_config(database_url: str) -> Config: - config = Config("alembic_poo.ini") - config.set_main_option("sqlalchemy.url", database_url) - return config - - @pytest.fixture def test_database_urls(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): app_database_path = tmp_path / "app_test.db" @@ -61,18 +49,6 @@ def test_database_urls(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): reset_db_caches() -@pytest.fixture -def ready_location_database(test_database_urls): - command.upgrade(_make_alembic_config(test_database_urls["location_url"]), "head") - return test_database_urls - - -@pytest.fixture -def ready_poo_database(test_database_urls): - command.upgrade(_make_poo_alembic_config(test_database_urls["poo_url"]), "head") - return test_database_urls - - @pytest.fixture def auth_database(test_database_urls, monkeypatch: pytest.MonkeyPatch): database_url = test_database_urls["app_url"] @@ -84,7 +60,7 @@ def auth_database(test_database_urls, monkeypatch: pytest.MonkeyPatch): @pytest.fixture -def app(ready_location_database, ready_poo_database, auth_database): +def app(auth_database): yield create_app() @@ -95,7 +71,7 @@ def client(app): @pytest.fixture -def location_client(ready_location_database, ready_poo_database, auth_database): +def location_client(auth_database): app_url = auth_database["app_url"] engine = create_engine(app_url, connect_args={"check_same_thread": False}) fastapi_app = create_app() @@ -105,7 +81,7 @@ def location_client(ready_location_database, ready_poo_database, auth_database): @pytest.fixture -def poo_client(ready_location_database, ready_poo_database, auth_database): +def poo_client(auth_database): app_url = auth_database["app_url"] engine = create_engine(app_url, connect_args={"check_same_thread": False}) fastapi_app = create_app() diff --git a/tests/test_app.py b/tests/test_app.py index 2857032..79b611d 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -9,7 +9,7 @@ from app.db import reset_db_caches from app.config import get_settings from app.main import create_app from scripts.app_db_adopt import APP_BASELINE_REVISION, adopt_or_initialize_app_db -from tests.conftest import _make_alembic_config, _make_app_alembic_config, _make_poo_alembic_config +from tests.conftest import _make_app_alembic_config async def _run_lifespan(app) -> None: @@ -38,16 +38,10 @@ def test_status_endpoint(client: TestClient) -> None: def test_app_start_fails_when_app_db_missing(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: missing_app_path = tmp_path / "missing_app.db" - poo_database_path = tmp_path / "poo_ready.db" - location_database_path = tmp_path / "location_ready.db" - command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head") - command.upgrade(_make_alembic_config(f"sqlite:///{location_database_path}"), "head") monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{missing_app_path}") monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") - monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}") - monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() reset_db_caches() @@ -86,10 +80,6 @@ def test_app_start_seeds_missing_config_from_env_without_overwriting_existing_va tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: app_database_url = _prepare_app_db(tmp_path) - location_database_path = tmp_path / "location_ready.db" - poo_database_path = tmp_path / "poo_ready.db" - command.upgrade(_make_alembic_config(f"sqlite:///{location_database_path}"), "head") - command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head") app_database_path = tmp_path / "app_ready.db" conn = sqlite3.connect(app_database_path) @@ -105,8 +95,6 @@ def test_app_start_seeds_missing_config_from_env_without_overwriting_existing_va monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") monkeypatch.setenv("APP_NAME", "Bootstrap Name") monkeypatch.setenv("HOME_ASSISTANT_BASE_URL", "http://bootstrap-ha.local:8123") - monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}") - monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() reset_db_caches() @@ -131,10 +119,6 @@ def test_app_start_syncs_app_hostname_from_env_even_when_db_has_old_value( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: app_database_url = _prepare_app_db(tmp_path) - location_database_path = tmp_path / "location_ready.db" - poo_database_path = tmp_path / "poo_ready.db" - command.upgrade(_make_alembic_config(f"sqlite:///{location_database_path}"), "head") - command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head") app_database_path = tmp_path / "app_ready.db" conn = sqlite3.connect(app_database_path) @@ -149,8 +133,6 @@ def test_app_start_syncs_app_hostname_from_env_even_when_db_has_old_value( monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") monkeypatch.setenv("APP_HOSTNAME", "new.example.com") - monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}") - monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") get_settings.cache_clear() reset_db_caches() @@ -167,98 +149,3 @@ def test_app_start_syncs_app_hostname_from_env_even_when_db_has_old_value( get_settings.cache_clear() reset_db_caches() - - -def test_app_start_fails_when_location_db_missing( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: - app_database_url = _prepare_app_db(tmp_path) - monkeypatch.setenv("APP_DATABASE_URL", app_database_url) - monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") - monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") - poo_database_path = tmp_path / "poo_ready.db" - command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head") - - monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{tmp_path / 'missing.db'}") - monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") - get_settings.cache_clear() - reset_db_caches() - - app = create_app() - with pytest.raises(RuntimeError, match="Run 'python scripts/location_db_adopt.py' first"): - anyio.run(_run_lifespan, app) - - get_settings.cache_clear() - reset_db_caches() - - -def test_app_start_fails_when_location_db_exists_but_is_not_adopted( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: - app_database_url = _prepare_app_db(tmp_path) - monkeypatch.setenv("APP_DATABASE_URL", app_database_url) - monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") - monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") - poo_database_path = tmp_path / "poo_ready.db" - command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head") - - database_path = tmp_path / "legacy_only.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE location ( - person TEXT NOT NULL, - datetime TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - altitude REAL, - PRIMARY KEY (person, datetime) - ) - """ - ) - conn.execute("PRAGMA user_version = 2") - conn.commit() - conn.close() - - monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{database_path}") - monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") - get_settings.cache_clear() - reset_db_caches() - - app = create_app() - with pytest.raises(RuntimeError, match="is not yet Alembic-managed"): - anyio.run(_run_lifespan, app) - - get_settings.cache_clear() - reset_db_caches() - - -def test_app_start_fails_when_location_db_revision_mismatches( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: - app_database_url = _prepare_app_db(tmp_path) - monkeypatch.setenv("APP_DATABASE_URL", app_database_url) - monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") - monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") - poo_database_path = tmp_path / "poo_ready.db" - command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head") - - database_path = tmp_path / "wrong_revision.db" - command.upgrade(_make_alembic_config(f"sqlite:///{database_path}"), "head") - - conn = sqlite3.connect(database_path) - conn.execute("UPDATE alembic_version SET version_num = 'wrong_revision'") - conn.commit() - conn.close() - - monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{database_path}") - monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_database_path}") - get_settings.cache_clear() - reset_db_caches() - - app = create_app() - with pytest.raises(RuntimeError, match="Location DB revision mismatch"): - anyio.run(_run_lifespan, app) - - get_settings.cache_clear() - reset_db_caches() diff --git a/tests/test_auth.py b/tests/test_auth.py index 2558a5e..f39849d 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -194,9 +194,6 @@ def test_config_page_update_persists_to_database( def test_config_page_shows_ticktick_oauth_link_when_ticktick_is_configured( - test_database_urls, - ready_location_database, - ready_poo_database, auth_database, monkeypatch, ) -> None: diff --git a/tests/test_deployment.py b/tests/test_deployment.py index f2b4f28..6bc10b2 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -4,18 +4,12 @@ import sqlite3 import anyio import pytest import yaml -from alembic import command from app.db import reset_db_caches from app.config import get_settings from app.main import create_app from scripts.app_db_adopt import APP_BASELINE_REVISION -from scripts.location_db_adopt import EXPECTED_USER_VERSION as LOCATION_USER_VERSION -from scripts.location_db_adopt import LOCATION_BASELINE_REVISION -from scripts.poo_db_adopt import EXPECTED_USER_VERSION as POO_USER_VERSION -from scripts.poo_db_adopt import POO_BASELINE_REVISION from scripts.run_migrations import run_all_migrations -from tests.conftest import _make_alembic_config, _make_poo_alembic_config PROJECT_ROOT = Path(__file__).resolve().parents[1] @@ -31,12 +25,8 @@ async def _run_lifespan(app) -> None: def _configure_database_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path | str]: app_path = tmp_path / "app.db" - location_path = tmp_path / "location.db" - poo_path = tmp_path / "poo.db" monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{app_path}") - monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_path}") - monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_path}") monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") monkeypatch.setenv("AUTH_COOKIE_SECURE_OVERRIDE", "false") @@ -46,58 +36,9 @@ def _configure_database_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> return { "app_path": app_path, "app_url": f"sqlite:///{app_path}", - "location_path": location_path, - "location_url": f"sqlite:///{location_path}", - "poo_path": poo_path, - "poo_url": f"sqlite:///{poo_path}", } -def _create_legacy_location_db(database_path: Path) -> None: - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE location ( - person TEXT NOT NULL, - datetime TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - altitude REAL, - PRIMARY KEY (person, datetime) - ) - """ - ) - conn.execute( - "INSERT INTO location (person, datetime, latitude, longitude, altitude) VALUES (?, ?, ?, ?, ?)", - ("alice", "2026-04-22T10:00:00Z", 1.23, 4.56, 7.89), - ) - conn.execute(f"PRAGMA user_version = {LOCATION_USER_VERSION}") - conn.commit() - conn.close() - - -def _create_legacy_poo_db(database_path: Path) -> None: - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE poo_records ( - timestamp TEXT NOT NULL, - status TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - PRIMARY KEY (timestamp) - ) - """ - ) - conn.execute( - "INSERT INTO poo_records (timestamp, status, latitude, longitude) VALUES (?, ?, ?, ?)", - ("2026-04-22T11:00:00Z", "complete", 9.87, 6.54), - ) - conn.execute(f"PRAGMA user_version = {POO_USER_VERSION}") - conn.commit() - conn.close() - - def test_compose_uses_migration_job_before_app() -> None: compose = _read_yaml("docker-compose.yml") override = _read_yaml("docker-compose.override.yml") @@ -131,12 +72,8 @@ def test_migration_runner_initializes_and_is_idempotent( first_run = run_all_migrations() second_run = run_all_migrations() - assert first_run == {"app": "initialized", "location": "initialized", "poo": "initialized"} - assert second_run == { - "app": "already_managed", - "location": "already_managed", - "poo": "already_managed", - } + assert first_run == {"app": "initialized"} + assert second_run == {"app": "already_managed"} conn = sqlite3.connect(database_urls["app_path"]) try: @@ -150,48 +87,9 @@ def test_migration_runner_initializes_and_is_idempotent( finally: conn.close() - assert {"auth_users", "auth_sessions", "app_config", "alembic_version"} <= tables - - conn = sqlite3.connect(database_urls["location_path"]) - try: - assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == LOCATION_BASELINE_REVISION - finally: - conn.close() - - conn = sqlite3.connect(database_urls["poo_path"]) - try: - assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == POO_BASELINE_REVISION - finally: - conn.close() - - get_settings.cache_clear() - reset_db_caches() - - -def test_migration_runner_adopts_legacy_sqlite_without_data_loss( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - database_urls = _configure_database_env(tmp_path, monkeypatch) - _create_legacy_location_db(database_urls["location_path"]) - _create_legacy_poo_db(database_urls["poo_path"]) - - results = run_all_migrations() - - assert results == {"app": "initialized", "location": "adopted", "poo": "adopted"} - - conn = sqlite3.connect(database_urls["location_path"]) - try: - assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == LOCATION_BASELINE_REVISION - assert conn.execute("SELECT COUNT(*) FROM location").fetchone()[0] == 1 - finally: - conn.close() - - conn = sqlite3.connect(database_urls["poo_path"]) - try: - assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == POO_BASELINE_REVISION - assert conn.execute("SELECT COUNT(*) FROM poo_records").fetchone()[0] == 1 - finally: - conn.close() + assert { + "auth_users", "auth_sessions", "app_config", "alembic_version", "location", "poo_records" + } <= tables get_settings.cache_clear() reset_db_caches() @@ -202,8 +100,6 @@ def test_app_startup_still_fails_closed_without_running_adoption( ) -> None: database_urls = _configure_database_env(tmp_path, monkeypatch) missing_app_path = database_urls["app_path"] - command.upgrade(_make_alembic_config(database_urls["location_url"]), "head") - command.upgrade(_make_poo_alembic_config(database_urls["poo_url"]), "head") app = create_app() with pytest.raises(RuntimeError, match="Run 'python scripts/app_db_adopt.py' first"): diff --git a/tests/test_homeassistant_inbound.py b/tests/test_homeassistant_inbound.py index 47407f7..04c2b79 100644 --- a/tests/test_homeassistant_inbound.py +++ b/tests/test_homeassistant_inbound.py @@ -156,8 +156,6 @@ def test_homeassistant_publish_rejects_invalid_ticktick_content(location_client) def test_homeassistant_publish_poo_get_latest_publishes_latest_status( - ready_location_database, - ready_poo_database, auth_database, ) -> None: from fastapi.testclient import TestClient @@ -215,8 +213,6 @@ def test_homeassistant_publish_poo_get_latest_publishes_latest_status( def test_homeassistant_publish_returns_internal_error_for_unknown_poo_action( - ready_location_database, - ready_poo_database, auth_database, ) -> None: from fastapi.testclient import TestClient diff --git a/tests/test_location.py b/tests/test_location.py index f510245..230851d 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -1,24 +1,8 @@ from datetime import datetime -from pathlib import Path -import sqlite3 import pytest -from alembic import command -from alembic.config import Config from sqlalchemy import text -from scripts.location_db_adopt import ( - EXPECTED_USER_VERSION, - LOCATION_BASELINE_REVISION, - LocationDatabaseAdoptionError, - adopt_or_initialize_location_db, -) - - -def _make_alembic_config(database_url: str) -> Config: - config = Config("alembic_location.ini") - config.set_main_option("sqlalchemy.url", database_url) - return config def test_location_record_endpoint_writes_row(location_client) -> None: client, engine = location_client @@ -193,136 +177,3 @@ def test_location_record_endpoint_defaults_invalid_altitude_to_zero(location_cli assert row.altitude == pytest.approx(0.0) -def test_location_db_adoption_initializes_new_db(tmp_path: Path) -> None: - database_path = tmp_path / "new_location.db" - result = adopt_or_initialize_location_db(f"sqlite:///{database_path}") - - assert result == "initialized" - assert database_path.exists() - - conn = sqlite3.connect(database_path) - try: - revision = conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] - location_table = conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'location'" - ).fetchone() - finally: - conn.close() - - assert revision == LOCATION_BASELINE_REVISION - assert location_table is not None - - -def test_location_db_adoption_validates_and_stamps_legacy_db(tmp_path: Path) -> None: - database_path = tmp_path / "legacy_location.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE location ( - person TEXT NOT NULL, - datetime TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - altitude REAL, - PRIMARY KEY (person, datetime) - ) - """ - ) - conn.execute(f"PRAGMA user_version = {EXPECTED_USER_VERSION}") - conn.commit() - conn.close() - - result = adopt_or_initialize_location_db(f"sqlite:///{database_path}") - - assert result == "adopted" - - conn = sqlite3.connect(database_path) - try: - revision = conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] - finally: - conn.close() - - assert revision == LOCATION_BASELINE_REVISION - - -def test_location_db_adoption_accepts_already_managed_matching_revision( - tmp_path: Path, -) -> None: - database_path = tmp_path / "managed_location.db" - command.upgrade(_make_alembic_config(f"sqlite:///{database_path}"), "head") - - result = adopt_or_initialize_location_db(f"sqlite:///{database_path}") - - assert result == "already_managed" - - -def test_location_db_adoption_fails_closed_on_alembic_revision_mismatch( - tmp_path: Path, -) -> None: - database_path = tmp_path / "wrong_revision.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE location ( - person TEXT NOT NULL, - datetime TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - altitude REAL, - PRIMARY KEY (person, datetime) - ) - """ - ) - conn.execute("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)") - conn.execute("INSERT INTO alembic_version (version_num) VALUES ('wrong_revision')") - conn.execute(f"PRAGMA user_version = {EXPECTED_USER_VERSION}") - conn.commit() - conn.close() - - with pytest.raises(LocationDatabaseAdoptionError, match="known migration revision"): - adopt_or_initialize_location_db(f"sqlite:///{database_path}") - - -def test_location_db_adoption_fails_closed_on_schema_mismatch(tmp_path: Path) -> None: - database_path = tmp_path / "bad_schema.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE location ( - person TEXT NOT NULL, - datetime TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - PRIMARY KEY (person, datetime) - ) - """ - ) - conn.execute(f"PRAGMA user_version = {EXPECTED_USER_VERSION}") - conn.commit() - conn.close() - - with pytest.raises(LocationDatabaseAdoptionError, match="schema does not match"): - adopt_or_initialize_location_db(f"sqlite:///{database_path}") - - -def test_location_db_adoption_fails_closed_on_user_version_mismatch(tmp_path: Path) -> None: - database_path = tmp_path / "bad_user_version.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE location ( - person TEXT NOT NULL, - datetime TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - altitude REAL, - PRIMARY KEY (person, datetime) - ) - """ - ) - conn.execute("PRAGMA user_version = 999") - conn.commit() - conn.close() - - with pytest.raises(LocationDatabaseAdoptionError, match="Expected PRAGMA user_version"): - adopt_or_initialize_location_db(f"sqlite:///{database_path}") diff --git a/tests/test_poo.py b/tests/test_poo.py index 9e9a4d5..f3c6eb3 100644 --- a/tests/test_poo.py +++ b/tests/test_poo.py @@ -1,17 +1,8 @@ -from pathlib import Path -import sqlite3 - import pytest from sqlalchemy import text from app.config import Settings, get_settings from app.dependencies import get_app_settings, get_homeassistant_client -from scripts.poo_db_adopt import ( - EXPECTED_USER_VERSION, - POO_BASELINE_REVISION, - PooDatabaseAdoptionError, - adopt_or_initialize_poo_db, -) class _FakeHomeAssistantClient: @@ -153,96 +144,3 @@ def test_poo_latest_endpoint_returns_ok_when_no_record_exists(poo_client_with_ov assert response.text == "" -def test_poo_db_adoption_initializes_new_db(tmp_path: Path) -> None: - database_path = tmp_path / "new_poo.db" - - result = adopt_or_initialize_poo_db(f"sqlite:///{database_path}") - - assert result == "initialized" - assert database_path.exists() - - conn = sqlite3.connect(database_path) - try: - revision = conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] - poo_table = conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'poo_records'" - ).fetchone() - finally: - conn.close() - - assert revision == POO_BASELINE_REVISION - assert poo_table is not None - - -def test_poo_db_adoption_validates_and_stamps_legacy_db(tmp_path: Path) -> None: - database_path = tmp_path / "legacy_poo.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE poo_records ( - timestamp TEXT NOT NULL, - status TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - PRIMARY KEY (timestamp) - ) - """ - ) - conn.execute(f"PRAGMA user_version = {EXPECTED_USER_VERSION}") - conn.commit() - conn.close() - - result = adopt_or_initialize_poo_db(f"sqlite:///{database_path}") - - assert result == "adopted" - - conn = sqlite3.connect(database_path) - try: - revision = conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] - finally: - conn.close() - - assert revision == POO_BASELINE_REVISION - - -def test_poo_db_adoption_fails_closed_on_schema_mismatch(tmp_path: Path) -> None: - database_path = tmp_path / "bad_poo_schema.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE poo_records ( - timestamp TEXT NOT NULL, - status TEXT NOT NULL, - latitude REAL NOT NULL, - PRIMARY KEY (timestamp) - ) - """ - ) - conn.execute(f"PRAGMA user_version = {EXPECTED_USER_VERSION}") - conn.commit() - conn.close() - - with pytest.raises(PooDatabaseAdoptionError, match="schema does not match"): - adopt_or_initialize_poo_db(f"sqlite:///{database_path}") - - -def test_poo_db_adoption_fails_closed_on_user_version_mismatch(tmp_path: Path) -> None: - database_path = tmp_path / "bad_poo_user_version.db" - conn = sqlite3.connect(database_path) - conn.execute( - """ - CREATE TABLE poo_records ( - timestamp TEXT NOT NULL, - status TEXT NOT NULL, - latitude REAL NOT NULL, - longitude REAL NOT NULL, - PRIMARY KEY (timestamp) - ) - """ - ) - conn.execute("PRAGMA user_version = 999") - conn.commit() - conn.close() - - with pytest.raises(PooDatabaseAdoptionError, match="Expected PRAGMA user_version"): - adopt_or_initialize_poo_db(f"sqlite:///{database_path}") diff --git a/tests/test_ticktick.py b/tests/test_ticktick.py index 036a486..f16cfab 100644 --- a/tests/test_ticktick.py +++ b/tests/test_ticktick.py @@ -209,9 +209,6 @@ def test_create_task_posts_expected_payload(monkeypatch: pytest.MonkeyPatch) -> def test_homeassistant_publish_creates_ticktick_action_task( - test_database_urls, - ready_location_database, - ready_poo_database, auth_database, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -255,9 +252,6 @@ def test_homeassistant_publish_creates_ticktick_action_task( def test_ticktick_auth_start_redirects_authenticated_user( - test_database_urls, - ready_location_database, - ready_poo_database, auth_database, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -291,9 +285,6 @@ def test_ticktick_auth_start_redirects_authenticated_user( def test_ticktick_auth_callback_persists_token( - test_database_urls, - ready_location_database, - ready_poo_database, auth_database, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -318,7 +309,7 @@ def test_ticktick_auth_callback_persists_token( assert response.status_code == 303 assert response.headers["location"] == "/config?ticktick_oauth=success" - conn = sqlite3.connect(test_database_urls["app_path"]) + conn = sqlite3.connect(auth_database["app_path"]) try: row = conn.execute( "SELECT value FROM app_config WHERE key = ?", @@ -332,9 +323,6 @@ def test_ticktick_auth_callback_persists_token( def test_ticktick_auth_callback_redirects_on_invalid_state( - test_database_urls, - ready_location_database, - ready_poo_database, auth_database, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -356,9 +344,6 @@ def test_ticktick_auth_callback_redirects_on_invalid_state( def test_ticktick_auth_callback_redirects_when_token_exchange_fails( - test_database_urls, - ready_location_database, - ready_poo_database, auth_database, monkeypatch: pytest.MonkeyPatch, ) -> None: From af8c602988fc12ecaa57a0ae28462c8ab2a0dd5c Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 16:57:54 +0200 Subject: [PATCH 5/9] M1-T05: drop location/poo database config from Settings and tests Remove the dead location_database_url / poo_database_url fields and the location_sqlite_path / poo_sqlite_path computed properties from Settings; drop them from the config-page payload and from .env.example. Update the test hardcodes (test_config, test_public_ip, test_smtp) and reduce the conftest test_database_urls fixture to the single app DB. The one-time migration script keeps reading legacy URLs from env/CLI, independent of Settings. pytest 95 passed; ruff clean (pre-existing only). --- .env.example | 2 -- app/config.py | 13 ------------- app/services/config_page.py | 2 -- tests/conftest.py | 10 ---------- tests/test_config.py | 10 +--------- tests/test_public_ip.py | 2 -- tests/test_smtp.py | 2 -- 7 files changed, 1 insertion(+), 40 deletions(-) diff --git a/.env.example b/.env.example index 4156f17..b0297cd 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,6 @@ APP_NAME=Home Automation Backend (Python) APP_ENV=production APP_HOSTNAME=home-automation.example.com APP_DATABASE_URL=sqlite:////app/data/app.db -LOCATION_DATABASE_URL=sqlite:////app/data/locationRecorder.db -POO_DATABASE_URL=sqlite:////app/data/pooRecorder.db AUTH_BOOTSTRAP_USERNAME=admin AUTH_BOOTSTRAP_PASSWORD=change-me diff --git a/app/config.py b/app/config.py index 29adab5..7140b61 100644 --- a/app/config.py +++ b/app/config.py @@ -12,9 +12,6 @@ class Settings(BaseSettings): app_hostname: str = "localhost:8000" app_database_url: str = "sqlite:///./data/app.db" - location_database_url: str = "sqlite:///./data/locationRecorder.db" - poo_database_url: str = "sqlite:///./data/pooRecorder.db" - ticktick_client_id: str = "" ticktick_client_secret: str = "" ticktick_token: str = "" @@ -77,21 +74,11 @@ class Settings(BaseSettings): raw_path = database_url[len(prefix) :] return Path(raw_path) - @computed_field - @property - def location_sqlite_path(self) -> Path | None: - return self._sqlite_path_from_url(self.location_database_url) - @computed_field @property def app_sqlite_path(self) -> Path | None: return self._sqlite_path_from_url(self.app_database_url) - @computed_field - @property - def poo_sqlite_path(self) -> Path | None: - return self._sqlite_path_from_url(self.poo_database_url) - @computed_field @property def auth_cookie_secure(self) -> bool: diff --git a/app/services/config_page.py b/app/services/config_page.py index 956417f..4deaaf4 100644 --- a/app/services/config_page.py +++ b/app/services/config_page.py @@ -260,8 +260,6 @@ def _settings_payload(settings: Settings) -> dict[str, Any]: "app_debug": settings.app_debug, "app_hostname": settings.app_hostname, "app_database_url": settings.app_database_url, - "location_database_url": settings.location_database_url, - "poo_database_url": settings.poo_database_url, "ticktick_client_id": settings.ticktick_client_id, "ticktick_client_secret": settings.ticktick_client_secret, "ticktick_token": settings.ticktick_token, diff --git a/tests/conftest.py b/tests/conftest.py index da18463..55537f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,15 +20,9 @@ def _make_app_alembic_config(database_url: str) -> Config: @pytest.fixture def test_database_urls(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): app_database_path = tmp_path / "app_test.db" - location_database_path = tmp_path / "location_test.db" - poo_database_path = tmp_path / "poo_placeholder.db" app_database_url = f"sqlite:///{app_database_path}" - location_database_url = f"sqlite:///{location_database_path}" - poo_database_url = f"sqlite:///{poo_database_path}" monkeypatch.setenv("APP_DATABASE_URL", app_database_url) - monkeypatch.setenv("LOCATION_DATABASE_URL", location_database_url) - monkeypatch.setenv("POO_DATABASE_URL", poo_database_url) monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") monkeypatch.setenv("AUTH_COOKIE_SECURE_OVERRIDE", "false") @@ -39,10 +33,6 @@ def test_database_urls(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): yield { "app_path": app_database_path, "app_url": app_database_url, - "location_path": location_database_path, - "location_url": location_database_url, - "poo_path": poo_database_path, - "poo_url": poo_database_url, } finally: get_settings.cache_clear() diff --git a/tests/test_config.py b/tests/test_config.py index 6aea8d0..458ebf0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,10 +1,8 @@ from app.config import Settings -def test_settings_support_two_independent_database_urls(monkeypatch) -> None: +def test_settings_load_core_fields_from_env(monkeypatch) -> None: monkeypatch.setenv("APP_DATABASE_URL", "sqlite:///./data/app.db") - monkeypatch.setenv("LOCATION_DATABASE_URL", "sqlite:///./data/locationRecorder.db") - monkeypatch.setenv("POO_DATABASE_URL", "sqlite:///./data/pooRecorder.db") monkeypatch.setenv("APP_HOSTNAME", "home.example.com") monkeypatch.setenv("POO_WEBHOOK_ID", "poo-hook") monkeypatch.setenv("POO_SENSOR_ENTITY_NAME", "sensor.test_poo_status") @@ -21,8 +19,6 @@ def test_settings_support_two_independent_database_urls(monkeypatch) -> None: settings = Settings(_env_file=None) assert settings.app_database_url == "sqlite:///./data/app.db" - assert settings.location_database_url == "sqlite:///./data/locationRecorder.db" - assert settings.poo_database_url == "sqlite:///./data/pooRecorder.db" assert settings.poo_webhook_id == "poo-hook" assert settings.poo_sensor_entity_name == "sensor.test_poo_status" assert settings.poo_sensor_friendly_name == "Poo Status" @@ -36,12 +32,8 @@ def test_settings_support_two_independent_database_urls(monkeypatch) -> None: assert settings.auth_bootstrap_password == "secret" assert settings.auth_session_cookie_name == "auth_cookie" assert settings.auth_session_ttl_hours == 8 - assert settings.location_sqlite_path is not None - assert settings.location_sqlite_path.name == "locationRecorder.db" assert settings.app_sqlite_path is not None assert settings.app_sqlite_path.name == "app.db" - assert settings.poo_sqlite_path is not None - assert settings.poo_sqlite_path.name == "pooRecorder.db" assert settings.auth_cookie_secure is True diff --git a/tests/test_public_ip.py b/tests/test_public_ip.py index a8a9558..6f88fdf 100644 --- a/tests/test_public_ip.py +++ b/tests/test_public_ip.py @@ -182,8 +182,6 @@ def _notification_settings() -> Settings: app_env="development", app_hostname="localhost:8000", app_database_url="sqlite:///./data/app.db", - location_database_url="sqlite:///./data/locationRecorder.db", - poo_database_url="sqlite:///./data/pooRecorder.db", auth_bootstrap_username="admin", auth_bootstrap_password="secret-password", smtp_enabled=True, diff --git a/tests/test_smtp.py b/tests/test_smtp.py index 2881da6..5d06635 100644 --- a/tests/test_smtp.py +++ b/tests/test_smtp.py @@ -40,8 +40,6 @@ def _smtp_settings(**overrides) -> Settings: "app_env": "development", "app_hostname": "localhost:8000", "app_database_url": "sqlite:///./data/app.db", - "location_database_url": "sqlite:///./data/locationRecorder.db", - "poo_database_url": "sqlite:///./data/pooRecorder.db", "auth_bootstrap_username": "admin", "auth_bootstrap_password": "secret-password", "smtp_enabled": True, From dc624bb7e5077a2ee19110657571d79088f10f4c Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 17:02:49 +0200 Subject: [PATCH 6/9] M1-T06: remove Grafana from compose and delete provisioning Drop the grafana service and the homeautomation_grafana_storage volume declaration from docker-compose.yml (migration and app services unchanged), and delete the grafana/ provisioning + dashboards directory. Visualization moves to the M2 React frontend; the named volume's actual data is retired manually as an ops step, never by automation. docker compose config -q passes; pytest 95 passed; ruff clean. --- docker-compose.yml | 19 -- grafana/dashboards/locationrecorder.json | 288 ------------------ grafana/dashboards/poorecorder.json | 231 -------------- grafana/provisioning/dashboards/provider.yaml | 13 - .../datasources/locationrecorder.yaml | 11 - .../provisioning/datasources/poorecorder.yaml | 11 - 6 files changed, 573 deletions(-) delete mode 100644 grafana/dashboards/locationrecorder.json delete mode 100644 grafana/dashboards/poorecorder.json delete mode 100644 grafana/provisioning/dashboards/provider.yaml delete mode 100644 grafana/provisioning/datasources/locationrecorder.yaml delete mode 100644 grafana/provisioning/datasources/poorecorder.yaml diff --git a/docker-compose.yml b/docker-compose.yml index b49aa84..caa1a4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,22 +25,3 @@ services: - ./data:/app/data - ./.env:/app/.env:ro - grafana: - image: grafana/grafana:latest - container_name: home-automation-grafana - depends_on: - - app - restart: unless-stopped - ports: - - "10.238.75.70:8882:3000" - environment: - GF_PLUGINS_PREINSTALL: frser-sqlite-datasource - volumes: - - ./data:/data/home-automation:ro - - ./grafana/provisioning:/etc/grafana/provisioning:ro - - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - - homeautomation_grafana_storage:/var/lib/grafana - -volumes: - homeautomation_grafana_storage: - name: homeautomation_grafana_storage diff --git a/grafana/dashboards/locationrecorder.json b/grafana/dashboards/locationrecorder.json deleted file mode 100644 index 5d417bd..0000000 --- a/grafana/dashboards/locationrecorder.json +++ /dev/null @@ -1,288 +0,0 @@ -{ - "apiVersion": "dashboard.grafana.app/v2", - "kind": "Dashboard", - "metadata": { - "name": "adzr6rv", - "namespace": "default", - "uid": "c5fc57e5-7fb5-4104-9861-023710ada568", - "resourceVersion": "1776634346371016", - "generation": 19, - "creationTimestamp": "2026-04-18T19:05:57Z", - "labels": { - "grafana.app/deprecatedInternalID": "945374452785152" - }, - "annotations": { - "grafana.app/createdBy": "user:ffjhknvgkvhtsc", - "grafana.app/folder": "", - "grafana.app/saved-from-ui": "Grafana v13.0.1 (a100054f)", - "grafana.app/updatedBy": "user:ffjhknvgkvhtsc", - "grafana.app/updatedTimestamp": "2026-04-19T21:32:26Z" - } - }, - "spec": { - "annotations": [ - { - "kind": "AnnotationQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "grafana", - "version": "v0", - "datasource": { - "name": "-- Grafana --" - }, - "spec": {} - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "builtIn": true - } - } - ], - "cursorSync": "Off", - "editable": true, - "elements": { - "panel-1": { - "kind": "Panel", - "spec": { - "id": 1, - "title": "轨迹", - "description": "", - "links": [], - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "frser-sqlite-datasource", - "version": "v0", - "datasource": { - "name": "ffjhr941d5iwwf" - }, - "spec": { - "queryText": "SELECT\n datetime AS time,\n latitude,\n longitude,\n altitude\nFROM location\nWHERE person = 'Jiangxue'\n AND datetime >= '2021-04-19T21:29:57.036Z'\n AND datetime <= '2026-04-19T21:29:57.036Z'\n AND latitude != 0\n AND longitude != 0\nORDER BY datetime;\n", - "queryType": "table", - "rawQueryText": "SELECT\n datetime AS time,\n latitude,\n longitude,\n altitude\nFROM location\nWHERE person = '$person'\n AND datetime >= '${__from:date:iso}'\n AND datetime <= '${__to:date:iso}'\n AND latitude != 0\n AND longitude != 0\nORDER BY datetime;\n", - "timeColumns": [ - "time", - "ts" - ] - } - }, - "refId": "A", - "hidden": false - } - } - ], - "transformations": [], - "queryOptions": {} - } - }, - "vizConfig": { - "kind": "VizConfig", - "group": "geomap", - "version": "13.0.1", - "spec": { - "options": { - "basemap": { - "config": { - "server": "streets" - }, - "name": "Layer 0", - "noRepeat": false, - "type": "default" - }, - "controls": { - "mouseWheelZoom": true, - "showAttribution": true, - "showDebug": false, - "showMeasure": false, - "showScale": false, - "showZoom": true - }, - "layers": [ - { - "config": { - "showLegend": false, - "style": { - "color": { - "fixed": "blue" - }, - "opacity": 0.7, - "rotation": { - "fixed": 0, - "max": 360, - "min": -360, - "mode": "mod" - }, - "size": { - "fixed": 3, - "max": 15, - "min": 2 - }, - "symbol": { - "fixed": "img/icons/marker/circle.svg", - "mode": "fixed" - }, - "symbolAlign": { - "horizontal": "center", - "vertical": "center" - }, - "textConfig": { - "fontSize": 12, - "offsetX": 0, - "offsetY": 0, - "textAlign": "center", - "textBaseline": "middle" - } - } - }, - "layer-tooltip": true, - "name": "path", - "tooltip": true, - "type": "markers" - } - ], - "tooltip": { - "mode": "details" - }, - "view": { - "allLayers": true, - "dashboardVariable": false, - "id": "fit", - "lat": 0, - "lon": 0, - "noRepeat": false, - "shared": false, - "zoom": 15 - } - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - } - ] - }, - "color": { - "mode": "thresholds" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - } - }, - "overrides": [] - } - } - } - } - } - }, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "x": 0, - "y": 0, - "width": 24, - "height": 18, - "element": { - "kind": "ElementReference", - "name": "panel-1" - } - } - } - ] - } - }, - "links": [], - "liveNow": false, - "preload": false, - "tags": [], - "timeSettings": { - "timezone": "browser", - "from": "now-5y", - "to": "now", - "autoRefresh": "", - "autoRefreshIntervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "hideTimepicker": false, - "fiscalYearStartMonth": 0 - }, - "title": "轨迹", - "variables": [ - { - "kind": "QueryVariable", - "spec": { - "name": "person", - "current": { - "text": "Jiangxue", - "value": "Jiangxue" - }, - "label": "person", - "hide": "dontHide", - "refresh": "onDashboardLoad", - "skipUrlSync": false, - "description": "", - "query": { - "kind": "DataQuery", - "group": "frser-sqlite-datasource", - "version": "v0", - "datasource": { - "name": "ffjhr941d5iwwf" - }, - "spec": { - "__legacyStringValue": "SELECT DISTINCT person\nFROM location\nORDER BY person;\n" - } - }, - "regex": "", - "regexApplyTo": "value", - "sort": "disabled", - "definition": "SELECT DISTINCT person\nFROM location\nORDER BY person;\n", - "options": [], - "multi": false, - "includeAll": false, - "allowCustomValue": true - } - } - ], - "preferences": { - "layout": { - "kind": "AutoGridLayout", - "spec": { - "maxColumnCount": 3, - "columnWidthMode": "standard", - "rowHeightMode": "standard", - "items": [] - } - } - } - } -} \ No newline at end of file diff --git a/grafana/dashboards/poorecorder.json b/grafana/dashboards/poorecorder.json deleted file mode 100644 index 006d079..0000000 --- a/grafana/dashboards/poorecorder.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "apiVersion": "dashboard.grafana.app/v2", - "kind": "Dashboard", - "metadata": { - "name": "adl5sjt", - "namespace": "default", - "uid": "d4c72406-9fc5-4b85-844b-be1250f1fa8b", - "resourceVersion": "1776606363367013", - "generation": 6, - "creationTimestamp": "2026-04-18T20:07:34Z", - "labels": { - "grafana.app/deprecatedInternalID": "960882027798528" - }, - "annotations": { - "grafana.app/createdBy": "user:ffjhknvgkvhtsc", - "grafana.app/folder": "", - "grafana.app/saved-from-ui": "Grafana v13.0.1 (a100054f)", - "grafana.app/updatedBy": "user:ffjhknvgkvhtsc", - "grafana.app/updatedTimestamp": "2026-04-19T13:46:03Z" - } - }, - "spec": { - "annotations": [ - { - "kind": "AnnotationQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "grafana", - "version": "v0", - "datasource": { - "name": "-- Grafana --" - }, - "spec": {} - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "builtIn": true - } - } - ], - "cursorSync": "Off", - "editable": true, - "elements": { - "panel-1": { - "kind": "Panel", - "spec": { - "id": 1, - "title": "Mika Poo", - "description": "Mika's poo", - "links": [], - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "frser-sqlite-datasource", - "version": "v0", - "datasource": { - "name": "ffjhkuu4hc3y8e" - }, - "spec": { - "queryText": "SELECT\n latitude,\n longitude,\n timestamp\nFROM poo_records\nWHERE timestamp >= '${__from:date:iso}'\n AND timestamp <= '${__to:date:iso}'\n AND latitude != 0\n AND longitude != 0\nORDER BY timestamp;\n", - "queryType": "table", - "rawQueryText": "SELECT\n latitude,\n longitude,\n timestamp\nFROM poo_records\nWHERE timestamp >= '${__from:date:iso}'\n AND timestamp <= '${__to:date:iso}'\n AND latitude != 0\n AND longitude != 0\nORDER BY timestamp;\n", - "timeColumns": [ - "time", - "ts" - ] - } - }, - "refId": "A", - "hidden": false - } - } - ], - "transformations": [], - "queryOptions": {} - } - }, - "vizConfig": { - "kind": "VizConfig", - "group": "geomap", - "version": "13.0.1", - "spec": { - "options": { - "basemap": { - "config": {}, - "name": "Layer 0", - "noRepeat": false, - "type": "default" - }, - "controls": { - "mouseWheelZoom": true, - "showAttribution": true, - "showDebug": false, - "showMeasure": false, - "showScale": false, - "showZoom": true - }, - "layers": [ - { - "config": { - "blur": 15, - "radius": 5, - "weight": { - "fixed": 1, - "max": 1, - "min": 0 - } - }, - "filterData": { - "id": "byRefId", - "options": "A" - }, - "location": { - "mode": "auto" - }, - "name": "Poo", - "tooltip": true, - "type": "heatmap" - } - ], - "tooltip": { - "mode": "details" - }, - "view": { - "allLayers": true, - "dashboardVariable": false, - "id": "zero", - "lat": 0, - "lon": 0, - "noRepeat": false, - "zoom": 1 - } - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": 0, - "color": "green" - }, - { - "value": 80, - "color": "red" - } - ] - }, - "color": { - "mode": "thresholds" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - } - }, - "overrides": [] - } - } - } - } - } - }, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "x": 0, - "y": 0, - "width": 24, - "height": 19, - "element": { - "kind": "ElementReference", - "name": "panel-1" - } - } - } - ] - } - }, - "links": [], - "liveNow": false, - "preload": false, - "tags": [], - "timeSettings": { - "timezone": "browser", - "from": "now-5y", - "to": "now", - "autoRefresh": "", - "autoRefreshIntervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "hideTimepicker": false, - "fiscalYearStartMonth": 0 - }, - "title": "Mika Poo", - "variables": [], - "preferences": { - "layout": { - "kind": "GridLayout", - "spec": { - "items": [] - } - } - } - } -} \ No newline at end of file diff --git a/grafana/provisioning/dashboards/provider.yaml b/grafana/provisioning/dashboards/provider.yaml deleted file mode 100644 index de78613..0000000 --- a/grafana/provisioning/dashboards/provider.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: 1 - -providers: - - name: home-automation-dashboards - orgId: 1 - folder: "" - type: file - disableDeletion: false - allowUiUpdates: false - updateIntervalSeconds: 30 - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: false \ No newline at end of file diff --git a/grafana/provisioning/datasources/locationrecorder.yaml b/grafana/provisioning/datasources/locationrecorder.yaml deleted file mode 100644 index 0c9749b..0000000 --- a/grafana/provisioning/datasources/locationrecorder.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 1 - -datasources: - - name: locationrecorder - uid: ffjhr941d5iwwf - type: frser-sqlite-datasource - access: proxy - isDefault: false - editable: false - jsonData: - path: /data/home-automation/locationRecorder.db \ No newline at end of file diff --git a/grafana/provisioning/datasources/poorecorder.yaml b/grafana/provisioning/datasources/poorecorder.yaml deleted file mode 100644 index 0fcd352..0000000 --- a/grafana/provisioning/datasources/poorecorder.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 1 - -datasources: - - name: poorecorder - uid: ffjhkuu4hc3y8e - type: frser-sqlite-datasource - access: proxy - isDefault: false - editable: false - jsonData: - path: /data/home-automation/pooRecorder.db \ No newline at end of file From 2f634006d2a72b72cccc8c30354db2235f402458 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 17:13:28 +0200 Subject: [PATCH 7/9] M1-T07: align docs to single-DB reality and re-export OpenAPI Rewrite README (single app.db + one alembic_app chain, legacy data moved once via scripts.migrate_legacy_data, accurate test list) and remove the Grafana Provisioning section. Update architecture-overview to the unified data layer (one Base, app-DB engine with WAL) and retire the alembic_location / alembic_poo sections. Mark M1 done in the roadmap. Re-export openapi/, which catches the spec up to the already-existing /config/smtp/test and /public-ip/check endpoints (purely additive; M1's DB-session dependency swap produced no schema change). pytest 95 passed; ruff clean (pre-existing only); OpenAPI export idempotent. --- README.md | 123 ++++++++++------------------------ docs/architecture-overview.md | 16 +---- docs/roadmap.md | 4 +- openapi/openapi.json | 72 ++++++++++++++++++++ openapi/openapi.yaml | 49 ++++++++++++++ 5 files changed, 161 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 9fd0eee..c1fb3b9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ 当前系统已经包含： - FastAPI Web 应用与服务端模板页面 -- SQLite + SQLAlchemy + Alembic 的三库结构 +- SQLite + SQLAlchemy + Alembic 的单库结构 - username/password + server-side session 鉴权 - runtime config 页面与 app DB 持久化 - public IPv4 monitor、历史持久化与定时检查 @@ -23,41 +23,32 @@ ## 当前配置现实 -当前系统仍然是三个独立的 SQLite 数据库文件，而不是单一数据库： +当前系统使用单一 SQLite 数据库文件（`app.db`），所有数据表都在其中： -- `app` 级共享数据使用自己的 DB 文件 -- `location` 模块使用自己的 DB 文件 -- `poo` 模块使用自己的 DB 文件 +- auth（单个 admin 用户、server-side session） +- runtime config 持久化（`app_config` 表） +- public IPv4 当前状态与变化历史 +- location 记录（`location` 表） +- poo 记录（`poo_records` 表） -当前阶段明确不借这次重构把这些 DB 合并。配置层已经显式反映这一点： +配置层只保留一个数据库环境变量： - `APP_DATABASE_URL` -- `LOCATION_DATABASE_URL` -- `POO_DATABASE_URL` -目前 auth、`location` 和 `poo` 都已经接到各自独立的数据库文件。 +`app.db` 不会在应用启动时自动创建，需要先运行： -其中 `app` 级共享 DB 当前主要用于： +```bash +python -m scripts.run_migrations +``` -- 单个 admin 用户 -- server-side session -- runtime config 持久化 -- public IPv4 当前状态与变化历史 - -这部分现在也使用 Alembic 管理： - -- `app db` 不会在应用启动时自动创建 -- 需要先运行 `python scripts/app_db_adopt.py` -- 这个脚本会创建新 DB 并建好 schema +该命令会通过 Alembic 将 `app.db` 初始化或升级到最新 head（含 `location` / `poo_records` 表）。 ## 当前目录 主要目录如下： - `app/`: FastAPI 应用代码 -- `alembic_app/`: App DB 的 Alembic migration 环境 -- `alembic_location/`: Location DB 的 Alembic migration 环境 -- `alembic_poo/`: Poo DB 的 Alembic migration 环境 +- `alembic_app/`: App DB 的 Alembic migration 环境（同时管理 `location` / `poo_records` 表） - `tests/`: pytest 测试 - `docs/`: 当前系统说明文档 - `scripts/`: 辅助脚本，例如 OpenAPI 导出 @@ -128,24 +119,22 @@ uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 ## 数据库与 Alembic -当前默认使用 SQLite，并区分三个数据库文件： +当前使用单一 SQLite 数据库文件： - App DB：`sqlite:///./data/app.db` -- Location DB：`sqlite:///./data/locationRecorder.db` -- Poo DB：`sqlite:///./data/pooRecorder.db` - 数据目录：`./data/` -初始化 migration 环境后，可继续添加模型并生成迁移： +所有模型（auth / config / public_ip / location / poo）共用同一个 `Base`，均通过单一 Alembic 链管理： -当前 `app`、`location` 和 `poo` 都已经有各自独立的 Alembic 链路。 - -- App Alembic 环境：`alembic_app.ini` + `alembic_app/` -- Location Alembic 环境：`alembic_location.ini` + `alembic_location/` -- Poo Alembic 环境：`alembic_poo.ini` + `alembic_poo/` +- Alembic 环境：`alembic_app.ini` + `alembic_app/` - 统一 migration job：`python -m scripts.run_migrations` -- App DB 初始化：`python scripts/app_db_adopt.py` -- Location DB 接管 / 初始化：`python scripts/location_db_adopt.py` -- Poo DB 接管 / 初始化：`python scripts/poo_db_adopt.py` +- App DB 接管 / 初始化：`python scripts/app_db_adopt.py` + +历史 location / poo 数据（旧版本遗留的独立 DB 文件）已通过以下脚本一次性迁移至 `app.db`（幂等，不删除旧文件）： + +```bash +python -m scripts.migrate_legacy_data +``` ## 基础鉴权 @@ -197,7 +186,7 @@ uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 这意味着： -- location / poo / app DB 地址仍然属于 bootstrap 范畴 +- app DB 地址（`APP_DATABASE_URL`）仍然属于 bootstrap 范畴 - 运行时可编辑配置主要通过 `app_config` 表持久化 - token / secret 这类运行时必须可取回的配置，目前允许明文存储在 config 表中 - 登录密码仍然单独使用 Argon2 哈希，不走 config 表明文存储 @@ -318,55 +307,6 @@ docker compose -f docker-compose.yml up -d docker compose logs -f app ``` -## Grafana Provisioning - -当前仓库支持通过 Grafana provisioning 自动加载 SQLite datasource 和 repo 内的 dashboard 导出文件。 - -需要保留的文件路径如下： - -- `grafana/provisioning/datasources/locationrecorder.yaml` -- `grafana/provisioning/datasources/poorecorder.yaml` -- `grafana/provisioning/dashboards/provider.yaml` -- `grafana/dashboards/locationrecorder.json` -- `grafana/dashboards/poorecorder.json` - -这些文件的职责分别是： - -- `grafana/provisioning/datasources/locationrecorder.yaml`：声明 `locationrecorder` SQLite datasource，并指向 `/data/home-automation/locationRecorder.db` -- `grafana/provisioning/datasources/poorecorder.yaml`：声明 `poorecorder` SQLite datasource，并指向 `/data/home-automation/pooRecorder.db` -- `grafana/provisioning/dashboards/provider.yaml`：告诉 Grafana 从 `/var/lib/grafana/dashboards` 扫描并加载 dashboard JSON -- `grafana/dashboards/locationrecorder.json`：location recorder dashboard 导出文件，内容本身不需要在 compose 中改写 -- `grafana/dashboards/poorecorder.json`：poo recorder dashboard 导出文件，内容本身不需要在 compose 中改写 - -当前 `docker-compose.yml` 中，Grafana service 需要挂载以下目录： - -- `./grafana/provisioning -> /etc/grafana/provisioning:ro` -- `./grafana/dashboards -> /var/lib/grafana/dashboards:ro` - -同时保留现有 named volume `homeautomation_grafana_storage:/var/lib/grafana` 作为 Grafana 运行态数据存储。 - -一键启动前，至少需要以下文件已经存在： - -- `grafana/provisioning/datasources/locationrecorder.yaml` -- `grafana/provisioning/datasources/poorecorder.yaml` -- `grafana/provisioning/dashboards/provider.yaml` -- `grafana/dashboards/locationrecorder.json` -- `grafana/dashboards/poorecorder.json` - -启动方式： - -```bash -docker compose up -d -``` - -启动后会发生的事情： - -- Grafana 容器会安装 `frser-sqlite-datasource` 插件 -- Grafana 会读取 `/etc/grafana/provisioning/datasources/` 下的 datasource YAML -- Grafana 会读取 `/etc/grafana/provisioning/dashboards/provider.yaml` -- Grafana 会从 `/var/lib/grafana/dashboards/` 自动导入两个 dashboard JSON -- 现有 Grafana named volume 继续负责保存 Grafana 运行态数据，不会覆盖 repo 内的 dashboard 与 provisioning 文件 - ## Container Image CI 项目提供了一个 release image workflow： @@ -411,9 +351,16 @@ pytest 当前测试包含： -- app 基本启动测试 -- `/status` endpoint 测试 -- 登录 / session 基础流程测试 +- app 启动与 `/status` 检查 +- 登录 / session / 鉴权流程 +- runtime config 读写 +- public IPv4 monitor +- SMTP 配置与测试发信 +- location / poo recorder 端点 +- Home Assistant inbound 集成 +- TickTick OAuth +- 部署与迁移（`run_migrations`） +- legacy 数据迁移脚本（`migrate_legacy_data`） ## OpenAPI 导出 diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index c7c853c..41d7239 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -23,10 +23,8 @@ - 基础路由注册 - `config.py` - 环境变量驱动的 settings -- `auth_db.py` - - app 级共享 auth 数据库 - `db.py` - - SQLAlchemy engine / session / Base + - 统一数据层：一个 `Base`、一个绑定 `app_database_url` 的 cached engine（SQLite WAL）、`get_engine` / `get_session_local` / `reset_db_caches` / `get_db_session` - `dependencies.py` - 通用依赖注入 - `api/` @@ -37,7 +35,7 @@ - 当前已迁入 `POST /poo/record` 与 `GET /poo/latest` - `models/` - SQLAlchemy models - - 当前 `auth`、`location` 与 `poo` 使用各自独立的数据库 base + - 所有模型（auth / config / public_ip / location / poo）共用同一个 `Base`，均落在单一 `app.db` 中 - `schemas/` - Pydantic schemas - `services/` @@ -53,17 +51,9 @@ - `static/` - 极简静态资源 -### `alembic_location/` - -Location DB 的 migration 基础设施。 - ### `alembic_app/` -App DB 的 migration 基础设施。 - -### `alembic_poo/` - -Poo DB 的 migration 基础设施。 +App DB 的唯一 Alembic migration 链，同时管理 `location` / `poo_records` 表。M1 将三个独立 DB 合并进 `app.db` 后，`alembic_location/` 与 `alembic_poo/` 已退役，全部由此链统一管理。 ### `tests/` diff --git a/docs/roadmap.md b/docs/roadmap.md index fb36116..36771e6 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -34,7 +34,7 @@ | 里程碑 | 主题 | 一句话 | | --- | --- | --- | -| **M1** | 单库化地基 | 把三库合并成单一 `app.db`，清理散落数据层，删掉 Grafana | +| **M1** ✅ | 单库化地基 | 把三库合并成单一 `app.db`，清理散落数据层，删掉 Grafana | | **M2** | 前端 v2 | React SPA 取代 Jinja，承载 config + 可视化 + 记录增删改 | | **M3** | 开放与移动端（远期试水） | token 鉴权 + React Native 移动端 | @@ -42,7 +42,7 @@ --- -## M1 — 单库化地基 +## M1 — 单库化地基（✅ 已完成） ### 目标 diff --git a/openapi/openapi.json b/openapi/openapi.json index b03671e..9465aa3 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -249,6 +249,27 @@ } } }, + "/config/smtp/test": { + "post": { + "tags": [ + "pages" + ], + "summary": "Smtp Test Submit", + "operationId": "smtp_test_submit_config_smtp_test_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "text/html": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/homeassistant/publish": { "post": { "tags": [ @@ -325,6 +346,27 @@ } } }, + "/public-ip/check": { + "get": { + "tags": [ + "public-ip" + ], + "summary": "Run Public Ip Check", + "operationId": "run_public_ip_check_public_ip_check_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PublicIPCheckResponse" + } + } + } + } + } + } + }, "/ticktick/auth/start": { "get": { "tags": [ @@ -443,6 +485,36 @@ "type": "object", "title": "HTTPValidationError" }, + "PublicIPCheckResponse": { + "properties": { + "status": { + "type": "string", + "enum": [ + "first_seen", + "unchanged", + "changed", + "error" + ], + "title": "Status" + }, + "checked_at": { + "type": "string", + "format": "date-time", + "title": "Checked At" + }, + "changed": { + "type": "boolean", + "title": "Changed" + } + }, + "type": "object", + "required": [ + "status", + "checked_at", + "changed" + ], + "title": "PublicIPCheckResponse" + }, "StatusResponse": { "properties": { "status": { diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index b0dde44..a091b27 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -155,6 +155,19 @@ paths: text/html: schema: type: string + /config/smtp/test: + post: + tags: + - pages + summary: Smtp Test Submit + operationId: smtp_test_submit_config_smtp_test_post + responses: + '200': + description: Successful Response + content: + text/html: + schema: + type: string /homeassistant/publish: post: tags: @@ -203,6 +216,19 @@ paths: content: application/json: schema: {} + /public-ip/check: + get: + tags: + - public-ip + summary: Run Public Ip Check + operationId: run_public_ip_check_public_ip_check_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PublicIPCheckResponse' /ticktick/auth/start: get: tags: @@ -285,6 +311,29 @@ components: title: Detail type: object title: HTTPValidationError + PublicIPCheckResponse: + properties: + status: + type: string + enum: + - first_seen + - unchanged + - changed + - error + title: Status + checked_at: + type: string + format: date-time + title: Checked At + changed: + type: boolean + title: Changed + type: object + required: + - status + - checked_at + - changed + title: PublicIPCheckResponse StatusResponse: properties: status: From 1cbe6c46d216975c7b9d86a41d9dca737bddd5b2 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 19:05:56 +0200 Subject: [PATCH 8/9] M1-rework: harden legacy-migration reconciliation to full-row equality Audit finding (review-notes/M1-full-review-1.md, FINDING 1): _reconcile only checked primary-key presence, so a source row skipped by INSERT OR IGNORE due to a value difference against a pre-existing same-PK target row would false-pass. Compare ALL columns with SQLite's NULL-safe IS operator instead, so reconciliation is a true full-row guarantee (idempotent re-runs still pass because the rows match column-for-column). Add tests for the value-mismatch abort and for idempotency under full-row reconciliation. Remove the now-unused pk_cols parameter. pytest 97 passed; ruff clean (pre-existing only); data-safety grep still empty. --- scripts/migrate_legacy_data.py | 24 ++++++------- tests/test_migrate_legacy_data.py | 59 ++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/scripts/migrate_legacy_data.py b/scripts/migrate_legacy_data.py index d36bfae..5cd675f 100644 --- a/scripts/migrate_legacy_data.py +++ b/scripts/migrate_legacy_data.py @@ -53,17 +53,20 @@ def _sqlite_path_from_url(url: str) -> Path: def _reconcile( conn: sqlite3.Connection, table: str, - pk_cols: list[str], + columns: list[str], source_count: int, ) -> int: """Verify every legacy source row is present in the main (app) table. - Returns the count of source rows present in main. - Raises RuntimeError if any rows are missing. + Matches on ALL columns using SQLite's NULL-safe IS operator so that nullable + columns (e.g. altitude) compare correctly. A row that was silently skipped + by INSERT OR IGNORE due to a value difference will NOT satisfy this predicate + even if its primary key is present in the target. + + Returns the count of source rows whose full-row data is present in main. + Raises RuntimeError if any rows are missing or differ in value. """ - join_cond = " AND ".join( - f"m.{col} = l.{col}" for col in pk_cols - ) + join_cond = " AND ".join(f"m.{col} IS l.{col}" for col in columns) sql = ( f"SELECT COUNT(*) FROM legacy.{table} l " f"WHERE EXISTS (SELECT 1 FROM main.{table} m WHERE {join_cond})" @@ -73,7 +76,7 @@ def _reconcile( missing = source_count - present raise RuntimeError( f"Reconciliation failed for table '{table}': " - f"{missing} of {source_count} source rows are missing from the app DB." + f"{missing} of {source_count} source rows are missing or differing in the app DB." ) return present @@ -114,7 +117,6 @@ def migrate_legacy_data( legacy_url=location_url, table="location", columns=["person", "datetime", "latitude", "longitude", "altitude"], - pk_cols=["person", "datetime"], dry_run=dry_run, ) @@ -124,7 +126,6 @@ def migrate_legacy_data( legacy_url=poo_url, table="poo_records", columns=["timestamp", "status", "latitude", "longitude"], - pk_cols=["timestamp"], dry_run=dry_run, ) @@ -137,7 +138,6 @@ def _migrate_table( legacy_url: str | None, table: str, columns: list[str], - pk_cols: list[str], dry_run: bool, ) -> dict: """Migrate a single table from a legacy DB into the app DB. @@ -186,8 +186,8 @@ def _migrate_table( (after_count,) = conn.execute(f"SELECT COUNT(*) FROM main.{table}").fetchone() copied = after_count - before_count - # Reconciliation: every source row must be present - _reconcile(conn, table, pk_cols, source_count) + # Reconciliation: every source row must be present with matching values + _reconcile(conn, table, columns, source_count) conn.execute("DETACH DATABASE legacy") finally: diff --git a/tests/test_migrate_legacy_data.py b/tests/test_migrate_legacy_data.py index bd6fa47..88dedd5 100644 --- a/tests/test_migrate_legacy_data.py +++ b/tests/test_migrate_legacy_data.py @@ -261,7 +261,7 @@ def test_reconcile_raises_on_missing_rows(tmp_path: Path) -> None: _reconcile( conn, table="location", - pk_cols=["person", "datetime"], + columns=["person", "datetime", "latitude", "longitude", "altitude"], source_count=len(LOCATION_ROWS), ) conn.execute("DETACH DATABASE legacy") @@ -326,6 +326,63 @@ def test_cli_exits_nonzero_on_reconciliation_failure( assert exc_info.value.code != 0 +def test_reconcile_catches_value_mismatch_not_just_pk(tmp_path: Path) -> None: + """Full-row reconciliation catches value mismatch that PK-only check would miss. + + Scenario: the app DB is PRE-POPULATED with a row that shares the same PK as + a legacy source row but has DIFFERENT non-PK column values. INSERT OR IGNORE + skips the source row (PK conflict), so the target retains the stale data. + The old PK-only reconciliation would have incorrectly reported success. + The new full-row reconciliation must detect the mismatch and raise. + """ + app_path, app_url = _upgraded_app_db(tmp_path) + + # Legacy source has a row: person="alice", datetime="2026-01-01T10:00:00Z", + # latitude=1.23, longitude=4.56, altitude=7.89 + legacy_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(legacy_path, [("alice", "2026-01-01T10:00:00Z", 1.23, 4.56, 7.89)]) + legacy_url = f"sqlite:///{legacy_path}" + + # App DB is pre-populated with the SAME PK but DIFFERENT non-PK values + # (latitude/longitude/altitude all differ from the source row) + conn = sqlite3.connect(app_path) + conn.execute( + "INSERT INTO location (person, datetime, latitude, longitude, altitude) " + "VALUES (?, ?, ?, ?, ?)", + ("alice", "2026-01-01T10:00:00Z", 99.0, 99.0, 99.0), + ) + conn.commit() + conn.close() + + # migrate_legacy_data must raise: the source row's data is NOT in the target + # (INSERT OR IGNORE skipped it because of PK conflict, retaining the 99.0 values) + with pytest.raises(RuntimeError, match="Reconciliation failed"): + migrate_legacy_data(app_url, legacy_url, None) + + +def test_full_row_reconciliation_idempotent_on_identical_data(tmp_path: Path) -> None: + """Second run on already-migrated data still reconciles cleanly. + + When the target already holds identical rows (from the first run), the full-row + IS predicate matches every column and reconciliation passes (no raise). + """ + app_path, app_url = _upgraded_app_db(tmp_path) + legacy_path = tmp_path / "locationRecorder.db" + _make_legacy_location_db(legacy_path, LOCATION_ROWS) + legacy_url = f"sqlite:///{legacy_path}" + + # First run: migrate all rows + result1 = migrate_legacy_data(app_url, legacy_url, None) + assert result1["location"]["copied"] == len(LOCATION_ROWS) + + # Second run: rows already present, INSERT OR IGNORE skips all, full-row + # reconciliation must still pass because values are identical + result2 = migrate_legacy_data(app_url, legacy_url, None) + assert result2["location"]["copied"] == 0 + assert result2["location"]["final"] == len(LOCATION_ROWS) + # No exception raised — idempotency holds under full-row reconciliation + + # --------------------------------------------------------------------------- # Test 4: dry_run # --------------------------------------------------------------------------- From a337b06c94ffd2567d65eab14d9295385db0deb9 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 12 Jun 2026 19:11:13 +0200 Subject: [PATCH 9/9] M1-rework: rename leftover pk_cols param in reconciliation test stubs Round-2 audit nit (review-notes/M1-full-review-2.md): two _always_fail monkeypatch stubs still named their (ignored) third positional parameter pk_cols after _reconcile switched to a full columns list. Rename to columns for consistency. Test-only, no behavior change. pytest 97 passed; ruff clean. --- tests/test_migrate_legacy_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_migrate_legacy_data.py b/tests/test_migrate_legacy_data.py index 88dedd5..ca24b53 100644 --- a/tests/test_migrate_legacy_data.py +++ b/tests/test_migrate_legacy_data.py @@ -277,7 +277,7 @@ def test_migrate_reconciliation_failure_raises(tmp_path: Path, monkeypatch: pyte _make_legacy_location_db(legacy_path, LOCATION_ROWS) legacy_url = f"sqlite:///{legacy_path}" - def _always_fail(conn, table, pk_cols, source_count): + def _always_fail(conn, table, columns, source_count): # Simulate a scenario where reconciliation finds rows missing raise RuntimeError( f"Reconciliation failed for table '{table}': " @@ -302,7 +302,7 @@ def test_cli_exits_nonzero_on_reconciliation_failure( legacy_url = f"sqlite:///{legacy_path}" # Patch _reconcile to always raise - def _always_fail(conn, table, pk_cols, source_count): + def _always_fail(conn, table, columns, source_count): raise RuntimeError( f"Reconciliation failed for table '{table}': 1 row missing." )