Merge pull request 'change adoption to separate step' (#4) from feature/add_separate_migration_container into main

Reviewed-on: #4

Dockerfile

@@ -23,3 +23,4 @@ RUN mkdir -p /app/data
 EXPOSE 8000
 
 ENTRYPOINT ["/app/docker/entrypoint.sh"]
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -107,9 +107,7 @@ cp .env.example .env
 3. 初始化数据库
 
 ```bash
-python scripts/app_db_adopt.py
-python scripts/location_db_adopt.py
-python scripts/poo_db_adopt.py
+python -m scripts.run_migrations
 ```
 
 4. 启动服务
@@ -141,6 +139,7 @@ uvicorn app.main:app --reload --host 0.0.0.0 --port 8000
 - App Alembic 环境：`alembic_app.ini` + `alembic_app/`
 - Location Alembic 环境：`alembic_location.ini` + `alembic_location/`
 - Poo Alembic 环境：`alembic_poo.ini` + `alembic_poo/`
+- 统一 migration job：`python -m scripts.run_migrations`
 - App DB 初始化：`python scripts/app_db_adopt.py`
 - Location DB 接管 / 初始化：`python scripts/location_db_adopt.py`
 - Poo DB 接管 / 初始化：`python scripts/poo_db_adopt.py`

docker-compose.override.yml

@@ -1,3 +1,6 @@
 services:
+  migration:
+    build: .
+
   app:
     build: .

docker-compose.yml

@@ -1,10 +1,24 @@
 services:
+  migration:
+    container_name: home-automation-migration
+    image: code.wanderingbadger.dev/tliu93/home-automation:latest
+    user: "1000:1000"
+    restart: "no"
+    init: true
+    command: ["python", "-m", "scripts.run_migrations"]
+    volumes:
+      - ./data:/app/data
+      - ./.env:/app/.env:ro
+
   app:
     container_name: home-automation-app
     image: code.wanderingbadger.dev/tliu93/home-automation:latest
     user: "1000:1000"
     restart: unless-stopped
     init: true
+    depends_on:
+      migration:
+        condition: service_completed_successfully
     ports:
       - "127.0.0.1:8881:8000"
     volumes:

docker/entrypoint.sh

@@ -2,8 +2,4 @@
 
 set -eu
 
-python scripts/app_db_adopt.py
-python scripts/location_db_adopt.py
-python scripts/poo_db_adopt.py
-
-exec uvicorn app.main:app --host 0.0.0.0 --port 8000
+exec "$@"

scripts/app_db_adopt.py

@@ -6,6 +6,8 @@ from pathlib import Path
 
 from alembic import command
 from alembic.config import Config
+from alembic.script import ScriptDirectory
+from alembic.util.exc import CommandError
 
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
 if str(PROJECT_ROOT) not in sys.path:
@@ -35,6 +37,24 @@ def _make_alembic_config(database_url: str) -> Config:
     return config
 
 
+def _expected_head_revision(alembic_config: Config) -> str:
+    script = ScriptDirectory.from_config(alembic_config)
+    heads = script.get_heads()
+    if len(heads) != 1:
+        raise AppDatabaseAdoptionError(
+            f"Expected exactly one Alembic head for app DB, got {len(heads)}"
+        )
+    return heads[0]
+
+
+def _is_known_revision(alembic_config: Config, revision: str) -> bool:
+    script = ScriptDirectory.from_config(alembic_config)
+    try:
+        return script.get_revision(revision) is not None
+    except CommandError:
+        return False
+
+
 def _alembic_version_table_exists(database_path: Path) -> bool:
     conn = sqlite3.connect(database_path)
     try:
@@ -75,6 +95,8 @@ def _list_user_tables(database_path: Path) -> list[str]:
 
 def validate_app_runtime_db(database_url: str) -> None:
     database_path = _database_path_from_url(database_url)
+    alembic_config = _make_alembic_config(database_url)
+    expected_revision = _expected_head_revision(alembic_config)
     if not database_path.exists():
         raise AppDatabaseAdoptionError(
             "App DB file was not found. Run 'python scripts/app_db_adopt.py' first to "
@@ -88,22 +110,28 @@ def validate_app_runtime_db(database_url: str) -> None:
         )
 
     current_revision = _fetch_alembic_revision(database_path)
-    if current_revision != APP_BASELINE_REVISION:
+    if current_revision != expected_revision:
         raise AppDatabaseAdoptionError(
             "App DB revision mismatch. Refusing to start the app: "
-            f"expected {APP_BASELINE_REVISION}, got {current_revision}"
+            f"expected {expected_revision}, got {current_revision}"
         )
 
 
 def adopt_or_initialize_app_db(database_url: str) -> str:
     database_path = _database_path_from_url(database_url)
     alembic_config = _make_alembic_config(database_url)
+    expected_revision = _expected_head_revision(alembic_config)
 
     if database_path.exists():
         if _alembic_version_table_exists(database_path):
             current_revision = _fetch_alembic_revision(database_path)
-            if current_revision == APP_BASELINE_REVISION:
+            if current_revision == expected_revision:
                 return "already_managed"
+            if not _is_known_revision(alembic_config, current_revision):
+                raise AppDatabaseAdoptionError(
+                    "App DB is already Alembic-managed but revision does not match "
+                    f"a known migration revision: got {current_revision}"
+                )
             command.upgrade(alembic_config, "head")
             return "upgraded"
 

scripts/location_db_adopt.py

@@ -6,6 +6,8 @@ from pathlib import Path
 
 from alembic import command
 from alembic.config import Config
+from alembic.script import ScriptDirectory
+from alembic.util.exc import CommandError
 
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
 if str(PROJECT_ROOT) not in sys.path:
@@ -43,6 +45,24 @@ def _make_alembic_config(database_url: str) -> Config:
     return config
 
 
+def _expected_head_revision(alembic_config: Config) -> str:
+    script = ScriptDirectory.from_config(alembic_config)
+    heads = script.get_heads()
+    if len(heads) != 1:
+        raise LocationDatabaseAdoptionError(
+            f"Expected exactly one Alembic head for location DB, got {len(heads)}"
+        )
+    return heads[0]
+
+
+def _is_known_revision(alembic_config: Config, revision: str) -> bool:
+    script = ScriptDirectory.from_config(alembic_config)
+    try:
+        return script.get_revision(revision) is not None
+    except CommandError:
+        return False
+
+
 def _location_table_exists(database_path: Path) -> bool:
     conn = sqlite3.connect(database_path)
     try:
@@ -117,6 +137,8 @@ def validate_legacy_location_db(database_url: str) -> None:
 
 def validate_location_runtime_db(database_url: str) -> None:
     database_path = _database_path_from_url(database_url)
+    alembic_config = _make_alembic_config(database_url)
+    expected_revision = _expected_head_revision(alembic_config)
     if not database_path.exists():
         raise LocationDatabaseAdoptionError(
             "Location DB file was not found. Run 'python scripts/location_db_adopt.py' "
@@ -131,30 +153,36 @@ def validate_location_runtime_db(database_url: str) -> None:
         )
 
     current_revision = _fetch_alembic_revision(database_path)
-    if current_revision != LOCATION_BASELINE_REVISION:
+    if current_revision != expected_revision:
         raise LocationDatabaseAdoptionError(
             "Location DB revision mismatch. Refusing to start the app: "
-            f"expected {LOCATION_BASELINE_REVISION}, got {current_revision}"
+            f"expected {expected_revision}, got {current_revision}"
         )
 
 
 def adopt_or_initialize_location_db(database_url: str) -> str:
     database_path = _database_path_from_url(database_url)
     alembic_config = _make_alembic_config(database_url)
+    expected_revision = _expected_head_revision(alembic_config)
 
     if database_path.exists():
         if _alembic_version_table_exists(database_path):
             current_revision = _fetch_alembic_revision(database_path)
-            if current_revision != LOCATION_BASELINE_REVISION:
+            if current_revision == expected_revision:
+                return "already_managed"
+            if not _is_known_revision(alembic_config, current_revision):
                 raise LocationDatabaseAdoptionError(
                     "Location DB is already Alembic-managed but revision does not match "
-                    f"the expected baseline: expected {LOCATION_BASELINE_REVISION}, "
-                    f"got {current_revision}"
+                    f"a known migration revision: got {current_revision}"
                 )
-            return "already_managed"
+            command.upgrade(alembic_config, "head")
+            return "upgraded"
 
         validate_legacy_location_db(database_url)
         command.stamp(alembic_config, LOCATION_BASELINE_REVISION)
+        if LOCATION_BASELINE_REVISION != expected_revision:
+            command.upgrade(alembic_config, "head")
+            return "upgraded"
         return "adopted"
 
     database_path.parent.mkdir(parents=True, exist_ok=True)

scripts/poo_db_adopt.py

@@ -6,6 +6,8 @@ from pathlib import Path
 
 from alembic import command
 from alembic.config import Config
+from alembic.script import ScriptDirectory
+from alembic.util.exc import CommandError
 
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
 if str(PROJECT_ROOT) not in sys.path:
@@ -42,6 +44,24 @@ def _make_alembic_config(database_url: str) -> Config:
     return config
 
 
+def _expected_head_revision(alembic_config: Config) -> str:
+    script = ScriptDirectory.from_config(alembic_config)
+    heads = script.get_heads()
+    if len(heads) != 1:
+        raise PooDatabaseAdoptionError(
+            f"Expected exactly one Alembic head for poo DB, got {len(heads)}"
+        )
+    return heads[0]
+
+
+def _is_known_revision(alembic_config: Config, revision: str) -> bool:
+    script = ScriptDirectory.from_config(alembic_config)
+    try:
+        return script.get_revision(revision) is not None
+    except CommandError:
+        return False
+
+
 def _poo_table_exists(database_path: Path) -> bool:
     conn = sqlite3.connect(database_path)
     try:
@@ -112,6 +132,8 @@ def validate_legacy_poo_db(database_url: str) -> None:
 
 def validate_poo_runtime_db(database_url: str) -> None:
     database_path = _database_path_from_url(database_url)
+    alembic_config = _make_alembic_config(database_url)
+    expected_revision = _expected_head_revision(alembic_config)
     if not database_path.exists():
         raise PooDatabaseAdoptionError(
             "Poo DB file was not found. Run 'python scripts/poo_db_adopt.py' first to "
@@ -126,30 +148,36 @@ def validate_poo_runtime_db(database_url: str) -> None:
         )
 
     current_revision = _fetch_alembic_revision(database_path)
-    if current_revision != POO_BASELINE_REVISION:
+    if current_revision != expected_revision:
         raise PooDatabaseAdoptionError(
             "Poo DB revision mismatch. Refusing to start the app: "
-            f"expected {POO_BASELINE_REVISION}, got {current_revision}"
+            f"expected {expected_revision}, got {current_revision}"
         )
 
 
 def adopt_or_initialize_poo_db(database_url: str) -> str:
     database_path = _database_path_from_url(database_url)
     alembic_config = _make_alembic_config(database_url)
+    expected_revision = _expected_head_revision(alembic_config)
 
     if database_path.exists():
         if _alembic_version_table_exists(database_path):
             current_revision = _fetch_alembic_revision(database_path)
-            if current_revision != POO_BASELINE_REVISION:
+            if current_revision == expected_revision:
+                return "already_managed"
+            if not _is_known_revision(alembic_config, current_revision):
                 raise PooDatabaseAdoptionError(
                     "Poo DB is already Alembic-managed but revision does not match "
-                    f"the expected baseline: expected {POO_BASELINE_REVISION}, "
-                    f"got {current_revision}"
+                    f"a known migration revision: got {current_revision}"
                 )
-            return "already_managed"
+            command.upgrade(alembic_config, "head")
+            return "upgraded"
 
         validate_legacy_poo_db(database_url)
         command.stamp(alembic_config, POO_BASELINE_REVISION)
+        if POO_BASELINE_REVISION != expected_revision:
+            command.upgrade(alembic_config, "head")
+            return "upgraded"
         return "adopted"
 
     database_path.parent.mkdir(parents=True, exist_ok=True)

scripts/run_migrations.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from app.config import get_settings
+from scripts.app_db_adopt import adopt_or_initialize_app_db
+from scripts.location_db_adopt import adopt_or_initialize_location_db
+from scripts.poo_db_adopt import adopt_or_initialize_poo_db
+
+
+def run_all_migrations() -> dict[str, str]:
+    settings = get_settings()
+    return {
+        "app": adopt_or_initialize_app_db(settings.app_database_url),
+        "location": adopt_or_initialize_location_db(settings.location_database_url),
+        "poo": adopt_or_initialize_poo_db(settings.poo_database_url),
+    }
+
+
+def main() -> None:
+    results = run_all_migrations()
+    for database_name, result in results.items():
+        print(f"{database_name}: {result}")
+
+
+if __name__ == "__main__":
+    main()

tests/test_app.py

@@ -37,12 +37,13 @@ def test_status_endpoint(client: TestClient) -> None:
 
 
 def test_app_start_fails_when_app_db_missing(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
+    missing_app_path = tmp_path / "missing_app.db"
     poo_database_path = tmp_path / "poo_ready.db"
     location_database_path = tmp_path / "location_ready.db"
     command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head")
     command.upgrade(_make_alembic_config(f"sqlite:///{location_database_path}"), "head")
 
-    monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{tmp_path / 'missing_app.db'}")
+    monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{missing_app_path}")
     monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin")
     monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password")
     monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}")
@@ -54,6 +55,8 @@ def test_app_start_fails_when_app_db_missing(tmp_path, monkeypatch: pytest.Monke
     with pytest.raises(RuntimeError, match="Run 'python scripts/app_db_adopt.py' first"):
         anyio.run(_run_lifespan, app)
 
+    assert not missing_app_path.exists()
+
     get_settings.cache_clear()
     reset_auth_db_caches()
 

tests/test_deployment.py

@@ -0,0 +1,213 @@
+from pathlib import Path
+import sqlite3
+
+import anyio
+import pytest
+import yaml
+from alembic import command
+
+from app.auth_db import reset_auth_db_caches
+from app.config import get_settings
+from app.main import create_app
+from scripts.app_db_adopt import APP_BASELINE_REVISION
+from scripts.location_db_adopt import EXPECTED_USER_VERSION as LOCATION_USER_VERSION
+from scripts.location_db_adopt import LOCATION_BASELINE_REVISION
+from scripts.poo_db_adopt import EXPECTED_USER_VERSION as POO_USER_VERSION
+from scripts.poo_db_adopt import POO_BASELINE_REVISION
+from scripts.run_migrations import run_all_migrations
+from tests.conftest import _make_alembic_config, _make_poo_alembic_config
+
+
+def _read_yaml(path: str) -> dict:
+    return yaml.safe_load(Path(path).read_text())
+
+
+async def _run_lifespan(app) -> None:
+    async with app.router.lifespan_context(app):
+        return None
+
+
+def _configure_database_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path | str]:
+    app_path = tmp_path / "app.db"
+    location_path = tmp_path / "location.db"
+    poo_path = tmp_path / "poo.db"
+
+    monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{app_path}")
+    monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_path}")
+    monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_path}")
+    monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin")
+    monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password")
+    monkeypatch.setenv("AUTH_COOKIE_SECURE_OVERRIDE", "false")
+    get_settings.cache_clear()
+    reset_auth_db_caches()
+
+    return {
+        "app_path": app_path,
+        "app_url": f"sqlite:///{app_path}",
+        "location_path": location_path,
+        "location_url": f"sqlite:///{location_path}",
+        "poo_path": poo_path,
+        "poo_url": f"sqlite:///{poo_path}",
+    }
+
+
+def _create_legacy_location_db(database_path: Path) -> None:
+    conn = sqlite3.connect(database_path)
+    conn.execute(
+        """
+        CREATE TABLE location (
+          person TEXT NOT NULL,
+          datetime TEXT NOT NULL,
+          latitude REAL NOT NULL,
+          longitude REAL NOT NULL,
+          altitude REAL,
+          PRIMARY KEY (person, datetime)
+        )
+        """
+    )
+    conn.execute(
+        "INSERT INTO location (person, datetime, latitude, longitude, altitude) VALUES (?, ?, ?, ?, ?)",
+        ("alice", "2026-04-22T10:00:00Z", 1.23, 4.56, 7.89),
+    )
+    conn.execute(f"PRAGMA user_version = {LOCATION_USER_VERSION}")
+    conn.commit()
+    conn.close()
+
+
+def _create_legacy_poo_db(database_path: Path) -> None:
+    conn = sqlite3.connect(database_path)
+    conn.execute(
+        """
+        CREATE TABLE poo_records (
+          timestamp TEXT NOT NULL,
+          status TEXT NOT NULL,
+          latitude REAL NOT NULL,
+          longitude REAL NOT NULL,
+          PRIMARY KEY (timestamp)
+        )
+        """
+    )
+    conn.execute(
+        "INSERT INTO poo_records (timestamp, status, latitude, longitude) VALUES (?, ?, ?, ?)",
+        ("2026-04-22T11:00:00Z", "complete", 9.87, 6.54),
+    )
+    conn.execute(f"PRAGMA user_version = {POO_USER_VERSION}")
+    conn.commit()
+    conn.close()
+
+
+def test_compose_uses_migration_job_before_app() -> None:
+    compose = _read_yaml("/home/tianyu/workspace/home-automation/docker-compose.yml")
+    override = _read_yaml("/home/tianyu/workspace/home-automation/docker-compose.override.yml")
+
+    migration_service = compose["services"]["migration"]
+    app_service = compose["services"]["app"]
+
+    assert migration_service["command"] == ["python", "-m", "scripts.run_migrations"]
+    assert migration_service["restart"] == "no"
+    assert app_service["depends_on"]["migration"]["condition"] == "service_completed_successfully"
+    assert override["services"]["migration"]["build"] == "."
+    assert override["services"]["app"]["build"] == "."
+
+
+def test_image_defaults_to_uvicorn_only() -> None:
+    dockerfile = Path("/home/tianyu/workspace/home-automation/Dockerfile").read_text()
+    entrypoint = Path("/home/tianyu/workspace/home-automation/docker/entrypoint.sh").read_text()
+
+    assert 'CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]' in dockerfile
+    assert 'exec "$@"' in entrypoint
+    assert "app_db_adopt" not in entrypoint
+    assert "location_db_adopt" not in entrypoint
+    assert "poo_db_adopt" not in entrypoint
+
+
+def test_migration_runner_initializes_and_is_idempotent(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    database_urls = _configure_database_env(tmp_path, monkeypatch)
+
+    first_run = run_all_migrations()
+    second_run = run_all_migrations()
+
+    assert first_run == {"app": "initialized", "location": "initialized", "poo": "initialized"}
+    assert second_run == {
+        "app": "already_managed",
+        "location": "already_managed",
+        "poo": "already_managed",
+    }
+
+    conn = sqlite3.connect(database_urls["app_path"])
+    try:
+        assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == APP_BASELINE_REVISION
+        tables = {
+            row[0]
+            for row in conn.execute(
+                "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
+            ).fetchall()
+        }
+    finally:
+        conn.close()
+
+    assert {"auth_users", "auth_sessions", "app_config", "alembic_version"} <= tables
+
+    conn = sqlite3.connect(database_urls["location_path"])
+    try:
+        assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == LOCATION_BASELINE_REVISION
+    finally:
+        conn.close()
+
+    conn = sqlite3.connect(database_urls["poo_path"])
+    try:
+        assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == POO_BASELINE_REVISION
+    finally:
+        conn.close()
+
+    get_settings.cache_clear()
+    reset_auth_db_caches()
+
+
+def test_migration_runner_adopts_legacy_sqlite_without_data_loss(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    database_urls = _configure_database_env(tmp_path, monkeypatch)
+    _create_legacy_location_db(database_urls["location_path"])
+    _create_legacy_poo_db(database_urls["poo_path"])
+
+    results = run_all_migrations()
+
+    assert results == {"app": "initialized", "location": "adopted", "poo": "adopted"}
+
+    conn = sqlite3.connect(database_urls["location_path"])
+    try:
+        assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == LOCATION_BASELINE_REVISION
+        assert conn.execute("SELECT COUNT(*) FROM location").fetchone()[0] == 1
+    finally:
+        conn.close()
+
+    conn = sqlite3.connect(database_urls["poo_path"])
+    try:
+        assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == POO_BASELINE_REVISION
+        assert conn.execute("SELECT COUNT(*) FROM poo_records").fetchone()[0] == 1
+    finally:
+        conn.close()
+
+    get_settings.cache_clear()
+    reset_auth_db_caches()
+
+
+def test_app_startup_still_fails_closed_without_running_adoption(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    database_urls = _configure_database_env(tmp_path, monkeypatch)
+    missing_app_path = database_urls["app_path"]
+    command.upgrade(_make_alembic_config(database_urls["location_url"]), "head")
+    command.upgrade(_make_poo_alembic_config(database_urls["poo_url"]), "head")
+
+    app = create_app()
+    with pytest.raises(RuntimeError, match="Run 'python scripts/app_db_adopt.py' first"):
+        anyio.run(_run_lifespan, app)
+
+    assert not Path(missing_app_path).exists()
+
+    get_settings.cache_clear()
+    reset_auth_db_caches()

tests/test_location.py

@@ -343,7 +343,7 @@ def test_location_db_adoption_fails_closed_on_alembic_revision_mismatch(
     conn.commit()
     conn.close()
 
-    with pytest.raises(LocationDatabaseAdoptionError, match="revision does not match"):
+    with pytest.raises(LocationDatabaseAdoptionError, match="known migration revision"):
         adopt_or_initialize_location_db(f"sqlite:///{database_path}")