From a76d6bfb71b8e2ecb93da3afa4489492883b753d Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Wed, 22 Apr 2026 13:28:00 +0200 Subject: [PATCH] change adoption to separate step --- Dockerfile | 1 + README.md | 5 +- docker-compose.override.yml | 3 + docker-compose.yml | 14 +++ docker/entrypoint.sh | 6 +- scripts/app_db_adopt.py | 34 +++++- scripts/location_db_adopt.py | 40 ++++++- scripts/poo_db_adopt.py | 40 ++++++- scripts/run_migrations.py | 25 ++++ tests/test_app.py | 5 +- tests/test_deployment.py | 213 +++++++++++++++++++++++++++++++++++ tests/test_location.py | 2 +- 12 files changed, 363 insertions(+), 25 deletions(-) create mode 100644 scripts/run_migrations.py create mode 100644 tests/test_deployment.py diff --git a/Dockerfile b/Dockerfile index d760173..26479da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,3 +23,4 @@ RUN mkdir -p /app/data EXPOSE 8000 ENTRYPOINT ["/app/docker/entrypoint.sh"] +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index 2c27cb8..dc5109b 100644 --- a/README.md +++ b/README.md @@ -107,9 +107,7 @@ cp .env.example .env 3. 初始化数据库 ```bash -python scripts/app_db_adopt.py -python scripts/location_db_adopt.py -python scripts/poo_db_adopt.py +python -m scripts.run_migrations ``` 4. 启动服务 @@ -141,6 +139,7 @@ uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 - App Alembic 环境：`alembic_app.ini` + `alembic_app/` - Location Alembic 环境：`alembic_location.ini` + `alembic_location/` - Poo Alembic 环境：`alembic_poo.ini` + `alembic_poo/` +- 统一 migration job：`python -m scripts.run_migrations` - App DB 初始化：`python scripts/app_db_adopt.py` - Location DB 接管 / 初始化：`python scripts/location_db_adopt.py` - Poo DB 接管 / 初始化：`python scripts/poo_db_adopt.py` diff --git a/docker-compose.override.yml b/docker-compose.override.yml index bae39b4..78f2dd7 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,3 +1,6 @@ services: + migration: + build: . + app: build: . \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 9035790..bc79bcd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,24 @@ services: + migration: + container_name: home-automation-migration + image: code.wanderingbadger.dev/tliu93/home-automation:latest + user: "1000:1000" + restart: "no" + init: true + command: ["python", "-m", "scripts.run_migrations"] + volumes: + - ./data:/app/data + - ./.env:/app/.env:ro + app: container_name: home-automation-app image: code.wanderingbadger.dev/tliu93/home-automation:latest user: "1000:1000" restart: unless-stopped init: true + depends_on: + migration: + condition: service_completed_successfully ports: - "127.0.0.1:8881:8000" volumes: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e69c8d1..06da036 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,8 +2,4 @@ set -eu -python scripts/app_db_adopt.py -python scripts/location_db_adopt.py -python scripts/poo_db_adopt.py - -exec uvicorn app.main:app --host 0.0.0.0 --port 8000 \ No newline at end of file +exec "$@" \ No newline at end of file diff --git a/scripts/app_db_adopt.py b/scripts/app_db_adopt.py index 3979f54..0e88286 100644 --- a/scripts/app_db_adopt.py +++ b/scripts/app_db_adopt.py @@ -6,6 +6,8 @@ from pathlib import Path from alembic import command from alembic.config import Config +from alembic.script import ScriptDirectory +from alembic.util.exc import CommandError PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: @@ -35,6 +37,24 @@ def _make_alembic_config(database_url: str) -> Config: return config +def _expected_head_revision(alembic_config: Config) -> str: + script = ScriptDirectory.from_config(alembic_config) + heads = script.get_heads() + if len(heads) != 1: + raise AppDatabaseAdoptionError( + f"Expected exactly one Alembic head for app DB, got {len(heads)}" + ) + return heads[0] + + +def _is_known_revision(alembic_config: Config, revision: str) -> bool: + script = ScriptDirectory.from_config(alembic_config) + try: + return script.get_revision(revision) is not None + except CommandError: + return False + + def _alembic_version_table_exists(database_path: Path) -> bool: conn = sqlite3.connect(database_path) try: @@ -75,6 +95,8 @@ def _list_user_tables(database_path: Path) -> list[str]: def validate_app_runtime_db(database_url: str) -> None: database_path = _database_path_from_url(database_url) + alembic_config = _make_alembic_config(database_url) + expected_revision = _expected_head_revision(alembic_config) if not database_path.exists(): raise AppDatabaseAdoptionError( "App DB file was not found. Run 'python scripts/app_db_adopt.py' first to " @@ -88,22 +110,28 @@ def validate_app_runtime_db(database_url: str) -> None: ) current_revision = _fetch_alembic_revision(database_path) - if current_revision != APP_BASELINE_REVISION: + if current_revision != expected_revision: raise AppDatabaseAdoptionError( "App DB revision mismatch. Refusing to start the app: " - f"expected {APP_BASELINE_REVISION}, got {current_revision}" + f"expected {expected_revision}, got {current_revision}" ) def adopt_or_initialize_app_db(database_url: str) -> str: database_path = _database_path_from_url(database_url) alembic_config = _make_alembic_config(database_url) + expected_revision = _expected_head_revision(alembic_config) if database_path.exists(): if _alembic_version_table_exists(database_path): current_revision = _fetch_alembic_revision(database_path) - if current_revision == APP_BASELINE_REVISION: + if current_revision == expected_revision: return "already_managed" + if not _is_known_revision(alembic_config, current_revision): + raise AppDatabaseAdoptionError( + "App DB is already Alembic-managed but revision does not match " + f"a known migration revision: got {current_revision}" + ) command.upgrade(alembic_config, "head") return "upgraded" diff --git a/scripts/location_db_adopt.py b/scripts/location_db_adopt.py index 5d8caa9..4424364 100644 --- a/scripts/location_db_adopt.py +++ b/scripts/location_db_adopt.py @@ -6,6 +6,8 @@ from pathlib import Path from alembic import command from alembic.config import Config +from alembic.script import ScriptDirectory +from alembic.util.exc import CommandError PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: @@ -43,6 +45,24 @@ def _make_alembic_config(database_url: str) -> Config: return config +def _expected_head_revision(alembic_config: Config) -> str: + script = ScriptDirectory.from_config(alembic_config) + heads = script.get_heads() + if len(heads) != 1: + raise LocationDatabaseAdoptionError( + f"Expected exactly one Alembic head for location DB, got {len(heads)}" + ) + return heads[0] + + +def _is_known_revision(alembic_config: Config, revision: str) -> bool: + script = ScriptDirectory.from_config(alembic_config) + try: + return script.get_revision(revision) is not None + except CommandError: + return False + + def _location_table_exists(database_path: Path) -> bool: conn = sqlite3.connect(database_path) try: @@ -117,6 +137,8 @@ def validate_legacy_location_db(database_url: str) -> None: def validate_location_runtime_db(database_url: str) -> None: database_path = _database_path_from_url(database_url) + alembic_config = _make_alembic_config(database_url) + expected_revision = _expected_head_revision(alembic_config) if not database_path.exists(): raise LocationDatabaseAdoptionError( "Location DB file was not found. Run 'python scripts/location_db_adopt.py' " @@ -131,30 +153,36 @@ def validate_location_runtime_db(database_url: str) -> None: ) current_revision = _fetch_alembic_revision(database_path) - if current_revision != LOCATION_BASELINE_REVISION: + if current_revision != expected_revision: raise LocationDatabaseAdoptionError( "Location DB revision mismatch. Refusing to start the app: " - f"expected {LOCATION_BASELINE_REVISION}, got {current_revision}" + f"expected {expected_revision}, got {current_revision}" ) def adopt_or_initialize_location_db(database_url: str) -> str: database_path = _database_path_from_url(database_url) alembic_config = _make_alembic_config(database_url) + expected_revision = _expected_head_revision(alembic_config) if database_path.exists(): if _alembic_version_table_exists(database_path): current_revision = _fetch_alembic_revision(database_path) - if current_revision != LOCATION_BASELINE_REVISION: + if current_revision == expected_revision: + return "already_managed" + if not _is_known_revision(alembic_config, current_revision): raise LocationDatabaseAdoptionError( "Location DB is already Alembic-managed but revision does not match " - f"the expected baseline: expected {LOCATION_BASELINE_REVISION}, " - f"got {current_revision}" + f"a known migration revision: got {current_revision}" ) - return "already_managed" + command.upgrade(alembic_config, "head") + return "upgraded" validate_legacy_location_db(database_url) command.stamp(alembic_config, LOCATION_BASELINE_REVISION) + if LOCATION_BASELINE_REVISION != expected_revision: + command.upgrade(alembic_config, "head") + return "upgraded" return "adopted" database_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/poo_db_adopt.py b/scripts/poo_db_adopt.py index f571afb..7540dce 100644 --- a/scripts/poo_db_adopt.py +++ b/scripts/poo_db_adopt.py @@ -6,6 +6,8 @@ from pathlib import Path from alembic import command from alembic.config import Config +from alembic.script import ScriptDirectory +from alembic.util.exc import CommandError PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: @@ -42,6 +44,24 @@ def _make_alembic_config(database_url: str) -> Config: return config +def _expected_head_revision(alembic_config: Config) -> str: + script = ScriptDirectory.from_config(alembic_config) + heads = script.get_heads() + if len(heads) != 1: + raise PooDatabaseAdoptionError( + f"Expected exactly one Alembic head for poo DB, got {len(heads)}" + ) + return heads[0] + + +def _is_known_revision(alembic_config: Config, revision: str) -> bool: + script = ScriptDirectory.from_config(alembic_config) + try: + return script.get_revision(revision) is not None + except CommandError: + return False + + def _poo_table_exists(database_path: Path) -> bool: conn = sqlite3.connect(database_path) try: @@ -112,6 +132,8 @@ def validate_legacy_poo_db(database_url: str) -> None: def validate_poo_runtime_db(database_url: str) -> None: database_path = _database_path_from_url(database_url) + alembic_config = _make_alembic_config(database_url) + expected_revision = _expected_head_revision(alembic_config) if not database_path.exists(): raise PooDatabaseAdoptionError( "Poo DB file was not found. Run 'python scripts/poo_db_adopt.py' first to " @@ -126,30 +148,36 @@ def validate_poo_runtime_db(database_url: str) -> None: ) current_revision = _fetch_alembic_revision(database_path) - if current_revision != POO_BASELINE_REVISION: + if current_revision != expected_revision: raise PooDatabaseAdoptionError( "Poo DB revision mismatch. Refusing to start the app: " - f"expected {POO_BASELINE_REVISION}, got {current_revision}" + f"expected {expected_revision}, got {current_revision}" ) def adopt_or_initialize_poo_db(database_url: str) -> str: database_path = _database_path_from_url(database_url) alembic_config = _make_alembic_config(database_url) + expected_revision = _expected_head_revision(alembic_config) if database_path.exists(): if _alembic_version_table_exists(database_path): current_revision = _fetch_alembic_revision(database_path) - if current_revision != POO_BASELINE_REVISION: + if current_revision == expected_revision: + return "already_managed" + if not _is_known_revision(alembic_config, current_revision): raise PooDatabaseAdoptionError( "Poo DB is already Alembic-managed but revision does not match " - f"the expected baseline: expected {POO_BASELINE_REVISION}, " - f"got {current_revision}" + f"a known migration revision: got {current_revision}" ) - return "already_managed" + command.upgrade(alembic_config, "head") + return "upgraded" validate_legacy_poo_db(database_url) command.stamp(alembic_config, POO_BASELINE_REVISION) + if POO_BASELINE_REVISION != expected_revision: + command.upgrade(alembic_config, "head") + return "upgraded" return "adopted" database_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/run_migrations.py b/scripts/run_migrations.py new file mode 100644 index 0000000..ba3af4e --- /dev/null +++ b/scripts/run_migrations.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from app.config import get_settings +from scripts.app_db_adopt import adopt_or_initialize_app_db +from scripts.location_db_adopt import adopt_or_initialize_location_db +from scripts.poo_db_adopt import adopt_or_initialize_poo_db + + +def run_all_migrations() -> dict[str, str]: + settings = get_settings() + return { + "app": adopt_or_initialize_app_db(settings.app_database_url), + "location": adopt_or_initialize_location_db(settings.location_database_url), + "poo": adopt_or_initialize_poo_db(settings.poo_database_url), + } + + +def main() -> None: + results = run_all_migrations() + for database_name, result in results.items(): + print(f"{database_name}: {result}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_app.py b/tests/test_app.py index cd9900d..05a57dc 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -37,12 +37,13 @@ def test_status_endpoint(client: TestClient) -> None: def test_app_start_fails_when_app_db_missing(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: + missing_app_path = tmp_path / "missing_app.db" poo_database_path = tmp_path / "poo_ready.db" location_database_path = tmp_path / "location_ready.db" command.upgrade(_make_poo_alembic_config(f"sqlite:///{poo_database_path}"), "head") command.upgrade(_make_alembic_config(f"sqlite:///{location_database_path}"), "head") - monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{tmp_path / 'missing_app.db'}") + monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{missing_app_path}") monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_database_path}") @@ -54,6 +55,8 @@ def test_app_start_fails_when_app_db_missing(tmp_path, monkeypatch: pytest.Monke with pytest.raises(RuntimeError, match="Run 'python scripts/app_db_adopt.py' first"): anyio.run(_run_lifespan, app) + assert not missing_app_path.exists() + get_settings.cache_clear() reset_auth_db_caches() diff --git a/tests/test_deployment.py b/tests/test_deployment.py new file mode 100644 index 0000000..36bd9d7 --- /dev/null +++ b/tests/test_deployment.py @@ -0,0 +1,213 @@ +from pathlib import Path +import sqlite3 + +import anyio +import pytest +import yaml +from alembic import command + +from app.auth_db import reset_auth_db_caches +from app.config import get_settings +from app.main import create_app +from scripts.app_db_adopt import APP_BASELINE_REVISION +from scripts.location_db_adopt import EXPECTED_USER_VERSION as LOCATION_USER_VERSION +from scripts.location_db_adopt import LOCATION_BASELINE_REVISION +from scripts.poo_db_adopt import EXPECTED_USER_VERSION as POO_USER_VERSION +from scripts.poo_db_adopt import POO_BASELINE_REVISION +from scripts.run_migrations import run_all_migrations +from tests.conftest import _make_alembic_config, _make_poo_alembic_config + + +def _read_yaml(path: str) -> dict: + return yaml.safe_load(Path(path).read_text()) + + +async def _run_lifespan(app) -> None: + async with app.router.lifespan_context(app): + return None + + +def _configure_database_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path | str]: + app_path = tmp_path / "app.db" + location_path = tmp_path / "location.db" + poo_path = tmp_path / "poo.db" + + monkeypatch.setenv("APP_DATABASE_URL", f"sqlite:///{app_path}") + monkeypatch.setenv("LOCATION_DATABASE_URL", f"sqlite:///{location_path}") + monkeypatch.setenv("POO_DATABASE_URL", f"sqlite:///{poo_path}") + monkeypatch.setenv("AUTH_BOOTSTRAP_USERNAME", "admin") + monkeypatch.setenv("AUTH_BOOTSTRAP_PASSWORD", "test-password") + monkeypatch.setenv("AUTH_COOKIE_SECURE_OVERRIDE", "false") + get_settings.cache_clear() + reset_auth_db_caches() + + return { + "app_path": app_path, + "app_url": f"sqlite:///{app_path}", + "location_path": location_path, + "location_url": f"sqlite:///{location_path}", + "poo_path": poo_path, + "poo_url": f"sqlite:///{poo_path}", + } + + +def _create_legacy_location_db(database_path: Path) -> None: + conn = sqlite3.connect(database_path) + conn.execute( + """ + CREATE TABLE location ( + person TEXT NOT NULL, + datetime TEXT NOT NULL, + latitude REAL NOT NULL, + longitude REAL NOT NULL, + altitude REAL, + PRIMARY KEY (person, datetime) + ) + """ + ) + conn.execute( + "INSERT INTO location (person, datetime, latitude, longitude, altitude) VALUES (?, ?, ?, ?, ?)", + ("alice", "2026-04-22T10:00:00Z", 1.23, 4.56, 7.89), + ) + conn.execute(f"PRAGMA user_version = {LOCATION_USER_VERSION}") + conn.commit() + conn.close() + + +def _create_legacy_poo_db(database_path: Path) -> None: + conn = sqlite3.connect(database_path) + conn.execute( + """ + CREATE TABLE poo_records ( + timestamp TEXT NOT NULL, + status TEXT NOT NULL, + latitude REAL NOT NULL, + longitude REAL NOT NULL, + PRIMARY KEY (timestamp) + ) + """ + ) + conn.execute( + "INSERT INTO poo_records (timestamp, status, latitude, longitude) VALUES (?, ?, ?, ?)", + ("2026-04-22T11:00:00Z", "complete", 9.87, 6.54), + ) + conn.execute(f"PRAGMA user_version = {POO_USER_VERSION}") + conn.commit() + conn.close() + + +def test_compose_uses_migration_job_before_app() -> None: + compose = _read_yaml("/home/tianyu/workspace/home-automation/docker-compose.yml") + override = _read_yaml("/home/tianyu/workspace/home-automation/docker-compose.override.yml") + + migration_service = compose["services"]["migration"] + app_service = compose["services"]["app"] + + assert migration_service["command"] == ["python", "-m", "scripts.run_migrations"] + assert migration_service["restart"] == "no" + assert app_service["depends_on"]["migration"]["condition"] == "service_completed_successfully" + assert override["services"]["migration"]["build"] == "." + assert override["services"]["app"]["build"] == "." + + +def test_image_defaults_to_uvicorn_only() -> None: + dockerfile = Path("/home/tianyu/workspace/home-automation/Dockerfile").read_text() + entrypoint = Path("/home/tianyu/workspace/home-automation/docker/entrypoint.sh").read_text() + + assert 'CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]' in dockerfile + assert 'exec "$@"' in entrypoint + assert "app_db_adopt" not in entrypoint + assert "location_db_adopt" not in entrypoint + assert "poo_db_adopt" not in entrypoint + + +def test_migration_runner_initializes_and_is_idempotent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + database_urls = _configure_database_env(tmp_path, monkeypatch) + + first_run = run_all_migrations() + second_run = run_all_migrations() + + assert first_run == {"app": "initialized", "location": "initialized", "poo": "initialized"} + assert second_run == { + "app": "already_managed", + "location": "already_managed", + "poo": "already_managed", + } + + conn = sqlite3.connect(database_urls["app_path"]) + try: + assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == APP_BASELINE_REVISION + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ).fetchall() + } + finally: + conn.close() + + assert {"auth_users", "auth_sessions", "app_config", "alembic_version"} <= tables + + conn = sqlite3.connect(database_urls["location_path"]) + try: + assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == LOCATION_BASELINE_REVISION + finally: + conn.close() + + conn = sqlite3.connect(database_urls["poo_path"]) + try: + assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == POO_BASELINE_REVISION + finally: + conn.close() + + get_settings.cache_clear() + reset_auth_db_caches() + + +def test_migration_runner_adopts_legacy_sqlite_without_data_loss( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + database_urls = _configure_database_env(tmp_path, monkeypatch) + _create_legacy_location_db(database_urls["location_path"]) + _create_legacy_poo_db(database_urls["poo_path"]) + + results = run_all_migrations() + + assert results == {"app": "initialized", "location": "adopted", "poo": "adopted"} + + conn = sqlite3.connect(database_urls["location_path"]) + try: + assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == LOCATION_BASELINE_REVISION + assert conn.execute("SELECT COUNT(*) FROM location").fetchone()[0] == 1 + finally: + conn.close() + + conn = sqlite3.connect(database_urls["poo_path"]) + try: + assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == POO_BASELINE_REVISION + assert conn.execute("SELECT COUNT(*) FROM poo_records").fetchone()[0] == 1 + finally: + conn.close() + + get_settings.cache_clear() + reset_auth_db_caches() + + +def test_app_startup_still_fails_closed_without_running_adoption( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + database_urls = _configure_database_env(tmp_path, monkeypatch) + missing_app_path = database_urls["app_path"] + command.upgrade(_make_alembic_config(database_urls["location_url"]), "head") + command.upgrade(_make_poo_alembic_config(database_urls["poo_url"]), "head") + + app = create_app() + with pytest.raises(RuntimeError, match="Run 'python scripts/app_db_adopt.py' first"): + anyio.run(_run_lifespan, app) + + assert not Path(missing_app_path).exists() + + get_settings.cache_clear() + reset_auth_db_caches() diff --git a/tests/test_location.py b/tests/test_location.py index 159c8b4..3af7c6e 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -343,7 +343,7 @@ def test_location_db_adoption_fails_closed_on_alembic_revision_mismatch( conn.commit() conn.close() - with pytest.raises(LocationDatabaseAdoptionError, match="revision does not match"): + with pytest.raises(LocationDatabaseAdoptionError, match="known migration revision"): adopt_or_initialize_location_db(f"sqlite:///{database_path}")