M8-R02: report DSMR MQTT source health

2026-08-24 06:45:30 +02:00
parent e59c192097
commit d9e82038dc
6 changed files with 483 additions and 9 deletions
@@ -31,6 +31,7 @@ from sqlalchemy import create_engine, select
 from sqlalchemy.orm import Session

 from app.models.energy import DsmrReading
+from app.models.meter_source import MeterSource


 # ---------------------------------------------------------------------------
@@ -174,6 +175,264 @@ def test_second_00_persists_full_frame(dsmr_db):
    assert payload["phase_voltage_l2"] is None


+def test_accepted_message_marks_source_online_and_updates_last_seen(dsmr_db):
+    """A valid, accepted telegram clears stale diagnostics without changing its payload."""
+    engine, SessionLocal = dsmr_db
+    settings = _make_settings(dsmr_sample_interval_s=10)
+    with Session(engine) as session:
+        source = session.get(MeterSource, 1)
+        assert source is not None
+        source.enabled = True
+        source.status = "error"
+        source.last_error = "old error"
+        session.commit()
+
+    _call_handle_message(_SAMPLE_TELEGRAM, settings, SessionLocal)
+
+    with Session(engine) as session:
+        source = session.get(MeterSource, 1)
+        assert source is not None
+        assert source.status == "online"
+        assert source.last_error is None
+        assert source.last_seen_at is not None
+
+
+def test_source_state_callback_persists_only_current_generation(dsmr_db, monkeypatch):
+    """Connection callbacks use a short session and stale generations leave health untouched."""
+    engine, SessionLocal = dsmr_db
+    from app.services import dsmr_ingest
+
+    snapshot = _make_settings()
+    current = object()
+    monkeypatch.setattr(dsmr_ingest, "_subscription_tokens", {snapshot.source_id: current})
+    with Session(engine) as session:
+        source = session.get(MeterSource, snapshot.source_id)
+        assert source is not None
+        source.enabled = True
+        session.commit()
+    with patch.object(dsmr_ingest, "get_session_local", return_value=SessionLocal):
+        dsmr_ingest.handle_captured_source_state(snapshot, current, "connecting")
+        dsmr_ingest.handle_captured_source_state(snapshot, current, "error")
+        dsmr_ingest.handle_captured_source_state(snapshot, object(), "online")
+
+    with Session(engine) as session:
+        source = session.get(MeterSource, snapshot.source_id)
+        assert source is not None
+        assert source.status == "error"
+        assert source.last_error == "MQTT connection failed."
+
+
+def test_source_health_is_isolated_between_sources(dsmr_db, monkeypatch):
+    """Connection and message health changes only touch their own source row."""
+    engine, SessionLocal = dsmr_db
+    from app.services import dsmr_ingest
+    from app.services.dsmr_ingest import DsmrSourceSnapshot
+
+    first = _make_settings()
+    second = DsmrSourceSnapshot(2, "second/topic", "", 10)
+    first_token = object()
+    second_token = object()
+    monkeypatch.setattr(
+        dsmr_ingest, "_subscription_tokens", {first.source_id: first_token, second.source_id: second_token}
+    )
+    with Session(engine) as session:
+        first_source = session.get(MeterSource, first.source_id)
+        assert first_source is not None
+        first_source.enabled = True
+        session.add(MeterSource(
+            id=2, name="Second DSMR", kind="dsmr_mqtt", enabled=True, config={},
+            status="online", created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc),
+        ))
+        session.commit()
+
+    with patch.object(dsmr_ingest, "get_session_local", return_value=SessionLocal):
+        dsmr_ingest.handle_captured_source_state(first, first_token, "connecting")
+        dsmr_ingest.handle_captured_source_state(second, second_token, "error")
+        dsmr_ingest.handle_message(_payload(_SAMPLE_TELEGRAM), first)
+
+    with Session(engine) as session:
+        first_source = session.get(MeterSource, first.source_id)
+        second_source = session.get(MeterSource, second.source_id)
+        assert first_source is not None and second_source is not None
+        assert first_source.status == "online"
+        assert first_source.last_error is None
+        assert first_source.last_seen_at is not None
+        assert second_source.status == "error"
+        assert second_source.last_error == "MQTT connection failed."
+        assert second_source.last_seen_at is None
+
+
+def test_disable_reconcile_persists_unknown_and_rejects_retained_state_callback(dsmr_db, monkeypatch):
+    """Disable invalidates the generation before clearing a stale online health state."""
+    engine, SessionLocal = dsmr_db
+    from app.services import dsmr_ingest
+
+    class FakeMqtt:
+        def __init__(self) -> None:
+            self.removed: list[int] = []
+            self.state_handler = None
+            self.active: set[int] = set()
+
+        def replace_source(self, source_id: int, **kwargs: object) -> bool:
+            self.active.add(source_id)
+            self.state_handler = kwargs["state_handler"]
+            return True
+
+        def remove_source(self, source_id: int) -> None:
+            self.removed.append(source_id)
+            self.active.discard(source_id)
+
+        def source_is_active(self, source_id: int) -> bool:
+            return source_id in self.active
+
+    fake = FakeMqtt()
+    snapshot = _make_settings()
+    monkeypatch.setattr(dsmr_ingest, "_subscriptions", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_client_ids", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_tokens", {})
+    monkeypatch.setattr(dsmr_ingest, "_tariffs", {})
+    monkeypatch.setattr(dsmr_ingest, "get_session_local", lambda: SessionLocal)
+    monkeypatch.setattr("app.integrations.mqtt.mqtt_manager", fake)
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [snapshot])
+    with Session(engine) as session:
+        source = session.get(MeterSource, snapshot.source_id)
+        assert source is not None
+        source.enabled = True
+        source.status = "online"
+        session.commit()
+    dsmr_ingest.apply_dsmr_subscription()
+    assert fake.state_handler is not None
+
+    with Session(engine) as session:
+        source = session.get(MeterSource, snapshot.source_id)
+        assert source is not None
+        source.enabled = False
+        session.commit()
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [])
+    dsmr_ingest.apply_dsmr_subscription()
+    fake.state_handler("online")
+
+    with Session(engine) as session:
+        source = session.get(MeterSource, snapshot.source_id)
+        assert source is not None
+        assert source.status == "unknown"
+        assert source.last_error is None
+    assert fake.removed == [snapshot.source_id]
+
+
+def test_topic_collision_persists_error_when_reconcile_removes_active_client(dsmr_db, monkeypatch):
+    """A rejected enabled source cannot retain health from its removed client."""
+    engine, SessionLocal = dsmr_db
+    from app.services import dsmr_ingest
+
+    class FakeMqtt:
+        def __init__(self) -> None:
+            self.removed: list[int] = []
+            self.active: set[int] = set()
+
+        def replace_source(self, source_id: int, **kwargs: object) -> bool:
+            self.active.add(source_id)
+            return True
+
+        def remove_source(self, source_id: int) -> None:
+            self.removed.append(source_id)
+            self.active.discard(source_id)
+
+        def source_is_active(self, source_id: int) -> bool:
+            return source_id in self.active
+
+    fake = FakeMqtt()
+    valid = _make_settings(dsmr_mqtt_topic="dsmr/telegram")
+    collision = _make_settings(dsmr_mqtt_topic="dsmr/telegram")
+    collision = dsmr_ingest.DsmrSourceSnapshot(
+        collision.source_id,
+        collision.topic,
+        collision.topic,
+        collision.sample_interval_s,
+    )
+    monkeypatch.setattr(dsmr_ingest, "_subscriptions", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_client_ids", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_tokens", {})
+    monkeypatch.setattr(dsmr_ingest, "_tariffs", {})
+    monkeypatch.setattr(dsmr_ingest, "get_session_local", lambda: SessionLocal)
+    monkeypatch.setattr("app.integrations.mqtt.mqtt_manager", fake)
+    with Session(engine) as session:
+        source = session.get(MeterSource, valid.source_id)
+        assert source is not None
+        source.enabled = True
+        source.status = "online"
+        session.commit()
+
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [valid])
+    dsmr_ingest.apply_dsmr_subscription()
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [collision])
+    dsmr_ingest.apply_dsmr_subscription()
+
+    with Session(engine) as session:
+        source = session.get(MeterSource, valid.source_id)
+        assert source is not None
+        assert source.enabled is True
+        assert source.status == "error"
+        assert source.last_error == "DSMR source configuration invalid."
+    assert fake.removed == [valid.source_id]
+    assert fake.active == set()
+
+
+def test_topic_collision_persists_error_on_startup_without_runtime_client(dsmr_db, monkeypatch):
+    """A collision corrects stale persisted online health without an installed client."""
+    engine, SessionLocal = dsmr_db
+    from app.services import dsmr_ingest
+
+    collision = dsmr_ingest.DsmrSourceSnapshot(1, "dsmr/telegram", "dsmr/telegram", 10)
+    monkeypatch.setattr(dsmr_ingest, "_subscriptions", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_client_ids", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_tokens", {})
+    monkeypatch.setattr(dsmr_ingest, "get_session_local", lambda: SessionLocal)
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [collision])
+    with Session(engine) as session:
+        source = session.get(MeterSource, collision.source_id)
+        assert source is not None
+        source.enabled = True
+        source.status = "online"
+        session.commit()
+
+    dsmr_ingest.apply_dsmr_subscription()
+
+    with Session(engine) as session:
+        source = session.get(MeterSource, collision.source_id)
+        assert source is not None
+        assert source.status == "error"
+        assert source.last_error == "DSMR source configuration invalid."
+
+
+def test_topic_collision_does_not_change_a_disabled_source(dsmr_db, monkeypatch):
+    """A concurrent disable is not overwritten by collision error handling."""
+    engine, SessionLocal = dsmr_db
+    from app.services import dsmr_ingest
+
+    collision = dsmr_ingest.DsmrSourceSnapshot(1, "dsmr/telegram", "dsmr/telegram", 10)
+    monkeypatch.setattr(dsmr_ingest, "_subscriptions", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_client_ids", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_tokens", {})
+    monkeypatch.setattr(dsmr_ingest, "get_session_local", lambda: SessionLocal)
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [collision])
+    with Session(engine) as session:
+        source = session.get(MeterSource, collision.source_id)
+        assert source is not None
+        source.enabled = False
+        source.status = "unknown"
+        source.last_error = None
+        session.commit()
+
+    dsmr_ingest.apply_dsmr_subscription()
+
+    with Session(engine) as session:
+        source = session.get(MeterSource, collision.source_id)
+        assert source is not None
+        assert source.status == "unknown"
+        assert source.last_error is None
+
+
 def test_second_10_persists(dsmr_db):
    """A telegram with second=10 (another 10s boundary) must also be persisted."""
    _, SessionLocal = dsmr_db
@@ -46,6 +46,7 @@ def fake_mqtt(monkeypatch):
    monkeypatch.setattr("app.integrations.mqtt.mqtt_manager", fake)
    monkeypatch.setattr(dsmr_ingest, "_subscriptions", {})
    monkeypatch.setattr(dsmr_ingest, "_subscription_client_ids", {})
+    monkeypatch.setattr(dsmr_ingest, "_subscription_tokens", {})
    monkeypatch.setattr(dsmr_ingest, "_tariffs", {})
    return fake

@@ -186,3 +187,17 @@ def test_same_snapshot_aba_rejects_retained_handler(fake_mqtt, monkeypatch):
    old_handler(b"stale")  # type: ignore[operator]
    fake_mqtt.handlers[1]["same"](b"fresh")  # type: ignore[operator]
    assert received == [(b"fresh", snapshot)]
+
+
+def test_source_state_handler_is_generation_scoped(fake_mqtt, monkeypatch):
+    snapshot = _source(1, "same", broker_host="one.test")
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [snapshot])
+    dsmr_ingest.apply_dsmr_subscription()
+    stale = fake_mqtt.replace_calls[-1][1]["state_handler"]
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [])
+    dsmr_ingest.apply_dsmr_subscription()
+    monkeypatch.setattr(dsmr_ingest, "_enabled_snapshots", lambda: [snapshot])
+    dsmr_ingest.apply_dsmr_subscription()
+
+    monkeypatch.setattr(dsmr_ingest, "get_session_local", lambda: (_ for _ in ()).throw(AssertionError()))
+    stale("error")  # type: ignore[operator]
@@ -302,6 +302,12 @@ def test_source_channel_binding_response_contract_and_discover_capabilities(auth
        _login(client)
        source = _create_source(client, config={"username": "private-user", "password": "private-secret"})
        channel_uuid = _add_channel(engine, source["uuid"])
+        with Session(engine) as session:
+            source_model = session.query(MeterSource).filter_by(uuid=source["uuid"]).one()
+            source_model.status = "online"
+            source_model.last_seen_at = datetime.now(UTC)
+            source_model.last_error = None
+            session.commit()
        listed = client.get("/api/energy/sources")
        assert listed.status_code == 200
        assert listed.json()["total"] >= 1
@@ -332,7 +338,7 @@ def test_source_channel_binding_response_contract_and_discover_capabilities(auth
            "uuid", "label", "suggested_commodity", "unit", "device_type", "latest_value",
            "latest_at", "latest_quality", "binding_count", "bound_meter_ids", "binding_summary",
        }
-        assert channels.json()["source_status"] == "unknown"
+        assert channels.json()["source_status"] == "online"

        meter = client.post("/api/energy/meters", headers={"X-CSRF-Token": _CSRF}, json={
            "label": "Contract meter", "started_at": "2030-01-01T00:00:00Z", "reason": "initial",
@@ -437,6 +437,88 @@ def test_source_sync_connack_before_connect_returns_subscribes_all_topics() -> N
    assert client.subscribed == ["telegram/topic", "tariff/topic"]


+def test_source_health_tracks_connack_disconnect_and_ignores_stale_callbacks() -> None:
+    """A source is connecting until CONNACK, and old generations cannot rewrite health."""
+    manager = MqttManager()
+    old_client = MagicMock()
+    new_client = MagicMock()
+    old_states: list[str] = []
+    new_states: list[str] = []
+    kwargs = {
+        "host": "broker.test",
+        "port": 1883,
+        "username": "",
+        "password": "",
+        "tls_enabled": False,
+        "subscriptions": {"topic": lambda _payload: None},
+    }
+    with patch("app.integrations.mqtt.mqtt.Client", side_effect=[old_client, new_client]):
+        assert manager.replace_source(1, **kwargs, state_handler=old_states.append)
+        assert old_states == ["connecting"]
+        assert manager.replace_source(1, **kwargs, state_handler=new_states.append)
+
+    accepted = MagicMock()
+    accepted.is_failure = False
+    old_client.on_connect(old_client, None, MagicMock(), accepted, None)
+    old_client.on_disconnect(old_client, None, MagicMock(), MagicMock(), None)
+    assert old_states == ["connecting"]
+
+    new_client.on_connect(new_client, None, MagicMock(), accepted, None)
+    new_client.on_disconnect(new_client, None, MagicMock(), MagicMock(), None)
+    assert new_states == ["connecting", "online", "error"]
+
+
+def test_source_health_rejected_connack_reports_error_without_disconnect() -> None:
+    """A failed CONNACK is distinct from a later disconnect callback."""
+    manager = MqttManager()
+    client = MagicMock()
+    states: list[str] = []
+    with patch("app.integrations.mqtt.mqtt.Client", return_value=client):
+        assert manager.replace_source(
+            1, host="broker.test", port=1883, username="", password="", tls_enabled=False,
+            subscriptions={"topic": lambda _payload: None}, state_handler=states.append,
+        )
+
+    accepted = MagicMock()
+    accepted.is_failure = False
+    refused = MagicMock()
+    refused.is_failure = True
+    client.on_connect(client, None, MagicMock(), accepted, None)
+    client.on_connect(client, None, MagicMock(), refused, None)
+
+    assert states == ["connecting", "online", "error"]
+    assert 1 not in manager._source_connected
+
+
+def test_source_health_disconnect_then_reconnect_connack_is_ordered_and_isolated() -> None:
+    """One source's reconnect sequence cannot change another source's health."""
+    manager = MqttManager()
+    first_client = MagicMock()
+    second_client = MagicMock()
+    first_states: list[str] = []
+    second_states: list[str] = []
+    kwargs = {
+        "host": "broker.test",
+        "port": 1883,
+        "username": "",
+        "password": "",
+        "tls_enabled": False,
+        "subscriptions": {"topic": lambda _payload: None},
+    }
+    with patch("app.integrations.mqtt.mqtt.Client", side_effect=[first_client, second_client]):
+        assert manager.replace_source(1, **kwargs, state_handler=first_states.append)
+        assert manager.replace_source(2, **kwargs, state_handler=second_states.append)
+
+    accepted = MagicMock()
+    accepted.is_failure = False
+    first_client.on_connect(first_client, None, MagicMock(), accepted, None)
+    first_client.on_disconnect(first_client, None, MagicMock(), MagicMock(), None)
+    first_client.on_connect(first_client, None, MagicMock(), accepted, None)
+
+    assert first_states == ["connecting", "online", "error", "online"]
+    assert second_states == ["connecting"]
+
+
 def test_source_teardown_with_joining_loop_stop_waits_for_callback_before_aba() -> None:
    """loop_stop may join a callback that needs the manager lock to finish."""
    manager = MqttManager()