tliu93/home-automation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

M6-T06: extend MqttManager with subscribe + DSMR ingest

Browse Source 
- mqtt.py: subscription registry, on_message dispatch (swallows handler
  errors), re-subscribe on (re)connect; publish/connect/disconnect/reconnect
  unchanged.
- app/services/dsmr_ingest.py: handle_message stores the full telegram frame
  (no allowlist; gas/phases/null preserved), 10s downsample by timestamp.second,
  source_id idempotency (select + IntegrityError rollback). Net-thread safe.
- main.py registers the DSMR subscription only when dsmr_ingest_enabled.
This commit is contained in:
tliu93
2026-06-23 21:52:57 +02:00
parent 7e266a21eb
commit ef48032adb
6 changed files with 920 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_dsmr_ingest.py
+394
View File
@@ -0,0 +1,394 @@
"""Tests for M6-T06: DSMR ingest service (handle_message).
All tests use a temporary SQLite database (upgraded to the full Alembic head)
so that the real DsmrReading model/constraints are exercised.
Covers:
- Sample telegram with second=00 (on interval) is persisted as a full-frame blob.
- Sample telegram with second=48 (off interval, 10s default) is discarded.
- Telegram with second=00 → all fields including gas (extra_device_*) and null
phases are stored verbatim.
- Duplicate source_id → not re-inserted (idempotency).
- source_id absent → row still inserted with source_id=None.
- dsmr_sample_interval_s=0 → no ZeroDivisionError; every telegram is persisted.
- Invalid JSON → silently discarded (no exception propagated).
- Missing 'timestamp' field → silently discarded.
- Unparseable 'timestamp' → silently discarded.
- Handler exception (e.g. bad DB) → does not propagate out of handle_message.
"""
from __future__ import annotations
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from alembic import command
from alembic.config import Config
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
from app.models.energy import DsmrReading
# ---------------------------------------------------------------------------
# Helpers / fixtures
# ---------------------------------------------------------------------------
def _alembic_config(database_url: str) -> Config:
cfg = Config("alembic_app.ini")
cfg.set_main_option("sqlalchemy.url", database_url)
return cfg
@pytest.fixture()
def dsmr_db(tmp_path: Path):
"""Temporary SQLite DB upgraded to the Alembic head; yields (engine, session_factory)."""
db_path = tmp_path / "dsmr_ingest_test.db"
db_url = f"sqlite:///{db_path}"
command.upgrade(_alembic_config(db_url), "head")
engine = create_engine(db_url, connect_args={"check_same_thread": False})
from sqlalchemy.orm import sessionmaker
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, class_=Session)
yield engine, SessionLocal
engine.dispose()
def _make_settings(
*,
dsmr_sample_interval_s: int = 10,
dsmr_ingest_enabled: bool = True,
dsmr_mqtt_topic: str = "dsmr/json",
):
s = MagicMock()
s.dsmr_sample_interval_s = dsmr_sample_interval_s
s.dsmr_ingest_enabled = dsmr_ingest_enabled
s.dsmr_mqtt_topic = dsmr_mqtt_topic
return s
# The reference telegram sample from §6.3 of the design doc.
# Numeric values are JSON strings; missing phases are null.
_SAMPLE_TELEGRAM = {
"id": 200086230,
"timestamp": "2026-06-23T12:16:00Z", # second=00, on 10s boundary
"electricity_delivered_1": "20915.154",
"electricity_returned_1": "2979.905",
"electricity_delivered_2": "15212.090",
"electricity_returned_2": "6786.406",
"electricity_currently_delivered": "0.000",
"phase_currently_delivered_l1": "0.000",
"phase_currently_delivered_l2": None, # absent phase
"extra_device_timestamp": "2026-06-23T12:15:00Z",
"extra_device_delivered": "6208.234",
"phase_voltage_l1": "237.0",
"phase_voltage_l2": None, # absent phase
}
# Telegram with second=48 (not divisible by 10) — should be discarded.
_SAMPLE_TELEGRAM_SECOND_48 = {
**_SAMPLE_TELEGRAM,
"id": 200086248,
"timestamp": "2026-06-23T12:16:48Z", # second=48
}
# Telegram with second=10 — another valid boundary.
_SAMPLE_TELEGRAM_SECOND_10 = {
**_SAMPLE_TELEGRAM,
"id": 200086210,
"timestamp": "2026-06-23T12:16:10Z", # second=10
}
def _payload(data: dict) -> bytes:
return json.dumps(data).encode()
# ---------------------------------------------------------------------------
# Helper: invoke handle_message with the test DB injected via patch
# ---------------------------------------------------------------------------
def _call_handle_message(
data: dict,
settings,
SessionLocal,
) -> None:
"""Call dsmr_ingest.handle_message with the test SessionLocal patched in."""
from app.services import dsmr_ingest
with patch.object(dsmr_ingest, "get_session_local", return_value=SessionLocal):
dsmr_ingest.handle_message(_payload(data), settings)
def _count_readings(SessionLocal) -> int:
with SessionLocal() as session:
return session.scalar(
__import__("sqlalchemy", fromlist=["func"]).func.count(DsmrReading.id)
)
def _get_readings(SessionLocal) -> list[DsmrReading]:
with SessionLocal() as session:
return session.scalars(select(DsmrReading)).all()
# ---------------------------------------------------------------------------
# 1. On-interval telegram is persisted (whole frame)
# ---------------------------------------------------------------------------
def test_second_00_persists_full_frame(dsmr_db):
"""A telegram with second=00 (10s boundary) must be persisted with the full payload."""
engine, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
_call_handle_message(_SAMPLE_TELEGRAM, settings, SessionLocal)
with Session(engine) as session:
readings = session.scalars(select(DsmrReading)).all()
assert len(readings) == 1
row = readings[0]
# Recorded_at must be parsed from the telegram timestamp.
assert row.recorded_at is not None
assert row.recorded_at.second == 0
# source_id must equal the telegram id.
assert row.source_id == _SAMPLE_TELEGRAM["id"]
# Full frame preserved — spot-check several fields.
payload = row.payload
assert payload["electricity_delivered_1"] == "20915.154"
assert payload["electricity_returned_2"] == "6786.406"
assert payload["extra_device_delivered"] == "6208.234"
assert payload["extra_device_timestamp"] == "2026-06-23T12:15:00Z"
# Null phases stored verbatim (not omitted, not converted).
assert "phase_currently_delivered_l2" in payload
assert payload["phase_currently_delivered_l2"] is None
assert "phase_voltage_l2" in payload
assert payload["phase_voltage_l2"] is None
def test_second_10_persists(dsmr_db):
"""A telegram with second=10 (another 10s boundary) must also be persisted."""
_, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
_call_handle_message(_SAMPLE_TELEGRAM_SECOND_10, settings, SessionLocal)
assert _count_readings(SessionLocal) == 1
# ---------------------------------------------------------------------------
# 2. Off-interval telegram is discarded
# ---------------------------------------------------------------------------
def test_second_48_discarded(dsmr_db):
"""A telegram with second=48 (not on 10s boundary) must be silently discarded."""
_, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
_call_handle_message(_SAMPLE_TELEGRAM_SECOND_48, settings, SessionLocal)
assert _count_readings(SessionLocal) == 0
def test_off_interval_not_multiple_of_10(dsmr_db):
"""Seconds 1–9, 11–19, etc. are all off-interval and must be discarded."""
_, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
for second in (1, 3, 7, 9, 11, 23, 47, 59):
ts = f"2026-06-23T12:16:{second:02d}Z"
data = {**_SAMPLE_TELEGRAM, "id": 200000000 + second, "timestamp": ts}
_call_handle_message(data, settings, SessionLocal)
assert _count_readings(SessionLocal) == 0
# ---------------------------------------------------------------------------
# 3. Idempotency — same source_id not re-inserted
# ---------------------------------------------------------------------------
def test_same_source_id_not_reinserted(dsmr_db):
"""Feeding the same telegram twice (same id) must result in exactly one row."""
_, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
_call_handle_message(_SAMPLE_TELEGRAM, settings, SessionLocal)
_call_handle_message(_SAMPLE_TELEGRAM, settings, SessionLocal) # duplicate
assert _count_readings(SessionLocal) == 1
def test_different_source_ids_are_independent(dsmr_db):
"""Different id values from different telegrams produce separate rows."""
_, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
_call_handle_message(_SAMPLE_TELEGRAM, settings, SessionLocal)
_call_handle_message(_SAMPLE_TELEGRAM_SECOND_10, settings, SessionLocal)
assert _count_readings(SessionLocal) == 2
# ---------------------------------------------------------------------------
# 4. Missing source_id — still persisted with source_id=None
# ---------------------------------------------------------------------------
def test_missing_id_persisted_with_source_id_none(dsmr_db):
"""Telegram without an 'id' field must be stored with source_id=None."""
engine, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
data = {k: v for k, v in _SAMPLE_TELEGRAM.items() if k != "id"}
_call_handle_message(data, settings, SessionLocal)
with Session(engine) as session:
readings = session.scalars(select(DsmrReading)).all()
assert len(readings) == 1
assert readings[0].source_id is None
# ---------------------------------------------------------------------------
# 5. interval=0 → no ZeroDivisionError; all telegrams persisted
# ---------------------------------------------------------------------------
def test_interval_zero_does_not_crash(dsmr_db):
"""When dsmr_sample_interval_s=0, no ZeroDivisionError; all telegrams kept."""
_, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=0)
for second in (0, 7, 13, 48, 59):
ts = f"2026-06-23T12:16:{second:02d}Z"
data = {**_SAMPLE_TELEGRAM, "id": 300000000 + second, "timestamp": ts}
_call_handle_message(data, settings, SessionLocal)
# All 5 should have been persisted (no sampling applied).
assert _count_readings(SessionLocal) == 5
# ---------------------------------------------------------------------------
# 6. Invalid / malformed payloads — silently discarded, no exception propagated
# ---------------------------------------------------------------------------
def test_invalid_json_discarded(dsmr_db):
"""Invalid JSON payload must be silently discarded without raising."""
_, SessionLocal = dsmr_db
settings = _make_settings()
from app.services import dsmr_ingest
with patch.object(dsmr_ingest, "get_session_local", return_value=SessionLocal):
# Must not raise
dsmr_ingest.handle_message(b"not-valid-json", settings)
assert _count_readings(SessionLocal) == 0
def test_missing_timestamp_discarded(dsmr_db):
"""Telegram without a 'timestamp' field must be discarded."""
_, SessionLocal = dsmr_db
settings = _make_settings()
data = {k: v for k, v in _SAMPLE_TELEGRAM.items() if k != "timestamp"}
_call_handle_message(data, settings, SessionLocal)
assert _count_readings(SessionLocal) == 0
def test_unparseable_timestamp_discarded(dsmr_db):
"""Telegram with a garbage 'timestamp' field must be discarded."""
_, SessionLocal = dsmr_db
settings = _make_settings()
data = {**_SAMPLE_TELEGRAM, "timestamp": "not-a-date"}
_call_handle_message(data, settings, SessionLocal)
assert _count_readings(SessionLocal) == 0
def test_handle_message_does_not_propagate_any_exception() -> None:
"""handle_message must never propagate any exception to the caller."""
from app.services import dsmr_ingest
settings = _make_settings()
def _explode():
raise RuntimeError("DB is on fire")
with patch.object(dsmr_ingest, "get_session_local", side_effect=_explode):
# Must not raise
dsmr_ingest.handle_message(_payload(_SAMPLE_TELEGRAM), settings)
# ---------------------------------------------------------------------------
# 7. Null phase values stored verbatim
# ---------------------------------------------------------------------------
def test_null_phases_stored_as_none(dsmr_db):
"""Null phase values must be preserved as JSON null (Python None) in payload."""
engine, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
_call_handle_message(_SAMPLE_TELEGRAM, settings, SessionLocal)
with Session(engine) as session:
reading = session.scalars(select(DsmrReading)).first()
assert reading is not None
assert reading.payload.get("phase_currently_delivered_l2") is None
assert reading.payload.get("phase_voltage_l2") is None
# ---------------------------------------------------------------------------
# 8. Gas / extra_device fields stored
# ---------------------------------------------------------------------------
def test_gas_fields_stored(dsmr_db):
"""extra_device_* (gas) fields must be present in the stored payload."""
engine, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
_call_handle_message(_SAMPLE_TELEGRAM, settings, SessionLocal)
with Session(engine) as session:
reading = session.scalars(select(DsmrReading)).first()
assert reading is not None
payload = reading.payload
assert "extra_device_delivered" in payload
assert payload["extra_device_delivered"] == "6208.234"
assert "extra_device_timestamp" in payload
# ---------------------------------------------------------------------------
# 9. Timestamp with +00:00 suffix (no Z) also works
# ---------------------------------------------------------------------------
def test_timestamp_with_utc_offset_suffix(dsmr_db):
"""Timestamps using '+00:00' instead of 'Z' must also be parsed correctly."""
_, SessionLocal = dsmr_db
settings = _make_settings(dsmr_sample_interval_s=10)
data = {
**_SAMPLE_TELEGRAM,
"id": 999999,
"timestamp": "2026-06-23T12:16:00+00:00", # no Z, uses +00:00
}
_call_handle_message(data, settings, SessionLocal)
assert _count_readings(SessionLocal) == 1
tests/test_mqtt_subscribe.py
+286
View File
@@ -0,0 +1,286 @@
"""Tests for M6-T06: MqttManager subscribe + on_message dispatch.
All paho interaction is mocked — no real broker is required.
Covers:
- subscribe() registers topic → handler in the internal registry.
- on_message dispatches the payload to the correct handler.
- A handler that raises does NOT propagate the exception out of on_message.
- Existing publish() behaviour is not regressed.
- When already connected, subscribe() calls client.subscribe() immediately.
- on_connect re-subscribes to all registered topics after (re-)connect.
- subscribe() before connect: topic is subscribed when connect fires.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
from app.integrations.mqtt import MqttManager
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_settings(
*,
mqtt_enabled: bool = True,
mqtt_broker_host: str = "broker.test",
mqtt_broker_port: int = 1883,
mqtt_username: str = "",
mqtt_password: str = "",
mqtt_tls_enabled: bool = False,
):
s = MagicMock()
s.mqtt_enabled = mqtt_enabled
s.mqtt_broker_host = mqtt_broker_host
s.mqtt_broker_port = mqtt_broker_port
s.mqtt_username = mqtt_username
s.mqtt_password = mqtt_password
s.mqtt_tls_enabled = mqtt_tls_enabled
return s
def _make_mqtt_message(topic: str, payload: bytes) -> MagicMock:
"""Construct a minimal mock of a paho MQTTMessage."""
msg = MagicMock()
msg.topic = topic
msg.payload = payload
return msg
# ---------------------------------------------------------------------------
# subscribe() — registry
# ---------------------------------------------------------------------------
def test_subscribe_registers_handler() -> None:
"""subscribe() must store the handler in the internal _subscriptions dict."""
manager = MqttManager()
handler = MagicMock()
manager.subscribe("test/topic", handler)
assert "test/topic" in manager._subscriptions
assert manager._subscriptions["test/topic"] is handler
def test_subscribe_overwrites_previous_handler() -> None:
"""Subscribing to the same topic twice replaces the handler."""
manager = MqttManager()
h1 = MagicMock()
h2 = MagicMock()
manager.subscribe("test/topic", h1)
manager.subscribe("test/topic", h2)
assert manager._subscriptions["test/topic"] is h2
def test_subscribe_when_not_connected_does_not_call_paho_subscribe() -> None:
"""subscribe() before connect must not attempt to call client.subscribe()."""
real_manager = MqttManager()
handler = MagicMock()
mock_client = MagicMock()
real_manager._client = None
real_manager._connected = False
real_manager.subscribe("dsmr/json", handler)
# No client → subscribe should not have been called on paho
mock_client.subscribe.assert_not_called()
def test_subscribe_when_connected_calls_paho_subscribe_immediately() -> None:
"""subscribe() while connected must call client.subscribe(topic) right away."""
manager = MqttManager()
mock_client = MagicMock()
manager._client = mock_client
manager._connected = True
handler = MagicMock()
manager.subscribe("dsmr/json", handler)
mock_client.subscribe.assert_called_once_with("dsmr/json")
# ---------------------------------------------------------------------------
# on_message dispatch
# ---------------------------------------------------------------------------
def _extract_on_message(manager: MqttManager, settings) -> object:
"""Connect with a mocked paho client and return the on_message callback that
was registered on the client mock."""
mock_client = MagicMock()
mock_client.connect.return_value = None
with patch("app.integrations.mqtt.mqtt.Client", return_value=mock_client):
manager.connect(settings)
# The callback was assigned as: client.on_message = _on_message
return mock_client.on_message
def test_on_message_dispatches_to_registered_handler() -> None:
"""on_message must invoke the handler registered for the message's topic."""
manager = MqttManager()
settings = _make_settings()
received_payloads: list[bytes] = []
def handler(payload: bytes) -> None:
received_payloads.append(payload)
manager.subscribe("dsmr/json", handler)
on_message = _extract_on_message(manager, settings)
msg = _make_mqtt_message("dsmr/json", b'{"id": 1}')
on_message(MagicMock(), None, msg) # simulate paho calling the callback
assert received_payloads == [b'{"id": 1}']
def test_on_message_ignores_unknown_topic() -> None:
"""on_message for an unregistered topic must not raise and must not call any handler."""
manager = MqttManager()
settings = _make_settings()
handler = MagicMock()
manager.subscribe("other/topic", handler)
on_message = _extract_on_message(manager, settings)
msg = _make_mqtt_message("unknown/topic", b"data")
# Must not raise
on_message(MagicMock(), None, msg)
handler.assert_not_called()
def test_on_message_swallows_handler_exception() -> None:
"""If the handler raises, on_message must NOT propagate the exception."""
manager = MqttManager()
settings = _make_settings()
def bad_handler(payload: bytes) -> None:
raise RuntimeError("handler exploded")
manager.subscribe("dsmr/json", bad_handler)
on_message = _extract_on_message(manager, settings)
msg = _make_mqtt_message("dsmr/json", b"{}")
# This must not raise
on_message(MagicMock(), None, msg)
def test_on_message_does_not_crash_on_handler_exception_multiple_calls() -> None:
"""After a handler exception the manager remains functional for subsequent messages."""
manager = MqttManager()
settings = _make_settings()
call_count = [0]
def flaky_handler(payload: bytes) -> None:
call_count[0] += 1
if call_count[0] == 1:
raise ValueError("first call fails")
manager.subscribe("dsmr/json", flaky_handler)
on_message = _extract_on_message(manager, settings)
msg1 = _make_mqtt_message("dsmr/json", b"first")
msg2 = _make_mqtt_message("dsmr/json", b"second")
on_message(MagicMock(), None, msg1) # should not raise despite exception in handler
on_message(MagicMock(), None, msg2) # handler called again
assert call_count[0] == 2
# ---------------------------------------------------------------------------
# on_connect re-subscribes registered topics
# ---------------------------------------------------------------------------
def test_on_connect_subscribes_all_registered_topics() -> None:
"""When connect fires successfully, _on_connect must subscribe every registered topic."""
manager = MqttManager()
settings = _make_settings()
manager.subscribe("topic/a", MagicMock())
manager.subscribe("topic/b", MagicMock())
mock_client = MagicMock()
mock_client.connect.return_value = None
# Capture the on_connect callback that was registered on the mock client
with patch("app.integrations.mqtt.mqtt.Client", return_value=mock_client):
manager.connect(settings)
on_connect_cb = mock_client.on_connect
# Simulate broker accepting the connection (reason_code.is_failure == False)
reason_code = MagicMock()
reason_code.is_failure = False
on_connect_cb(mock_client, None, MagicMock(), reason_code, None)
# client.subscribe should have been called once per topic during on_connect
subscribe_calls = [c[0][0] for c in mock_client.subscribe.call_args_list]
assert "topic/a" in subscribe_calls
assert "topic/b" in subscribe_calls
def test_on_connect_does_not_subscribe_on_failure() -> None:
"""When the broker rejects the connection, _on_connect must NOT subscribe."""
manager = MqttManager()
settings = _make_settings()
manager.subscribe("topic/a", MagicMock())
mock_client = MagicMock()
mock_client.connect.return_value = None
with patch("app.integrations.mqtt.mqtt.Client", return_value=mock_client):
manager.connect(settings)
on_connect_cb = mock_client.on_connect
reason_code = MagicMock()
reason_code.is_failure = True
mock_client.subscribe.reset_mock()
on_connect_cb(mock_client, None, MagicMock(), reason_code, None)
mock_client.subscribe.assert_not_called()
# ---------------------------------------------------------------------------
# Existing publish() is not regressed
# ---------------------------------------------------------------------------
def test_publish_still_works_after_subscribe_registered() -> None:
"""Adding a subscription must not break publish() behaviour."""
manager = MqttManager()
mock_client = MagicMock()
mock_client.connect.return_value = None
manager._client = mock_client
manager._connected = True
manager.subscribe("dsmr/json", MagicMock())
manager.publish("home/sensor", b"data", retain=True, qos=1)
mock_client.publish.assert_called_once_with(
"home/sensor", payload=b"data", qos=1, retain=True
)
def test_publish_skipped_when_not_connected_regression() -> None:
"""publish() must remain a no-op when not connected, even with subscriptions."""
manager = MqttManager()
manager.subscribe("dsmr/json", MagicMock())
# No client set up — publish should be silent
manager.publish("topic", b"payload") # must not raise