tliu93/home-automation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 17 Wiki Activity

M8-T04: reconcile DSMR ingest from meter sources

Browse Source
This commit is contained in:
tliu93
2026-08-23 21:22:05 +02:00
parent 28486a83c7
commit 2e125dbd53
9 changed files with 1074 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/config_page.py
-25
View File
@@ -129,27 +129,6 @@ CONFIG_FIELDS: tuple[ConfigField, ...] = (
"HA State Topic Prefix",
),
ConfigField("Modbus", "MODBUS_POLLING_ENABLED", "modbus_polling_enabled", "Modbus Polling Enabled", input_type="checkbox"),
ConfigField(
"DSMR",
"DSMR_INGEST_ENABLED",
"dsmr_ingest_enabled",
"DSMR Ingest Enabled",
input_type="checkbox",
),
ConfigField("DSMR", "DSMR_MQTT_TOPIC", "dsmr_mqtt_topic", "DSMR MQTT Topic"),
ConfigField(
"DSMR",
"DSMR_SAMPLE_INTERVAL_S",
"dsmr_sample_interval_s",
"DSMR Sample Interval (s)",
input_type="number",
),
ConfigField(
"DSMR",
"DSMR_TARIFF_TOPIC",
"dsmr_tariff_topic",
"DSMR Tariff Topic",
),
ConfigField(
"Tibber",
"TIBBER_API_TOKEN",
@@ -358,10 +337,6 @@ def _settings_payload(settings: Settings) -> dict[str, Any]:
"ha_discovery_enabled": settings.ha_discovery_enabled,
"ha_discovery_prefix": settings.ha_discovery_prefix,
"ha_state_topic_prefix": settings.ha_state_topic_prefix,
"dsmr_ingest_enabled": settings.dsmr_ingest_enabled,
"dsmr_mqtt_topic": settings.dsmr_mqtt_topic,
"dsmr_sample_interval_s": settings.dsmr_sample_interval_s,
"dsmr_tariff_topic": settings.dsmr_tariff_topic,
"tibber_api_token": settings.tibber_api_token,
"tibber_home_id": settings.tibber_home_id,
}
app/services/dsmr_ingest.py
+276 -247
View File
@@ -1,46 +1,21 @@
"""DSMR telegram ingest service.
Subscribes to the DSMR Reader MQTT topic (``dsmr/json``) and persists
down-sampled DSMR telegram frames to the ``dsmr_reading`` table.
Design decisions
----------------
- **Whole-frame storage**: the entire parsed telegram dict is stored as a JSON
blob in ``DsmrReading.payload``; no field allow-list is applied. This lets
future commodities (gas, heating, three-phase) be accommodated without a
table-schema change.
- **10-second down-sampling** (configurable via ``dsmr_sample_interval_s``):
only telegrams whose ``timestamp`` second falls on an exact multiple of the
interval are persisted. This reduces write volume from ~60 rows/min to ~6
rows/min while guaranteeing that every 15-minute boundary (second=00) is
captured.
- **Idempotency**: the telegram's own ``id`` field is stored as ``source_id``
with a UNIQUE constraint. A second delivery of the same telegram (e.g. after
a broker reconnect) is silently skipped.
- **Network-thread safety**: ``handle_message`` is called from paho's background
loop thread. It opens and closes its own short-lived SQLAlchemy session and
swallows all exceptions so that a buggy payload or transient DB error never
crashes the paho loop or drops the MQTT connection.
- **Numeric values kept as strings**: the DSMR Reader emits all numeric readings
as JSON strings (e.g. ``"20915.154"``). They are stored verbatim; conversion
to ``Decimal`` is deferred to the billing engine (T07) where precision matters.
- **Null phases**: some telegrams omit certain phase readings (``null`` in JSON);
these are stored as-is without special handling.
"""
"""DSMR MQTT ingest, keyed by durable meter-source identity."""
from __future__ import annotations
import json
import logging
import threading
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import TYPE_CHECKING
import sqlalchemy.exc
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.db import get_session_local
from app.models.energy import DsmrReading
from app.models.energy import DsmrReading, Meter
from app.models.meter_source import MeterSource, MeterSourceBinding, MeterSourceChannel
if TYPE_CHECKING:
from app.config import Settings
@@ -48,250 +23,304 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# Tracks the DSMR topic currently subscribed via the MQTT manager, so a config
# change can unsubscribe the old topic before subscribing the new one.
_current_dsmr_topic: str | None = None
@dataclass(frozen=True, slots=True)
class DsmrSourceSnapshot:
"""The only DSMR runtime configuration a network handler may use."""
# Tracks the DSMR tariff topic currently subscribed via the MQTT manager.
_current_tariff_topic: str | None = None
source_id: int
topic: str
tariff_topic: str
sample_interval_s: int
broker_host: str = ""
broker_port: int = 1883
username: str = ""
password: str = ""
tls_enabled: bool = False
# Current electricity tariff: 1 = dal/off-peak, 2 = normal/peak.
# Written by the paho network thread, read by the publish job — guarded by a lock.
_current_tariff: int | None = None
_subscriptions: dict[int, DsmrSourceSnapshot] = {}
_subscription_lock = threading.RLock()
# A configuration value is not an ownership identity: disable and re-enable
# can produce an equal snapshot. Each installed handler therefore captures a
# fresh token and verifies object identity before it can write or update tariff.
_subscription_tokens: dict[int, object] = {}
_reconcile_lock = threading.RLock()
_tariffs: dict[int, int] = {}
_tariff_lock = threading.Lock()
# Kept only for legacy direct test callers of set_current_tariff(value). Runtime
# handlers never write this value; production callers resolve a binding first.
_current_tariff: int | None = None
_UNSET = object()
def get_current_tariff() -> int | None:
"""Return the most recently received electricity tariff (1 or 2), or None."""
with _tariff_lock:
def get_current_tariff(meter_source_id: int | None = None) -> int | None:
"""Return a source tariff, or resolve the active electricity binding.
The no-argument form is retained for the pre-M8 expose integration. It
opens a short session to select the current electricity binding, so a
source's MQTT callback can never make another source's tariff current.
``_current_tariff`` is solely a test-era fallback when no binding database
is available; runtime MQTT handlers do not update it.
"""
if meter_source_id is None:
session_local = get_session_local()
session = session_local()
try:
source_id = _current_electricity_source_id(session, datetime.now(timezone.utc))
if source_id is not None:
return get_current_tariff(source_id)
except Exception:
logger.debug("DSMR legacy tariff lookup could not resolve a binding", exc_info=True)
finally:
session.close()
return _current_tariff
def set_current_tariff(value: int | None) -> None:
"""Set the current electricity tariff (1 or 2), or clear it with None."""
with _tariff_lock:
global _current_tariff
_current_tariff = value
return _tariffs.get(meter_source_id)
def handle_tariff_message(payload_bytes: bytes) -> None:
"""Parse one DSMR tariff MQTT payload and update the in-memory tariff state.
Called from the paho network thread; *must* swallow all exceptions so that
a bad payload never crashes the loop or drops the broker connection.
Accepts payload as bytes or str (paho can deliver either). Ignores
whitespace. Only accepts integer values 1 or 2; anything else is discarded
and the previous known tariff is preserved.
Parameters
----------
payload_bytes:
Raw bytes from the MQTT message (may also be a str in some paho versions).
"""
try:
# Decode bytes → str if needed; strip surrounding whitespace.
if isinstance(payload_bytes, (bytes, bytearray)):
raw = payload_bytes.decode("utf-8", errors="replace").strip()
def set_current_tariff(meter_source_id: int, value: int | None | object = _UNSET) -> None:
"""Set or clear an individual source's tariff state."""
global _current_tariff
if value is _UNSET:
# Compatibility with older direct callers. Do not route runtime source
# updates through this global fallback.
_current_tariff = meter_source_id if meter_source_id in (1, 2) else None
return
with _tariff_lock:
if value is None:
_tariffs.pop(meter_source_id, None)
else:
raw = str(payload_bytes).strip()
_tariffs[meter_source_id] = value
def _current_electricity_source_id(session: Session, at: datetime) -> int | None:
"""Return the DSMR source bound to electricity at ``at``, if any."""
return session.scalar(
select(MeterSource.id)
.join(MeterSourceChannel, MeterSourceChannel.source_id == MeterSource.id)
.join(MeterSourceBinding, MeterSourceBinding.channel_id == MeterSourceChannel.id)
.join(Meter, Meter.id == MeterSourceBinding.meter_id)
.where(
Meter.commodity == "electricity",
MeterSourceBinding.started_at <= at,
(MeterSourceBinding.ended_at.is_(None)) | (MeterSourceBinding.ended_at > at),
)
.order_by(MeterSourceBinding.started_at.desc())
.limit(1)
)
def get_current_electricity_tariff(session: Session, at: datetime | None = None) -> int | None:
"""Resolve tariff through the current electricity binding, never globally."""
source_id = _current_electricity_source_id(session, at or datetime.now(timezone.utc))
if source_id is None:
return None
return get_current_tariff(source_id)
def handle_tariff_message(payload_bytes: bytes, meter_source_id: int) -> None:
"""Parse one source's tariff payload without raising in paho's thread."""
try:
raw = (
payload_bytes.decode("utf-8", errors="replace").strip()
if isinstance(payload_bytes, (bytes, bytearray))
else str(payload_bytes).strip()
)
value = int(raw)
if value not in (1, 2):
logger.debug(
"dsmr_ingest.handle_tariff_message: unexpected tariff value %d (expected 1 or 2, ignored).",
value,
)
return
set_current_tariff(value)
logger.debug("dsmr_ingest.handle_tariff_message: tariff updated to %d.", value)
if value in (1, 2):
set_current_tariff(meter_source_id, value)
except Exception:
# Malformed payload (e.g. non-numeric); swallow silently to protect network thread.
logger.debug(
"dsmr_ingest.handle_tariff_message: could not parse payload %r (ignored).",
payload_bytes,
)
logger.debug("DSMR tariff payload ignored for source_id=%s", meter_source_id)
def apply_dsmr_subscription(settings: "Settings") -> None:
"""(Re)apply the DSMR MQTT subscriptions to match *settings* — restart-free.
Call this at startup and after every config save. It makes the live MQTT
subscriptions reflect the current ``dsmr_ingest_enabled`` / ``dsmr_mqtt_topic``
/ ``dsmr_sample_interval_s`` / ``dsmr_tariff_topic`` settings without an app
restart:
- **Disabled** → unsubscribe any active DSMR and tariff subscriptions.
- **Enabled** → (re)subscribe to ``dsmr_mqtt_topic`` with a handler bound to
a *fresh* settings snapshot, so a changed sample interval also takes effect.
Also subscribe to ``dsmr_tariff_topic`` when non-empty.
- **Topic changed** → unsubscribe the old topic before subscribing the new one.
Idempotent and safe to call when MQTT is not connected (the subscription is
queued in the manager and established on the next connect).
"""
# Imported here (not at module top) to avoid a circular import at app start.
from app.integrations.mqtt import mqtt_manager
global _current_dsmr_topic, _current_tariff_topic
if not settings.dsmr_ingest_enabled:
if _current_dsmr_topic is not None:
mqtt_manager.unsubscribe(_current_dsmr_topic)
logger.info("DSMR ingest disabled — unsubscribed from topic=%s.", _current_dsmr_topic)
_current_dsmr_topic = None
if _current_tariff_topic is not None:
mqtt_manager.unsubscribe(_current_tariff_topic)
logger.info(
"DSMR ingest disabled — unsubscribed from tariff topic=%s.",
_current_tariff_topic,
)
_current_tariff_topic = None
return
# --- Main DSMR telegram topic ---
topic = settings.dsmr_mqtt_topic
if _current_dsmr_topic is not None and _current_dsmr_topic != topic:
mqtt_manager.unsubscribe(_current_dsmr_topic)
# Re-subscribe (overwrites any existing handler for this topic) with a fresh
# settings snapshot so dsmr_sample_interval_s changes take effect too.
snapshot = settings
mqtt_manager.subscribe(topic, lambda payload: handle_message(payload, snapshot))
_current_dsmr_topic = topic
logger.info("DSMR ingest enabled — subscribed to topic=%s.", topic)
# --- DSMR tariff topic (dual-tariff slot indicator) ---
tariff_topic = settings.dsmr_tariff_topic if settings.dsmr_tariff_topic else ""
if tariff_topic:
if _current_tariff_topic is not None and _current_tariff_topic != tariff_topic:
mqtt_manager.unsubscribe(_current_tariff_topic)
mqtt_manager.subscribe(tariff_topic, lambda payload: handle_tariff_message(payload))
_current_tariff_topic = tariff_topic
logger.info("DSMR tariff topic — subscribed to topic=%s.", tariff_topic)
else:
# tariff_topic is empty → unsubscribe any existing tariff subscription.
if _current_tariff_topic is not None:
mqtt_manager.unsubscribe(_current_tariff_topic)
logger.info(
"DSMR tariff topic cleared — unsubscribed from topic=%s.",
_current_tariff_topic,
)
_current_tariff_topic = None
def _snapshot(source: MeterSource) -> DsmrSourceSnapshot:
config = source.config
return DsmrSourceSnapshot(
source_id=source.id,
topic=str(config.get("topic", "dsmr/json")),
tariff_topic=str(config.get("tariff_topic", "")),
sample_interval_s=int(config.get("sample_interval_s", 10)),
broker_host=str(config.get("broker_host", "")),
broker_port=int(config.get("broker_port", 1883)),
username=str(config.get("username", "")),
password=str(config.get("password", "")),
tls_enabled=bool(config.get("tls_enabled", False)),
)
def handle_message(payload_bytes: bytes, settings: "Settings") -> None:
"""Parse one DSMR MQTT payload and persist it if it passes the sample filter.
Called from the paho network thread; *must* swallow all exceptions so that
a bad payload or transient error does not crash the loop or drop the broker
connection.
Parameters
----------
payload_bytes:
Raw bytes from the MQTT message.
settings:
Runtime settings snapshot (captured at subscription time). Used for
``dsmr_sample_interval_s``.
"""
try:
_handle_message_inner(payload_bytes, settings)
except Exception:
logger.exception("dsmr_ingest.handle_message: unexpected error (swallowed).")
def _handle_message_inner(payload_bytes: bytes, settings: "Settings") -> None:
"""Inner implementation — may raise; caller wraps in try/except."""
# --- 1. Parse JSON ---
try:
data: dict = json.loads(payload_bytes)
except (json.JSONDecodeError, ValueError):
logger.debug("dsmr_ingest: invalid JSON payload (skipped).")
return
if not isinstance(data, dict):
logger.debug("dsmr_ingest: payload is not a JSON object (skipped).")
return
# --- 2. Parse timestamp ---
raw_ts = data.get("timestamp")
if raw_ts is None:
logger.debug("dsmr_ingest: missing 'timestamp' field (skipped).")
return
try:
# Python 3.11+ accepts the trailing 'Z' directly; for 3.10 compat we
# replace 'Z' with '+00:00' before parsing.
ts_str = raw_ts if not isinstance(raw_ts, str) else raw_ts.replace("Z", "+00:00")
ts_utc: datetime = datetime.fromisoformat(ts_str)
# Ensure it is timezone-aware UTC.
if ts_utc.tzinfo is None:
ts_utc = ts_utc.replace(tzinfo=timezone.utc)
except (ValueError, TypeError, AttributeError):
logger.debug(
"dsmr_ingest: cannot parse 'timestamp' value %r (skipped).", raw_ts
)
return
# --- 3. Down-sample: only persist if second falls on interval boundary ---
interval = settings.dsmr_sample_interval_s
if interval > 0 and (ts_utc.second % interval) != 0:
# This telegram is between sample points; discard silently.
return
# --- 4. Extract source_id (telegram's own id) — stored only as a reference,
# NOT used for uniqueness/idempotency (it overflows and gets reset). ---
source_id: int | None = data.get("id")
if source_id is not None and not isinstance(source_id, int):
# Unexpected type — treat as missing rather than raising.
logger.debug(
"dsmr_ingest: 'id' field has unexpected type %s (ignoring).",
type(source_id).__name__,
)
source_id = None
# --- 5. Persist to database ---
# Idempotency is keyed on recorded_at (the telegram timestamp), which is
# telegram-id-independent: a single P1 meter emits one telegram per second,
# and down-sampling keeps at most one per interval-aligned second. The
# UNIQUE(recorded_at) constraint is the backstop for the IntegrityError race.
def _enabled_snapshots() -> list[DsmrSourceSnapshot]:
session_local = get_session_local()
session = session_local()
try:
existing = session.scalar(
select(DsmrReading).where(DsmrReading.recorded_at == ts_utc)
)
if existing is not None:
logger.debug(
"dsmr_ingest: recorded_at=%s already in DB, skipping.",
ts_utc.isoformat(),
sources = session.scalars(
select(MeterSource).where(MeterSource.kind == "dsmr_mqtt", MeterSource.enabled.is_(True))
).all()
return [_snapshot(source) for source in sources]
finally:
session.close()
def apply_dsmr_subscription(_settings: "Settings | None" = None) -> None:
"""Reconcile enabled DSMR source subscriptions from the database.
The optional, ignored settings parameter preserves the old config-route
call shape while ensuring flat DSMR settings are no longer read.
"""
from app.integrations.mqtt import mqtt_manager
try:
desired = {snapshot.source_id: snapshot for snapshot in _enabled_snapshots()}
except Exception:
logger.exception("DSMR subscription reconcile failed while reading sources")
return
# Do not retain _subscription_lock while stopping MQTT clients: a callback
# may currently hold it through its complete dispatch, and MqttManager
# waits for that callback before teardown returns.
with _reconcile_lock:
# Every source owns a distinct MQTT client, so equal topics from
# different sources/brokers are dispatchable. A telegram and tariff
# topic on the *same* client would overwrite one handler, however.
for source_id, snapshot in list(desired.items()):
if snapshot.tariff_topic and snapshot.topic == snapshot.tariff_topic:
logger.error("DSMR source_id=%s rejected: telegram/tariff topic collision", source_id)
desired.pop(source_id)
with _subscription_lock:
stale_source_ids = [
source_id
for source_id, current in _subscriptions.items()
if desired.get(source_id) != current
]
for source_id in stale_source_ids:
_subscriptions.pop(source_id, None)
_subscription_tokens.pop(source_id, None)
set_current_tariff(source_id, None)
for source_id in stale_source_ids:
mqtt_manager.remove_source(source_id)
for source_id, snapshot in desired.items():
with _subscription_lock:
current = _subscriptions.get(source_id)
if current == snapshot and mqtt_manager.source_is_active(source_id):
continue
if current is not None:
# The client went inactive outside reconcile. Invalidate its
# old token before rebuilding the same snapshot.
with _subscription_lock:
if _subscriptions.get(source_id) == current:
_subscriptions.pop(source_id, None)
_subscription_tokens.pop(source_id, None)
mqtt_manager.remove_source(source_id)
token = object()
handlers = {
snapshot.topic: lambda payload, captured=snapshot, captured_token=token: (
handle_captured_message(payload, captured, captured_token)
)
}
if snapshot.tariff_topic:
handlers[snapshot.tariff_topic] = (
lambda payload, captured=snapshot, captured_token=token: (
handle_captured_tariff_message(payload, captured, captured_token)
)
)
with _subscription_lock:
_subscriptions[source_id] = snapshot
_subscription_tokens[source_id] = token
applied = mqtt_manager.replace_source(
source_id,
host=snapshot.broker_host,
port=snapshot.broker_port,
username=snapshot.username,
password=snapshot.password,
tls_enabled=snapshot.tls_enabled,
subscriptions=handlers,
)
if not applied:
with _subscription_lock:
if _subscription_tokens.get(source_id) is token:
_subscriptions.pop(source_id, None)
_subscription_tokens.pop(source_id, None)
def handle_message(payload_bytes: bytes, snapshot: DsmrSourceSnapshot) -> None:
"""Persist one down-sampled frame under its captured source identity."""
try:
_handle_message_inner(payload_bytes, snapshot)
except Exception:
logger.exception("DSMR ingest handler failed for source_id=%s (swallowed)", snapshot.source_id)
def handle_captured_message(
payload_bytes: bytes, snapshot: DsmrSourceSnapshot, token: object | None = None
) -> None:
"""Run a broker callback only while its exact source generation is active."""
with _subscription_lock:
if token is not None:
if _subscription_tokens.get(snapshot.source_id) is not token:
return
elif _subscriptions.get(snapshot.source_id) != snapshot:
return
handle_message(payload_bytes, snapshot)
def handle_captured_tariff_message(
payload_bytes: bytes, snapshot: DsmrSourceSnapshot, token: object | None = None
) -> None:
"""Ignore tariff callbacks retained from a removed/replaced source."""
with _subscription_lock:
if token is not None:
if _subscription_tokens.get(snapshot.source_id) is not token:
return
elif _subscriptions.get(snapshot.source_id) != snapshot:
return
handle_tariff_message(payload_bytes, snapshot.source_id)
reading = DsmrReading(
recorded_at=ts_utc,
source_id=source_id,
payload=data, # full frame, verbatim
def _handle_message_inner(payload_bytes: bytes, snapshot: DsmrSourceSnapshot) -> None:
try:
data = json.loads(payload_bytes)
except (json.JSONDecodeError, ValueError):
return
if not isinstance(data, dict):
return
try:
raw_ts = data["timestamp"]
ts_utc = datetime.fromisoformat(raw_ts.replace("Z", "+00:00"))
if ts_utc.tzinfo is None:
ts_utc = ts_utc.replace(tzinfo=timezone.utc)
except (KeyError, ValueError, TypeError, AttributeError):
return
if snapshot.sample_interval_s > 0 and ts_utc.second % snapshot.sample_interval_s:
return
telegram_id = data.get("id")
if telegram_id is not None and not isinstance(telegram_id, int):
telegram_id = None
session_local = get_session_local()
session = session_local()
try:
exists = session.scalar(
select(DsmrReading.id).where(
DsmrReading.meter_source_id == snapshot.source_id,
DsmrReading.recorded_at == ts_utc,
)
)
session.add(reading)
session.commit()
logger.debug(
"dsmr_ingest: persisted reading recorded_at=%s source_id=%s.",
ts_utc.isoformat(),
source_id,
)
if exists is None:
session.add(
DsmrReading(
meter_source_id=snapshot.source_id,
recorded_at=ts_utc,
telegram_id=telegram_id,
payload=data,
)
)
session.commit()
except sqlalchemy.exc.IntegrityError:
# Race / duplicate: another insert beat us to the same recorded_at.
session.rollback()
logger.debug(
"dsmr_ingest: IntegrityError for recorded_at=%s (duplicate, skipped).",
ts_utc.isoformat(),
)
except Exception:
session.rollback()
logger.exception("dsmr_ingest: DB error (swallowed).")
logger.exception("DSMR database write failed for source_id=%s", snapshot.source_id)
finally:
session.close()