M8-T04: reconcile DSMR ingest from meter sources
This commit is contained in:
+198
-2
@@ -30,6 +30,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import paho.mqtt.client as mqtt
|
||||
@@ -50,6 +51,15 @@ MQTT_SETTINGS_KEYS = {
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SourceClientState:
|
||||
"""One installed source generation and its in-flight callback count."""
|
||||
|
||||
client: mqtt.Client
|
||||
generation: int
|
||||
in_flight: int = 0
|
||||
|
||||
|
||||
def _is_configured(settings: Settings) -> bool:
|
||||
"""Return True if MQTT is enabled *and* the broker host is set."""
|
||||
return bool(settings.mqtt_enabled and settings.mqtt_broker_host)
|
||||
@@ -72,11 +82,26 @@ class MqttManager:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._client: mqtt.Client | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._lock = threading.RLock()
|
||||
# Source replacement/removal is serialized independently from callback
|
||||
# bookkeeping. In particular, paho's loop_stop() joins its network
|
||||
# thread, whose callback completion also needs ``_lock``.
|
||||
self._source_lifecycle_lock = threading.Lock()
|
||||
self._source_idle = threading.Condition(self._lock)
|
||||
self._connected = False
|
||||
# topic → handler registry; persists across reconnects so subscriptions
|
||||
# are automatically re-established when the client reconnects.
|
||||
self._subscriptions: dict[str, Callable[[bytes], None]] = {}
|
||||
# DSMR sources are independent connections: their credentials and TLS
|
||||
# configuration belong to MeterSource.config, not app_config.
|
||||
self._source_clients: dict[int, mqtt.Client] = {}
|
||||
self._source_subscriptions: dict[int, dict[str, Callable[[bytes], None]]] = {}
|
||||
self._source_connected: set[int] = set()
|
||||
# Each replacement gets a distinct identity. A paho callback can run
|
||||
# after its client was stopped, so source id alone is not sufficient.
|
||||
self._source_generations: dict[int, int] = {}
|
||||
self._source_states: dict[int, _SourceClientState] = {}
|
||||
self._next_source_generation = 0
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public properties
|
||||
@@ -115,6 +140,11 @@ class MqttManager:
|
||||
"""
|
||||
with self._lock:
|
||||
self._stop_client()
|
||||
with self._source_lifecycle_lock:
|
||||
with self._lock:
|
||||
source_ids = list(self._source_clients)
|
||||
for source_id in source_ids:
|
||||
self._stop_source_client(source_id)
|
||||
|
||||
def reconnect(self, settings: Settings) -> None:
|
||||
"""Disconnect the current client (if any) and reconnect with *settings*.
|
||||
@@ -196,8 +226,134 @@ class MqttManager:
|
||||
except Exception:
|
||||
logger.exception("MQTT unsubscribe error (topic=%s).", topic)
|
||||
|
||||
def replace_source(
|
||||
self,
|
||||
source_id: int,
|
||||
*,
|
||||
host: str,
|
||||
port: int,
|
||||
username: str,
|
||||
password: str,
|
||||
tls_enabled: bool,
|
||||
subscriptions: dict[str, Callable[[bytes], None]],
|
||||
) -> bool:
|
||||
"""Replace one source-owned client and its handlers.
|
||||
|
||||
This intentionally does not touch the legacy app-wide client or any
|
||||
other source client. It is also safe for a source to be temporarily
|
||||
unconfigured: handlers are retained in the source registry but no
|
||||
connection is attempted until a host is supplied.
|
||||
"""
|
||||
with self._source_lifecycle_lock:
|
||||
self._stop_source_client(source_id)
|
||||
if not host:
|
||||
return False
|
||||
self._next_source_generation += 1
|
||||
generation = self._next_source_generation
|
||||
captured_subscriptions = dict(subscriptions)
|
||||
client = mqtt.Client(
|
||||
callback_api_version=mqtt.CallbackAPIVersion.VERSION2,
|
||||
client_id=f"home-automation-dsmr-{source_id}",
|
||||
)
|
||||
|
||||
def _on_connect(
|
||||
connected_client: mqtt.Client,
|
||||
_userdata: object,
|
||||
_flags: mqtt.ConnectFlags,
|
||||
reason_code: mqtt.ReasonCode,
|
||||
_properties: mqtt.Properties | None,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
if not self._is_current_source_client(source_id, generation, connected_client):
|
||||
return
|
||||
if reason_code.is_failure:
|
||||
logger.warning("DSMR MQTT connection refused for source_id=%s", source_id)
|
||||
return
|
||||
self._source_connected.add(source_id)
|
||||
for topic in captured_subscriptions:
|
||||
try:
|
||||
connected_client.subscribe(topic)
|
||||
except Exception:
|
||||
logger.exception("DSMR MQTT re-subscribe failed for source_id=%s", source_id)
|
||||
|
||||
def _on_disconnect(
|
||||
disconnected_client: mqtt.Client,
|
||||
_userdata: object,
|
||||
_flags: mqtt.DisconnectFlags,
|
||||
_reason_code: mqtt.ReasonCode,
|
||||
_properties: mqtt.Properties | None,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
if not self._is_current_source_client(source_id, generation, disconnected_client):
|
||||
return
|
||||
self._source_connected.discard(source_id)
|
||||
|
||||
def _on_message(
|
||||
message_client: mqtt.Client,
|
||||
_userdata: object,
|
||||
message: mqtt.MQTTMessage,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
if not self._is_current_source_client(source_id, generation, message_client):
|
||||
return
|
||||
handler = captured_subscriptions.get(message.topic)
|
||||
state = self._source_states.get(source_id)
|
||||
if handler is None or state is None:
|
||||
return
|
||||
# This permit covers the entire handler call. Teardown first
|
||||
# invalidates the state and then waits for all permits, so an
|
||||
# old callback cannot run after teardown returns.
|
||||
state.in_flight += 1
|
||||
try:
|
||||
handler(message.payload)
|
||||
except Exception:
|
||||
logger.exception("DSMR source handler raised (source_id=%s)", source_id)
|
||||
finally:
|
||||
with self._lock:
|
||||
state.in_flight -= 1
|
||||
if state.in_flight == 0:
|
||||
self._source_idle.notify_all()
|
||||
|
||||
client.on_connect = _on_connect
|
||||
client.on_disconnect = _on_disconnect
|
||||
client.on_message = _on_message
|
||||
if tls_enabled:
|
||||
try:
|
||||
client.tls_set()
|
||||
except Exception:
|
||||
logger.exception("DSMR MQTT TLS setup failed for source_id=%s", source_id)
|
||||
return False
|
||||
if username:
|
||||
client.username_pw_set(username=username, password=password or None)
|
||||
# Register ownership before network processing begins. A broker
|
||||
# may deliver CONNACK synchronously from connect(), or on the loop
|
||||
# thread before connect() returns.
|
||||
with self._lock:
|
||||
self._source_clients[source_id] = client
|
||||
self._source_subscriptions[source_id] = captured_subscriptions
|
||||
self._source_generations[source_id] = generation
|
||||
self._source_states[source_id] = _SourceClientState(client, generation)
|
||||
client.loop_start()
|
||||
try:
|
||||
client.connect(host=host, port=port, keepalive=60)
|
||||
except Exception:
|
||||
logger.exception("DSMR MQTT connect failed (source_id=%s, host=%s)", source_id, host)
|
||||
self._stop_source_client(source_id)
|
||||
return False
|
||||
return True
|
||||
|
||||
def remove_source(self, source_id: int) -> None:
|
||||
"""Drop one source client and its handlers, including queued callbacks."""
|
||||
with self._source_lifecycle_lock:
|
||||
self._stop_source_client(source_id)
|
||||
|
||||
def source_is_active(self, source_id: int) -> bool:
|
||||
"""Whether a source-owned client is currently installed for callbacks."""
|
||||
with self._lock:
|
||||
return source_id in self._source_clients
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers — must be called with self._lock held
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _start_client(self, settings: Settings) -> None:
|
||||
@@ -333,6 +489,46 @@ class MqttManager:
|
||||
logger.debug("MQTT loop_stop raised (ignoring).", exc_info=True)
|
||||
logger.info("MQTT client stopped.")
|
||||
|
||||
def _stop_source_client(self, source_id: int) -> None:
|
||||
"""Detach then stop a source client without blocking callback bookkeeping.
|
||||
|
||||
Callers hold ``_source_lifecycle_lock``. The first phase makes the
|
||||
generation unreachable while holding ``_lock``. Paho operations and
|
||||
the in-flight wait are deliberately outside that lock: loop_stop()
|
||||
joins paho's network thread, and an active callback needs ``_lock`` to
|
||||
release its permit in ``_on_message``'s finally block.
|
||||
"""
|
||||
with self._lock:
|
||||
state = self._source_states.pop(source_id, None)
|
||||
client = self._source_clients.pop(source_id, None)
|
||||
self._source_subscriptions.pop(source_id, None)
|
||||
self._source_connected.discard(source_id)
|
||||
# Invalidate callbacks even when there was no successfully
|
||||
# installed client (for example after a failed replacement).
|
||||
self._source_generations.pop(source_id, None)
|
||||
if client is not None:
|
||||
try:
|
||||
client.disconnect()
|
||||
except Exception:
|
||||
logger.debug("DSMR MQTT disconnect raised (source_id=%s)", source_id, exc_info=True)
|
||||
try:
|
||||
client.loop_stop()
|
||||
except Exception:
|
||||
logger.debug("DSMR MQTT loop_stop raised (source_id=%s)", source_id, exc_info=True)
|
||||
if state is not None:
|
||||
with self._lock:
|
||||
while state.in_flight:
|
||||
self._source_idle.wait()
|
||||
|
||||
def _is_current_source_client(
|
||||
self, source_id: int, generation: int, client: mqtt.Client
|
||||
) -> bool:
|
||||
"""Check callback ownership while ``_lock`` is held."""
|
||||
return (
|
||||
self._source_generations.get(source_id) == generation
|
||||
and self._source_clients.get(source_id) is client
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level singleton — shared across lifespan and route handlers
|
||||
|
||||
+3
-4
@@ -274,10 +274,9 @@ async def lifespan(_: FastAPI):
|
||||
_startup_session.close()
|
||||
mqtt_manager.connect(_startup_runtime_settings)
|
||||
|
||||
# DSMR ingest: subscribe to the configured MQTT topic when enabled. The same
|
||||
# applier is called from PUT /api/config, so toggling DSMR via the UI takes
|
||||
# effect without an app restart.
|
||||
apply_dsmr_subscription(_startup_runtime_settings)
|
||||
# DSMR sources carry their own runtime configuration and are reconciled
|
||||
# after the MQTT manager is connected.
|
||||
apply_dsmr_subscription()
|
||||
|
||||
yield
|
||||
|
||||
|
||||
@@ -129,27 +129,6 @@ CONFIG_FIELDS: tuple[ConfigField, ...] = (
|
||||
"HA State Topic Prefix",
|
||||
),
|
||||
ConfigField("Modbus", "MODBUS_POLLING_ENABLED", "modbus_polling_enabled", "Modbus Polling Enabled", input_type="checkbox"),
|
||||
ConfigField(
|
||||
"DSMR",
|
||||
"DSMR_INGEST_ENABLED",
|
||||
"dsmr_ingest_enabled",
|
||||
"DSMR Ingest Enabled",
|
||||
input_type="checkbox",
|
||||
),
|
||||
ConfigField("DSMR", "DSMR_MQTT_TOPIC", "dsmr_mqtt_topic", "DSMR MQTT Topic"),
|
||||
ConfigField(
|
||||
"DSMR",
|
||||
"DSMR_SAMPLE_INTERVAL_S",
|
||||
"dsmr_sample_interval_s",
|
||||
"DSMR Sample Interval (s)",
|
||||
input_type="number",
|
||||
),
|
||||
ConfigField(
|
||||
"DSMR",
|
||||
"DSMR_TARIFF_TOPIC",
|
||||
"dsmr_tariff_topic",
|
||||
"DSMR Tariff Topic",
|
||||
),
|
||||
ConfigField(
|
||||
"Tibber",
|
||||
"TIBBER_API_TOKEN",
|
||||
@@ -358,10 +337,6 @@ def _settings_payload(settings: Settings) -> dict[str, Any]:
|
||||
"ha_discovery_enabled": settings.ha_discovery_enabled,
|
||||
"ha_discovery_prefix": settings.ha_discovery_prefix,
|
||||
"ha_state_topic_prefix": settings.ha_state_topic_prefix,
|
||||
"dsmr_ingest_enabled": settings.dsmr_ingest_enabled,
|
||||
"dsmr_mqtt_topic": settings.dsmr_mqtt_topic,
|
||||
"dsmr_sample_interval_s": settings.dsmr_sample_interval_s,
|
||||
"dsmr_tariff_topic": settings.dsmr_tariff_topic,
|
||||
"tibber_api_token": settings.tibber_api_token,
|
||||
"tibber_home_id": settings.tibber_home_id,
|
||||
}
|
||||
|
||||
+276
-247
@@ -1,46 +1,21 @@
|
||||
"""DSMR telegram ingest service.
|
||||
|
||||
Subscribes to the DSMR Reader MQTT topic (``dsmr/json``) and persists
|
||||
down-sampled DSMR telegram frames to the ``dsmr_reading`` table.
|
||||
|
||||
Design decisions
|
||||
----------------
|
||||
- **Whole-frame storage**: the entire parsed telegram dict is stored as a JSON
|
||||
blob in ``DsmrReading.payload``; no field allow-list is applied. This lets
|
||||
future commodities (gas, heating, three-phase) be accommodated without a
|
||||
table-schema change.
|
||||
- **10-second down-sampling** (configurable via ``dsmr_sample_interval_s``):
|
||||
only telegrams whose ``timestamp`` second falls on an exact multiple of the
|
||||
interval are persisted. This reduces write volume from ~60 rows/min to ~6
|
||||
rows/min while guaranteeing that every 15-minute boundary (second=00) is
|
||||
captured.
|
||||
- **Idempotency**: the telegram's own ``id`` field is stored as ``source_id``
|
||||
with a UNIQUE constraint. A second delivery of the same telegram (e.g. after
|
||||
a broker reconnect) is silently skipped.
|
||||
- **Network-thread safety**: ``handle_message`` is called from paho's background
|
||||
loop thread. It opens and closes its own short-lived SQLAlchemy session and
|
||||
swallows all exceptions so that a buggy payload or transient DB error never
|
||||
crashes the paho loop or drops the MQTT connection.
|
||||
- **Numeric values kept as strings**: the DSMR Reader emits all numeric readings
|
||||
as JSON strings (e.g. ``"20915.154"``). They are stored verbatim; conversion
|
||||
to ``Decimal`` is deferred to the billing engine (T07) where precision matters.
|
||||
- **Null phases**: some telegrams omit certain phase readings (``null`` in JSON);
|
||||
these are stored as-is without special handling.
|
||||
"""
|
||||
"""DSMR MQTT ingest, keyed by durable meter-source identity."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sqlalchemy.exc
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import get_session_local
|
||||
from app.models.energy import DsmrReading
|
||||
from app.models.energy import DsmrReading, Meter
|
||||
from app.models.meter_source import MeterSource, MeterSourceBinding, MeterSourceChannel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.config import Settings
|
||||
@@ -48,250 +23,304 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Tracks the DSMR topic currently subscribed via the MQTT manager, so a config
|
||||
# change can unsubscribe the old topic before subscribing the new one.
|
||||
_current_dsmr_topic: str | None = None
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DsmrSourceSnapshot:
|
||||
"""The only DSMR runtime configuration a network handler may use."""
|
||||
|
||||
# Tracks the DSMR tariff topic currently subscribed via the MQTT manager.
|
||||
_current_tariff_topic: str | None = None
|
||||
source_id: int
|
||||
topic: str
|
||||
tariff_topic: str
|
||||
sample_interval_s: int
|
||||
broker_host: str = ""
|
||||
broker_port: int = 1883
|
||||
username: str = ""
|
||||
password: str = ""
|
||||
tls_enabled: bool = False
|
||||
|
||||
# Current electricity tariff: 1 = dal/off-peak, 2 = normal/peak.
|
||||
# Written by the paho network thread, read by the publish job — guarded by a lock.
|
||||
_current_tariff: int | None = None
|
||||
|
||||
_subscriptions: dict[int, DsmrSourceSnapshot] = {}
|
||||
_subscription_lock = threading.RLock()
|
||||
# A configuration value is not an ownership identity: disable and re-enable
|
||||
# can produce an equal snapshot. Each installed handler therefore captures a
|
||||
# fresh token and verifies object identity before it can write or update tariff.
|
||||
_subscription_tokens: dict[int, object] = {}
|
||||
_reconcile_lock = threading.RLock()
|
||||
_tariffs: dict[int, int] = {}
|
||||
_tariff_lock = threading.Lock()
|
||||
# Kept only for legacy direct test callers of set_current_tariff(value). Runtime
|
||||
# handlers never write this value; production callers resolve a binding first.
|
||||
_current_tariff: int | None = None
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
def get_current_tariff() -> int | None:
|
||||
"""Return the most recently received electricity tariff (1 or 2), or None."""
|
||||
with _tariff_lock:
|
||||
def get_current_tariff(meter_source_id: int | None = None) -> int | None:
|
||||
"""Return a source tariff, or resolve the active electricity binding.
|
||||
|
||||
The no-argument form is retained for the pre-M8 expose integration. It
|
||||
opens a short session to select the current electricity binding, so a
|
||||
source's MQTT callback can never make another source's tariff current.
|
||||
``_current_tariff`` is solely a test-era fallback when no binding database
|
||||
is available; runtime MQTT handlers do not update it.
|
||||
"""
|
||||
if meter_source_id is None:
|
||||
session_local = get_session_local()
|
||||
session = session_local()
|
||||
try:
|
||||
source_id = _current_electricity_source_id(session, datetime.now(timezone.utc))
|
||||
if source_id is not None:
|
||||
return get_current_tariff(source_id)
|
||||
except Exception:
|
||||
logger.debug("DSMR legacy tariff lookup could not resolve a binding", exc_info=True)
|
||||
finally:
|
||||
session.close()
|
||||
return _current_tariff
|
||||
|
||||
|
||||
def set_current_tariff(value: int | None) -> None:
|
||||
"""Set the current electricity tariff (1 or 2), or clear it with None."""
|
||||
with _tariff_lock:
|
||||
global _current_tariff
|
||||
_current_tariff = value
|
||||
return _tariffs.get(meter_source_id)
|
||||
|
||||
|
||||
def handle_tariff_message(payload_bytes: bytes) -> None:
|
||||
"""Parse one DSMR tariff MQTT payload and update the in-memory tariff state.
|
||||
|
||||
Called from the paho network thread; *must* swallow all exceptions so that
|
||||
a bad payload never crashes the loop or drops the broker connection.
|
||||
|
||||
Accepts payload as bytes or str (paho can deliver either). Ignores
|
||||
whitespace. Only accepts integer values 1 or 2; anything else is discarded
|
||||
and the previous known tariff is preserved.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
payload_bytes:
|
||||
Raw bytes from the MQTT message (may also be a str in some paho versions).
|
||||
"""
|
||||
try:
|
||||
# Decode bytes → str if needed; strip surrounding whitespace.
|
||||
if isinstance(payload_bytes, (bytes, bytearray)):
|
||||
raw = payload_bytes.decode("utf-8", errors="replace").strip()
|
||||
def set_current_tariff(meter_source_id: int, value: int | None | object = _UNSET) -> None:
|
||||
"""Set or clear an individual source's tariff state."""
|
||||
global _current_tariff
|
||||
if value is _UNSET:
|
||||
# Compatibility with older direct callers. Do not route runtime source
|
||||
# updates through this global fallback.
|
||||
_current_tariff = meter_source_id if meter_source_id in (1, 2) else None
|
||||
return
|
||||
with _tariff_lock:
|
||||
if value is None:
|
||||
_tariffs.pop(meter_source_id, None)
|
||||
else:
|
||||
raw = str(payload_bytes).strip()
|
||||
_tariffs[meter_source_id] = value
|
||||
|
||||
|
||||
def _current_electricity_source_id(session: Session, at: datetime) -> int | None:
|
||||
"""Return the DSMR source bound to electricity at ``at``, if any."""
|
||||
return session.scalar(
|
||||
select(MeterSource.id)
|
||||
.join(MeterSourceChannel, MeterSourceChannel.source_id == MeterSource.id)
|
||||
.join(MeterSourceBinding, MeterSourceBinding.channel_id == MeterSourceChannel.id)
|
||||
.join(Meter, Meter.id == MeterSourceBinding.meter_id)
|
||||
.where(
|
||||
Meter.commodity == "electricity",
|
||||
MeterSourceBinding.started_at <= at,
|
||||
(MeterSourceBinding.ended_at.is_(None)) | (MeterSourceBinding.ended_at > at),
|
||||
)
|
||||
.order_by(MeterSourceBinding.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
|
||||
def get_current_electricity_tariff(session: Session, at: datetime | None = None) -> int | None:
|
||||
"""Resolve tariff through the current electricity binding, never globally."""
|
||||
source_id = _current_electricity_source_id(session, at or datetime.now(timezone.utc))
|
||||
if source_id is None:
|
||||
return None
|
||||
return get_current_tariff(source_id)
|
||||
|
||||
|
||||
def handle_tariff_message(payload_bytes: bytes, meter_source_id: int) -> None:
|
||||
"""Parse one source's tariff payload without raising in paho's thread."""
|
||||
try:
|
||||
raw = (
|
||||
payload_bytes.decode("utf-8", errors="replace").strip()
|
||||
if isinstance(payload_bytes, (bytes, bytearray))
|
||||
else str(payload_bytes).strip()
|
||||
)
|
||||
value = int(raw)
|
||||
if value not in (1, 2):
|
||||
logger.debug(
|
||||
"dsmr_ingest.handle_tariff_message: unexpected tariff value %d (expected 1 or 2, ignored).",
|
||||
value,
|
||||
)
|
||||
return
|
||||
|
||||
set_current_tariff(value)
|
||||
logger.debug("dsmr_ingest.handle_tariff_message: tariff updated to %d.", value)
|
||||
|
||||
if value in (1, 2):
|
||||
set_current_tariff(meter_source_id, value)
|
||||
except Exception:
|
||||
# Malformed payload (e.g. non-numeric); swallow silently to protect network thread.
|
||||
logger.debug(
|
||||
"dsmr_ingest.handle_tariff_message: could not parse payload %r (ignored).",
|
||||
payload_bytes,
|
||||
)
|
||||
logger.debug("DSMR tariff payload ignored for source_id=%s", meter_source_id)
|
||||
|
||||
|
||||
def apply_dsmr_subscription(settings: "Settings") -> None:
|
||||
"""(Re)apply the DSMR MQTT subscriptions to match *settings* — restart-free.
|
||||
|
||||
Call this at startup and after every config save. It makes the live MQTT
|
||||
subscriptions reflect the current ``dsmr_ingest_enabled`` / ``dsmr_mqtt_topic``
|
||||
/ ``dsmr_sample_interval_s`` / ``dsmr_tariff_topic`` settings without an app
|
||||
restart:
|
||||
|
||||
- **Disabled** → unsubscribe any active DSMR and tariff subscriptions.
|
||||
- **Enabled** → (re)subscribe to ``dsmr_mqtt_topic`` with a handler bound to
|
||||
a *fresh* settings snapshot, so a changed sample interval also takes effect.
|
||||
Also subscribe to ``dsmr_tariff_topic`` when non-empty.
|
||||
- **Topic changed** → unsubscribe the old topic before subscribing the new one.
|
||||
|
||||
Idempotent and safe to call when MQTT is not connected (the subscription is
|
||||
queued in the manager and established on the next connect).
|
||||
"""
|
||||
# Imported here (not at module top) to avoid a circular import at app start.
|
||||
from app.integrations.mqtt import mqtt_manager
|
||||
|
||||
global _current_dsmr_topic, _current_tariff_topic
|
||||
|
||||
if not settings.dsmr_ingest_enabled:
|
||||
if _current_dsmr_topic is not None:
|
||||
mqtt_manager.unsubscribe(_current_dsmr_topic)
|
||||
logger.info("DSMR ingest disabled — unsubscribed from topic=%s.", _current_dsmr_topic)
|
||||
_current_dsmr_topic = None
|
||||
if _current_tariff_topic is not None:
|
||||
mqtt_manager.unsubscribe(_current_tariff_topic)
|
||||
logger.info(
|
||||
"DSMR ingest disabled — unsubscribed from tariff topic=%s.",
|
||||
_current_tariff_topic,
|
||||
)
|
||||
_current_tariff_topic = None
|
||||
return
|
||||
|
||||
# --- Main DSMR telegram topic ---
|
||||
topic = settings.dsmr_mqtt_topic
|
||||
if _current_dsmr_topic is not None and _current_dsmr_topic != topic:
|
||||
mqtt_manager.unsubscribe(_current_dsmr_topic)
|
||||
|
||||
# Re-subscribe (overwrites any existing handler for this topic) with a fresh
|
||||
# settings snapshot so dsmr_sample_interval_s changes take effect too.
|
||||
snapshot = settings
|
||||
mqtt_manager.subscribe(topic, lambda payload: handle_message(payload, snapshot))
|
||||
_current_dsmr_topic = topic
|
||||
logger.info("DSMR ingest enabled — subscribed to topic=%s.", topic)
|
||||
|
||||
# --- DSMR tariff topic (dual-tariff slot indicator) ---
|
||||
tariff_topic = settings.dsmr_tariff_topic if settings.dsmr_tariff_topic else ""
|
||||
if tariff_topic:
|
||||
if _current_tariff_topic is not None and _current_tariff_topic != tariff_topic:
|
||||
mqtt_manager.unsubscribe(_current_tariff_topic)
|
||||
mqtt_manager.subscribe(tariff_topic, lambda payload: handle_tariff_message(payload))
|
||||
_current_tariff_topic = tariff_topic
|
||||
logger.info("DSMR tariff topic — subscribed to topic=%s.", tariff_topic)
|
||||
else:
|
||||
# tariff_topic is empty → unsubscribe any existing tariff subscription.
|
||||
if _current_tariff_topic is not None:
|
||||
mqtt_manager.unsubscribe(_current_tariff_topic)
|
||||
logger.info(
|
||||
"DSMR tariff topic cleared — unsubscribed from topic=%s.",
|
||||
_current_tariff_topic,
|
||||
)
|
||||
_current_tariff_topic = None
|
||||
def _snapshot(source: MeterSource) -> DsmrSourceSnapshot:
|
||||
config = source.config
|
||||
return DsmrSourceSnapshot(
|
||||
source_id=source.id,
|
||||
topic=str(config.get("topic", "dsmr/json")),
|
||||
tariff_topic=str(config.get("tariff_topic", "")),
|
||||
sample_interval_s=int(config.get("sample_interval_s", 10)),
|
||||
broker_host=str(config.get("broker_host", "")),
|
||||
broker_port=int(config.get("broker_port", 1883)),
|
||||
username=str(config.get("username", "")),
|
||||
password=str(config.get("password", "")),
|
||||
tls_enabled=bool(config.get("tls_enabled", False)),
|
||||
)
|
||||
|
||||
|
||||
def handle_message(payload_bytes: bytes, settings: "Settings") -> None:
|
||||
"""Parse one DSMR MQTT payload and persist it if it passes the sample filter.
|
||||
|
||||
Called from the paho network thread; *must* swallow all exceptions so that
|
||||
a bad payload or transient error does not crash the loop or drop the broker
|
||||
connection.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
payload_bytes:
|
||||
Raw bytes from the MQTT message.
|
||||
settings:
|
||||
Runtime settings snapshot (captured at subscription time). Used for
|
||||
``dsmr_sample_interval_s``.
|
||||
"""
|
||||
try:
|
||||
_handle_message_inner(payload_bytes, settings)
|
||||
except Exception:
|
||||
logger.exception("dsmr_ingest.handle_message: unexpected error (swallowed).")
|
||||
|
||||
|
||||
def _handle_message_inner(payload_bytes: bytes, settings: "Settings") -> None:
|
||||
"""Inner implementation — may raise; caller wraps in try/except."""
|
||||
# --- 1. Parse JSON ---
|
||||
try:
|
||||
data: dict = json.loads(payload_bytes)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
logger.debug("dsmr_ingest: invalid JSON payload (skipped).")
|
||||
return
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.debug("dsmr_ingest: payload is not a JSON object (skipped).")
|
||||
return
|
||||
|
||||
# --- 2. Parse timestamp ---
|
||||
raw_ts = data.get("timestamp")
|
||||
if raw_ts is None:
|
||||
logger.debug("dsmr_ingest: missing 'timestamp' field (skipped).")
|
||||
return
|
||||
|
||||
try:
|
||||
# Python 3.11+ accepts the trailing 'Z' directly; for 3.10 compat we
|
||||
# replace 'Z' with '+00:00' before parsing.
|
||||
ts_str = raw_ts if not isinstance(raw_ts, str) else raw_ts.replace("Z", "+00:00")
|
||||
ts_utc: datetime = datetime.fromisoformat(ts_str)
|
||||
# Ensure it is timezone-aware UTC.
|
||||
if ts_utc.tzinfo is None:
|
||||
ts_utc = ts_utc.replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
logger.debug(
|
||||
"dsmr_ingest: cannot parse 'timestamp' value %r (skipped).", raw_ts
|
||||
)
|
||||
return
|
||||
|
||||
# --- 3. Down-sample: only persist if second falls on interval boundary ---
|
||||
interval = settings.dsmr_sample_interval_s
|
||||
if interval > 0 and (ts_utc.second % interval) != 0:
|
||||
# This telegram is between sample points; discard silently.
|
||||
return
|
||||
|
||||
# --- 4. Extract source_id (telegram's own id) — stored only as a reference,
|
||||
# NOT used for uniqueness/idempotency (it overflows and gets reset). ---
|
||||
source_id: int | None = data.get("id")
|
||||
if source_id is not None and not isinstance(source_id, int):
|
||||
# Unexpected type — treat as missing rather than raising.
|
||||
logger.debug(
|
||||
"dsmr_ingest: 'id' field has unexpected type %s (ignoring).",
|
||||
type(source_id).__name__,
|
||||
)
|
||||
source_id = None
|
||||
|
||||
# --- 5. Persist to database ---
|
||||
# Idempotency is keyed on recorded_at (the telegram timestamp), which is
|
||||
# telegram-id-independent: a single P1 meter emits one telegram per second,
|
||||
# and down-sampling keeps at most one per interval-aligned second. The
|
||||
# UNIQUE(recorded_at) constraint is the backstop for the IntegrityError race.
|
||||
def _enabled_snapshots() -> list[DsmrSourceSnapshot]:
|
||||
session_local = get_session_local()
|
||||
session = session_local()
|
||||
try:
|
||||
existing = session.scalar(
|
||||
select(DsmrReading).where(DsmrReading.recorded_at == ts_utc)
|
||||
)
|
||||
if existing is not None:
|
||||
logger.debug(
|
||||
"dsmr_ingest: recorded_at=%s already in DB, skipping.",
|
||||
ts_utc.isoformat(),
|
||||
sources = session.scalars(
|
||||
select(MeterSource).where(MeterSource.kind == "dsmr_mqtt", MeterSource.enabled.is_(True))
|
||||
).all()
|
||||
return [_snapshot(source) for source in sources]
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def apply_dsmr_subscription(_settings: "Settings | None" = None) -> None:
|
||||
"""Reconcile enabled DSMR source subscriptions from the database.
|
||||
|
||||
The optional, ignored settings parameter preserves the old config-route
|
||||
call shape while ensuring flat DSMR settings are no longer read.
|
||||
"""
|
||||
from app.integrations.mqtt import mqtt_manager
|
||||
|
||||
try:
|
||||
desired = {snapshot.source_id: snapshot for snapshot in _enabled_snapshots()}
|
||||
except Exception:
|
||||
logger.exception("DSMR subscription reconcile failed while reading sources")
|
||||
return
|
||||
|
||||
# Do not retain _subscription_lock while stopping MQTT clients: a callback
|
||||
# may currently hold it through its complete dispatch, and MqttManager
|
||||
# waits for that callback before teardown returns.
|
||||
with _reconcile_lock:
|
||||
# Every source owns a distinct MQTT client, so equal topics from
|
||||
# different sources/brokers are dispatchable. A telegram and tariff
|
||||
# topic on the *same* client would overwrite one handler, however.
|
||||
for source_id, snapshot in list(desired.items()):
|
||||
if snapshot.tariff_topic and snapshot.topic == snapshot.tariff_topic:
|
||||
logger.error("DSMR source_id=%s rejected: telegram/tariff topic collision", source_id)
|
||||
desired.pop(source_id)
|
||||
|
||||
with _subscription_lock:
|
||||
stale_source_ids = [
|
||||
source_id
|
||||
for source_id, current in _subscriptions.items()
|
||||
if desired.get(source_id) != current
|
||||
]
|
||||
for source_id in stale_source_ids:
|
||||
_subscriptions.pop(source_id, None)
|
||||
_subscription_tokens.pop(source_id, None)
|
||||
set_current_tariff(source_id, None)
|
||||
|
||||
for source_id in stale_source_ids:
|
||||
mqtt_manager.remove_source(source_id)
|
||||
|
||||
for source_id, snapshot in desired.items():
|
||||
with _subscription_lock:
|
||||
current = _subscriptions.get(source_id)
|
||||
if current == snapshot and mqtt_manager.source_is_active(source_id):
|
||||
continue
|
||||
if current is not None:
|
||||
# The client went inactive outside reconcile. Invalidate its
|
||||
# old token before rebuilding the same snapshot.
|
||||
with _subscription_lock:
|
||||
if _subscriptions.get(source_id) == current:
|
||||
_subscriptions.pop(source_id, None)
|
||||
_subscription_tokens.pop(source_id, None)
|
||||
mqtt_manager.remove_source(source_id)
|
||||
token = object()
|
||||
handlers = {
|
||||
snapshot.topic: lambda payload, captured=snapshot, captured_token=token: (
|
||||
handle_captured_message(payload, captured, captured_token)
|
||||
)
|
||||
}
|
||||
if snapshot.tariff_topic:
|
||||
handlers[snapshot.tariff_topic] = (
|
||||
lambda payload, captured=snapshot, captured_token=token: (
|
||||
handle_captured_tariff_message(payload, captured, captured_token)
|
||||
)
|
||||
)
|
||||
with _subscription_lock:
|
||||
_subscriptions[source_id] = snapshot
|
||||
_subscription_tokens[source_id] = token
|
||||
applied = mqtt_manager.replace_source(
|
||||
source_id,
|
||||
host=snapshot.broker_host,
|
||||
port=snapshot.broker_port,
|
||||
username=snapshot.username,
|
||||
password=snapshot.password,
|
||||
tls_enabled=snapshot.tls_enabled,
|
||||
subscriptions=handlers,
|
||||
)
|
||||
if not applied:
|
||||
with _subscription_lock:
|
||||
if _subscription_tokens.get(source_id) is token:
|
||||
_subscriptions.pop(source_id, None)
|
||||
_subscription_tokens.pop(source_id, None)
|
||||
|
||||
|
||||
def handle_message(payload_bytes: bytes, snapshot: DsmrSourceSnapshot) -> None:
|
||||
"""Persist one down-sampled frame under its captured source identity."""
|
||||
try:
|
||||
_handle_message_inner(payload_bytes, snapshot)
|
||||
except Exception:
|
||||
logger.exception("DSMR ingest handler failed for source_id=%s (swallowed)", snapshot.source_id)
|
||||
|
||||
|
||||
def handle_captured_message(
|
||||
payload_bytes: bytes, snapshot: DsmrSourceSnapshot, token: object | None = None
|
||||
) -> None:
|
||||
"""Run a broker callback only while its exact source generation is active."""
|
||||
with _subscription_lock:
|
||||
if token is not None:
|
||||
if _subscription_tokens.get(snapshot.source_id) is not token:
|
||||
return
|
||||
elif _subscriptions.get(snapshot.source_id) != snapshot:
|
||||
return
|
||||
handle_message(payload_bytes, snapshot)
|
||||
|
||||
|
||||
def handle_captured_tariff_message(
|
||||
payload_bytes: bytes, snapshot: DsmrSourceSnapshot, token: object | None = None
|
||||
) -> None:
|
||||
"""Ignore tariff callbacks retained from a removed/replaced source."""
|
||||
with _subscription_lock:
|
||||
if token is not None:
|
||||
if _subscription_tokens.get(snapshot.source_id) is not token:
|
||||
return
|
||||
elif _subscriptions.get(snapshot.source_id) != snapshot:
|
||||
return
|
||||
handle_tariff_message(payload_bytes, snapshot.source_id)
|
||||
|
||||
reading = DsmrReading(
|
||||
recorded_at=ts_utc,
|
||||
source_id=source_id,
|
||||
payload=data, # full frame, verbatim
|
||||
|
||||
def _handle_message_inner(payload_bytes: bytes, snapshot: DsmrSourceSnapshot) -> None:
|
||||
try:
|
||||
data = json.loads(payload_bytes)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return
|
||||
if not isinstance(data, dict):
|
||||
return
|
||||
try:
|
||||
raw_ts = data["timestamp"]
|
||||
ts_utc = datetime.fromisoformat(raw_ts.replace("Z", "+00:00"))
|
||||
if ts_utc.tzinfo is None:
|
||||
ts_utc = ts_utc.replace(tzinfo=timezone.utc)
|
||||
except (KeyError, ValueError, TypeError, AttributeError):
|
||||
return
|
||||
if snapshot.sample_interval_s > 0 and ts_utc.second % snapshot.sample_interval_s:
|
||||
return
|
||||
telegram_id = data.get("id")
|
||||
if telegram_id is not None and not isinstance(telegram_id, int):
|
||||
telegram_id = None
|
||||
|
||||
session_local = get_session_local()
|
||||
session = session_local()
|
||||
try:
|
||||
exists = session.scalar(
|
||||
select(DsmrReading.id).where(
|
||||
DsmrReading.meter_source_id == snapshot.source_id,
|
||||
DsmrReading.recorded_at == ts_utc,
|
||||
)
|
||||
)
|
||||
session.add(reading)
|
||||
session.commit()
|
||||
logger.debug(
|
||||
"dsmr_ingest: persisted reading recorded_at=%s source_id=%s.",
|
||||
ts_utc.isoformat(),
|
||||
source_id,
|
||||
)
|
||||
if exists is None:
|
||||
session.add(
|
||||
DsmrReading(
|
||||
meter_source_id=snapshot.source_id,
|
||||
recorded_at=ts_utc,
|
||||
telegram_id=telegram_id,
|
||||
payload=data,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
# Race / duplicate: another insert beat us to the same recorded_at.
|
||||
session.rollback()
|
||||
logger.debug(
|
||||
"dsmr_ingest: IntegrityError for recorded_at=%s (duplicate, skipped).",
|
||||
ts_utc.isoformat(),
|
||||
)
|
||||
except Exception:
|
||||
session.rollback()
|
||||
logger.exception("dsmr_ingest: DB error (swallowed).")
|
||||
logger.exception("DSMR database write failed for source_id=%s", snapshot.source_id)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
Reference in New Issue
Block a user