DSMR hardening: restart-free config + decouple from telegram id
Config hot-reload (no restart): - MqttManager.unsubscribe(); apply_dsmr_subscription(settings) re-applies the DSMR subscription (subscribe/unsubscribe/topic-change, fresh snapshot so the sample interval also takes effect). - Called from lifespan AND PUT /api/config, so toggling DSMR ingest in the UI takes effect immediately without restarting the app. Decouple ingest from the DSMR telegram id (overflows / resets to zero): - Migration 20260624_12: drop UNIQUE(source_id), make recorded_at UNIQUE. - Idempotency now keys on recorded_at (telegram timestamp); the table's own autoincrement PK is the stable identity. source_id kept only as a reference. - Regression test: colliding telegram ids no longer drop new data.
This commit is contained in:
@@ -86,6 +86,12 @@ def put_config(
|
||||
logger.info("MQTT settings changed — triggering reconnect.")
|
||||
mqtt_manager.reconnect(refreshed_settings)
|
||||
|
||||
# Re-apply the DSMR subscription so enabling/disabling DSMR ingest (or changing
|
||||
# its topic / sample interval) takes effect immediately, without an app restart.
|
||||
# Done after any MQTT reconnect so it operates on the current client.
|
||||
from app.services.dsmr_ingest import apply_dsmr_subscription
|
||||
apply_dsmr_subscription(refreshed_settings)
|
||||
|
||||
sections_raw = build_config_sections(db, refreshed_settings)
|
||||
return ConfigUpdateResponse(sections=_sections_from_raw(sections_raw))
|
||||
|
||||
|
||||
@@ -175,6 +175,27 @@ class MqttManager:
|
||||
except Exception:
|
||||
logger.exception("MQTT subscribe error (topic=%s).", topic)
|
||||
|
||||
def unsubscribe(self, topic: str) -> None:
|
||||
"""Remove the handler for *topic* and unsubscribe from the broker.
|
||||
|
||||
Idempotent: unknown topics are ignored. Used to apply config changes
|
||||
without a restart (e.g. when DSMR ingest is turned off or its topic
|
||||
changes). If the client is connected, the broker unsubscribe is sent
|
||||
immediately; either way the handler is removed from the registry so it
|
||||
will not be re-subscribed on the next ``_on_connect``.
|
||||
"""
|
||||
self._subscriptions.pop(topic, None)
|
||||
with self._lock:
|
||||
client = self._client
|
||||
connected = self._connected
|
||||
|
||||
if client is not None and connected:
|
||||
try:
|
||||
client.unsubscribe(topic)
|
||||
logger.debug("MQTT unsubscribed from topic=%s.", topic)
|
||||
except Exception:
|
||||
logger.exception("MQTT unsubscribe error (topic=%s).", topic)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers — must be called with self._lock held
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
+5
-16
@@ -29,7 +29,7 @@ from app.config import get_settings
|
||||
from app.integrations.mqtt import mqtt_manager
|
||||
from app.services.auth import AuthBootstrapError, initialize_auth_schema
|
||||
from app.services.config_page import build_runtime_settings, seed_missing_config_from_bootstrap, sync_app_hostname_from_bootstrap
|
||||
from app.services.dsmr_ingest import handle_message as dsmr_handle_message
|
||||
from app.services.dsmr_ingest import apply_dsmr_subscription
|
||||
from app.services.public_ip import check_public_ipv4_and_notify
|
||||
from app.services.modbus_poll import poll_all_enabled_devices, BASE_POLL_TICK_SECONDS
|
||||
from app.services.ha_discovery import publish_discovery, publish_states
|
||||
@@ -246,21 +246,10 @@ async def lifespan(_: FastAPI):
|
||||
_startup_session.close()
|
||||
mqtt_manager.connect(_startup_runtime_settings)
|
||||
|
||||
# DSMR ingest: subscribe to the configured MQTT topic when enabled.
|
||||
# The handler captures a snapshot of the current runtime settings (for the
|
||||
# dsmr_sample_interval_s value). Runtime setting changes take effect after
|
||||
# a server restart or an explicit reconnect — consistent with other MQTT
|
||||
# configuration in this project.
|
||||
if _startup_runtime_settings.dsmr_ingest_enabled:
|
||||
_dsmr_topic = _startup_runtime_settings.dsmr_mqtt_topic
|
||||
_dsmr_settings_snapshot = _startup_runtime_settings
|
||||
mqtt_manager.subscribe(
|
||||
_dsmr_topic,
|
||||
lambda payload: dsmr_handle_message(payload, _dsmr_settings_snapshot),
|
||||
)
|
||||
logger.info("DSMR ingest enabled — subscribed to topic=%s.", _dsmr_topic)
|
||||
else:
|
||||
logger.debug("DSMR ingest disabled — not subscribing to DSMR topic.")
|
||||
# DSMR ingest: subscribe to the configured MQTT topic when enabled. The same
|
||||
# applier is called from PUT /api/config, so toggling DSMR via the UI takes
|
||||
# effect without an app restart.
|
||||
apply_dsmr_subscription(_startup_runtime_settings)
|
||||
|
||||
yield
|
||||
|
||||
|
||||
+18
-12
@@ -22,27 +22,33 @@ from app.db import Base
|
||||
class DsmrReading(Base):
|
||||
"""One down-sampled DSMR telegram stored as a full JSON blob.
|
||||
|
||||
``recorded_at`` is a real indexed column (not inside the payload) so that
|
||||
time-range queries are efficient. ``source_id`` holds the telegram's own
|
||||
integer ``id`` and is used for idempotent upsert (skip if already present).
|
||||
The entire telegram frame is stored verbatim in ``payload``; no field
|
||||
allow-list is applied so future commodities (gas, heating, three-phase)
|
||||
are accommodated without a schema change.
|
||||
Identity & idempotency are **independent of the DSMR Reader's telegram id**
|
||||
(that field overflows and must be manually reset to zero — a known DSMR
|
||||
quirk — so relying on it for uniqueness risks silently dropping new data).
|
||||
The table's own autoincrement ``id`` PK is the stable internal identity, and
|
||||
``recorded_at`` (the telegram timestamp) is the UNIQUE de-duplication key: a
|
||||
single P1 meter emits exactly one telegram per timestamp.
|
||||
|
||||
``recorded_at`` is a real column (not inside the payload) so time-range
|
||||
queries are efficient. The entire telegram frame is stored verbatim in
|
||||
``payload``; no field allow-list is applied so future commodities (gas,
|
||||
heating, three-phase) are accommodated without a schema change.
|
||||
"""
|
||||
|
||||
__tablename__ = "dsmr_reading"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
|
||||
# UTC timestamp of the sample — real column so it can be indexed efficiently.
|
||||
# UTC timestamp of the sample — real column, UNIQUE (telegram-id-independent
|
||||
# idempotency key). The unique index also serves time-range queries.
|
||||
recorded_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True
|
||||
DateTime(timezone=True), nullable=False, unique=True
|
||||
)
|
||||
|
||||
# Telegram's own auto-increment id (DSMR Reader assigns it). Used for
|
||||
# idempotent ingestion: if the same telegram arrives twice we skip it.
|
||||
# Nullable because some DSMR sources may not emit an id.
|
||||
source_id: Mapped[int | None] = mapped_column(Integer, unique=True, nullable=True)
|
||||
# Telegram's own id (DSMR Reader assigns it). Stored only as a reference /
|
||||
# debugging aid — NOT used for uniqueness or idempotency (it overflows and
|
||||
# gets reset to zero). Nullable because some DSMR sources may not emit one.
|
||||
source_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
# Full telegram frame as a JSON object; values are typically JSON strings
|
||||
# (e.g. "20915.154") — callers must cast to Decimal before arithmetic.
|
||||
|
||||
+61
-14
@@ -47,6 +47,50 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Tracks the DSMR topic currently subscribed via the MQTT manager, so a config
|
||||
# change can unsubscribe the old topic before subscribing the new one.
|
||||
_current_dsmr_topic: str | None = None
|
||||
|
||||
|
||||
def apply_dsmr_subscription(settings: "Settings") -> None:
|
||||
"""(Re)apply the DSMR MQTT subscription to match *settings* — restart-free.
|
||||
|
||||
Call this at startup and after every config save. It makes the live MQTT
|
||||
subscription reflect the current ``dsmr_ingest_enabled`` / ``dsmr_mqtt_topic``
|
||||
/ ``dsmr_sample_interval_s`` settings without an app restart:
|
||||
|
||||
- **Disabled** → unsubscribe any active DSMR subscription.
|
||||
- **Enabled** → (re)subscribe to ``dsmr_mqtt_topic`` with a handler bound to
|
||||
a *fresh* settings snapshot, so a changed sample interval also takes effect.
|
||||
- **Topic changed** → unsubscribe the old topic before subscribing the new one.
|
||||
|
||||
Idempotent and safe to call when MQTT is not connected (the subscription is
|
||||
queued in the manager and established on the next connect).
|
||||
"""
|
||||
# Imported here (not at module top) to avoid a circular import at app start.
|
||||
from app.integrations.mqtt import mqtt_manager
|
||||
|
||||
global _current_dsmr_topic
|
||||
|
||||
if not settings.dsmr_ingest_enabled:
|
||||
if _current_dsmr_topic is not None:
|
||||
mqtt_manager.unsubscribe(_current_dsmr_topic)
|
||||
logger.info("DSMR ingest disabled — unsubscribed from topic=%s.", _current_dsmr_topic)
|
||||
_current_dsmr_topic = None
|
||||
return
|
||||
|
||||
topic = settings.dsmr_mqtt_topic
|
||||
if _current_dsmr_topic is not None and _current_dsmr_topic != topic:
|
||||
mqtt_manager.unsubscribe(_current_dsmr_topic)
|
||||
|
||||
# Re-subscribe (overwrites any existing handler for this topic) with a fresh
|
||||
# settings snapshot so dsmr_sample_interval_s changes take effect too.
|
||||
snapshot = settings
|
||||
mqtt_manager.subscribe(topic, lambda payload: handle_message(payload, snapshot))
|
||||
_current_dsmr_topic = topic
|
||||
logger.info("DSMR ingest enabled — subscribed to topic=%s.", topic)
|
||||
|
||||
|
||||
def handle_message(payload_bytes: bytes, settings: "Settings") -> None:
|
||||
"""Parse one DSMR MQTT payload and persist it if it passes the sample filter.
|
||||
|
||||
@@ -107,7 +151,8 @@ def _handle_message_inner(payload_bytes: bytes, settings: "Settings") -> None:
|
||||
# This telegram is between sample points; discard silently.
|
||||
return
|
||||
|
||||
# --- 4. Extract source_id (telegram's own id field, for idempotency) ---
|
||||
# --- 4. Extract source_id (telegram's own id) — stored only as a reference,
|
||||
# NOT used for uniqueness/idempotency (it overflows and gets reset). ---
|
||||
source_id: int | None = data.get("id")
|
||||
if source_id is not None and not isinstance(source_id, int):
|
||||
# Unexpected type — treat as missing rather than raising.
|
||||
@@ -118,19 +163,22 @@ def _handle_message_inner(payload_bytes: bytes, settings: "Settings") -> None:
|
||||
source_id = None
|
||||
|
||||
# --- 5. Persist to database ---
|
||||
# Idempotency is keyed on recorded_at (the telegram timestamp), which is
|
||||
# telegram-id-independent: a single P1 meter emits one telegram per second,
|
||||
# and down-sampling keeps at most one per interval-aligned second. The
|
||||
# UNIQUE(recorded_at) constraint is the backstop for the IntegrityError race.
|
||||
session_local = get_session_local()
|
||||
session = session_local()
|
||||
try:
|
||||
# Idempotency check: if source_id is known and already exists, skip.
|
||||
if source_id is not None:
|
||||
existing = session.scalar(
|
||||
select(DsmrReading).where(DsmrReading.source_id == source_id)
|
||||
existing = session.scalar(
|
||||
select(DsmrReading).where(DsmrReading.recorded_at == ts_utc)
|
||||
)
|
||||
if existing is not None:
|
||||
logger.debug(
|
||||
"dsmr_ingest: recorded_at=%s already in DB, skipping.",
|
||||
ts_utc.isoformat(),
|
||||
)
|
||||
if existing is not None:
|
||||
logger.debug(
|
||||
"dsmr_ingest: source_id=%d already in DB, skipping.", source_id
|
||||
)
|
||||
return
|
||||
return
|
||||
|
||||
reading = DsmrReading(
|
||||
recorded_at=ts_utc,
|
||||
@@ -145,12 +193,11 @@ def _handle_message_inner(payload_bytes: bytes, settings: "Settings") -> None:
|
||||
source_id,
|
||||
)
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
# Race condition: another handler inserted the same source_id between our
|
||||
# check and our insert. Roll back and ignore.
|
||||
# Race / duplicate: another insert beat us to the same recorded_at.
|
||||
session.rollback()
|
||||
logger.debug(
|
||||
"dsmr_ingest: IntegrityError for source_id=%s (duplicate, skipped).",
|
||||
source_id,
|
||||
"dsmr_ingest: IntegrityError for recorded_at=%s (duplicate, skipped).",
|
||||
ts_utc.isoformat(),
|
||||
)
|
||||
except Exception:
|
||||
session.rollback()
|
||||
|
||||
Reference in New Issue
Block a user