MQTT-FIX-T01: make main MQTT client lifecycle idempotent

app/integrations/mqtt.py

@@ -94,6 +94,12 @@ class MqttManager:
     def __init__(self) -> None:
         self._client: mqtt.Client | None = None
         self._lock = threading.RLock()
+        # Serialize ownership changes without holding the callback lock while
+        # loop_stop() joins Paho's network thread.
+        self._lifecycle_lock = threading.Lock()
+        self._client_generation = 0
+        self._client_settings: tuple[object, ...] | None = None
+        self._client_loop_started = False
         # Source replacement/removal is serialized independently from callback
         # bookkeeping.  In particular, paho's loop_stop() joins its network
         # thread, whose callback completion also needs ``_lock``.
@@ -121,7 +127,8 @@ class MqttManager:
     @property
     def is_connected(self) -> bool:
         """True if the underlying paho client is currently connected."""
-        return self._connected
+        with self._lock:
+            return self._connected
 
     # ------------------------------------------------------------------
     # Public interface
@@ -141,21 +148,29 @@ class MqttManager:
             logger.debug("MQTT not configured or not enabled — skipping connect.")
             return
 
-        with self._lock:
-            self._start_client(settings)
+        with self._lifecycle_lock:
+            with self._lock:
+                if self._client is not None and self._client_settings == self._settings_key(settings):
+                    thread = getattr(self._client, "_thread", None)
+                    is_alive = getattr(thread, "is_alive", None)
+                    if thread is None and not self._client_loop_started:
+                        return
+                    if thread is not None and (not callable(is_alive) or is_alive()):
+                        return
+            self._replace_client(settings)
 
     def disconnect(self) -> None:
         """Disconnect from the broker and stop the background loop thread.
 
         No-op if no client is active.
         """
-        with self._lock:
+        with self._lifecycle_lock:
             self._stop_client()
-        with self._source_lifecycle_lock:
-            with self._lock:
-                source_ids = list(self._source_clients)
-            for source_id in source_ids:
-                self._stop_source_client(source_id)
+            with self._source_lifecycle_lock:
+                with self._lock:
+                    source_ids = list(self._source_clients)
+                for source_id in source_ids:
+                    self._stop_source_client(source_id)
 
     def reconnect(self, settings: Settings) -> None:
         """Disconnect the current client (if any) and reconnect with *settings*.
@@ -163,9 +178,23 @@ class MqttManager:
         Call this after saving updated MQTT configuration values.
         No-op if MQTT is not enabled or broker host is not set.
         """
-        with self._lock:
-            self._stop_client()
-        self.connect(settings)
+        if not _is_configured(settings):
+            # A configuration update only owns the application-wide client.
+            # DSMR sources have independent settings and lifecycle; only app
+            # shutdown via disconnect() is allowed to stop them all.
+            with self._lifecycle_lock:
+                self._stop_client()
+            return
+        with self._lifecycle_lock:
+            with self._lock:
+                if self._client is not None and self._client_settings == self._settings_key(settings):
+                    thread = getattr(self._client, "_thread", None)
+                    is_alive = getattr(thread, "is_alive", None)
+                    if thread is None and not self._client_loop_started:
+                        return
+                    if thread is not None and (not callable(is_alive) or is_alive()):
+                        return
+            self._replace_client(settings)
 
     def publish(
         self,
@@ -183,8 +212,9 @@ class MqttManager:
         """
         with self._lock:
             client = self._client
+            connected = self._connected
 
-        if client is None or not self._connected:
+        if client is None or not connected:
             logger.debug("MQTT publish skipped — not connected (topic=%s).", topic)
             return False
 
@@ -212,7 +242,8 @@ class MqttManager:
         Handler exceptions are swallowed by the ``on_message`` dispatcher so
         that a buggy handler can never crash the paho loop thread.
         """
-        self._subscriptions[topic] = handler
+        with self._lock:
+            self._subscriptions[topic] = handler
         # Grab the client reference outside the lock to avoid holding the lock
         # while calling back into paho (which may itself acquire internal locks).
         with self._lock:
@@ -235,8 +266,8 @@ class MqttManager:
         immediately; either way the handler is removed from the registry so it
         will not be re-subscribed on the next ``_on_connect``.
         """
-        self._subscriptions.pop(topic, None)
         with self._lock:
+            self._subscriptions.pop(topic, None)
             client = self._client
             connected = self._connected
 
@@ -388,13 +419,36 @@ class MqttManager:
     # Internal helpers
     # ------------------------------------------------------------------
 
-    def _start_client(self, settings: Settings) -> None:
-        """Build a fresh paho Client, configure it, and call loop_start + connect."""
-        client = mqtt.Client(
-            callback_api_version=mqtt.CallbackAPIVersion.VERSION2,
-            client_id=settings.mqtt_client_id,
+    @staticmethod
+    def _settings_key(settings: Settings) -> tuple[object, ...]:
+        return (
+            settings.mqtt_enabled,
+            settings.mqtt_broker_host,
+            settings.mqtt_broker_port,
+            settings.mqtt_username,
+            settings.mqtt_password,
+            settings.mqtt_tls_enabled,
+            settings.mqtt_client_id,
         )
 
+    def _replace_client(self, settings: Settings) -> None:
+        self._stop_client()
+        self._start_client(settings)
+
+    def _start_client(self, settings: Settings) -> None:
+        """Install one generation before starting Paho's reconnecting loop."""
+        with self._lock:
+            self._client_generation += 1
+            generation = self._client_generation
+        try:
+            client = mqtt.Client(
+                callback_api_version=mqtt.CallbackAPIVersion.VERSION2,
+                client_id=settings.mqtt_client_id,
+            )
+        except Exception:
+            logger.exception("MQTT client initialization failed.")
+            return
+
         # Callbacks — VERSION2 on_connect signature:
         # (client, userdata, connect_flags, reason_code, properties)
         def _on_connect(
@@ -404,21 +458,21 @@ class MqttManager:
             reason_code: mqtt.ReasonCode,
             _properties: mqtt.Properties | None,
         ) -> None:
-            if reason_code.is_failure:
-                logger.warning("MQTT connection refused: %s", reason_code)
-                self._connected = False
-            else:
-                logger.info("MQTT connected (reason_code=%s).", reason_code)
+            with self._lock:
+                if self._client is not _client or self._client_generation != generation:
+                    return
+                if reason_code.is_failure:
+                    self._connected = False
+                    logger.warning("MQTT connection refused: %s", reason_code)
+                    return
                 self._connected = True
-                # Re-establish all registered subscriptions after (re-)connect.
-                for sub_topic in self._subscriptions:
-                    try:
-                        _client.subscribe(sub_topic)
-                        logger.debug("MQTT re-subscribed to topic=%s.", sub_topic)
-                    except Exception:
-                        logger.exception(
-                            "MQTT re-subscribe failed (topic=%s).", sub_topic
-                        )
+                topics = tuple(self._subscriptions)
+            logger.info("MQTT connected (reason_code=%s).", reason_code)
+            for sub_topic in topics:
+                try:
+                    _client.subscribe(sub_topic)
+                except Exception:
+                    logger.exception("MQTT re-subscribe failed (topic=%s).", sub_topic)
 
         # VERSION2 on_disconnect signature:
         # (client, userdata, disconnect_flags, reason_code, properties)
@@ -429,7 +483,10 @@ class MqttManager:
             reason_code: mqtt.ReasonCode,
             _properties: mqtt.Properties | None,
         ) -> None:
-            self._connected = False
+            with self._lock:
+                if self._client is not _client or self._client_generation != generation:
+                    return
+                self._connected = False
             if reason_code.is_failure:
                 logger.warning("MQTT disconnected unexpectedly (reason_code=%s).", reason_code)
             else:
@@ -441,7 +498,10 @@ class MqttManager:
             _userdata: object,
             message: mqtt.MQTTMessage,
         ) -> None:
-            handler = self._subscriptions.get(message.topic)
+            with self._lock:
+                if self._client is not _client or self._client_generation != generation:
+                    return
+                handler = self._subscriptions.get(message.topic)
             if handler is None:
                 logger.debug(
                     "MQTT on_message: no handler for topic=%s.", message.topic
@@ -469,35 +529,57 @@ class MqttManager:
 
         # Credentials — never log the password
         if settings.mqtt_username:
-            client.username_pw_set(
-                username=settings.mqtt_username,
-                password=settings.mqtt_password or None,
-            )
-
-        # Start background loop thread *before* connect so paho can handle
-        # the connection handshake asynchronously.
-        client.loop_start()
+            try:
+                client.username_pw_set(
+                    username=settings.mqtt_username,
+                    password=settings.mqtt_password or None,
+                )
+            except Exception:
+                logger.exception("MQTT credentials setup failed.")
+                return
 
+        # Publish ownership before network processing/callbacks can begin.
+        with self._lock:
+            self._client = client
+            self._client_settings = self._settings_key(settings)
+            self._connected = False
+            self._client_loop_started = False
         try:
-            client.connect(
+            client.connect_async(
                 host=settings.mqtt_broker_host,
                 port=settings.mqtt_broker_port,
                 keepalive=60,
             )
+            result = client.loop_start()
+            if isinstance(result, int) and result != mqtt.MQTT_ERR_SUCCESS:
+                raise RuntimeError(f"MQTT loop_start failed (rc={result})")
+            with self._lock:
+                if self._client is client and self._client_generation == generation:
+                    self._client_loop_started = True
         except Exception:
+            # Remove ownership before joining the loop; callbacks from this
+            # generation become inert immediately.
+            with self._lock:
+                if self._client is client and self._client_generation == generation:
+                    self._client = None
+                    self._client_settings = None
+                    self._connected = False
+                    self._client_loop_started = False
             # Log without leaking the password
             logger.exception(
                 "MQTT connect failed (host=%s, port=%s). Stopping loop.",
                 settings.mqtt_broker_host,
                 settings.mqtt_broker_port,
             )
+            try:
+                client.disconnect()
+            except Exception:
+                pass
             try:
                 client.loop_stop()
             except Exception:
                 pass
             return
-
-        self._client = client
         logger.info(
             "MQTT client started (host=%s, port=%s).",
             settings.mqtt_broker_host,
@@ -506,11 +588,17 @@ class MqttManager:
 
     def _stop_client(self) -> None:
         """Disconnect and stop the background loop.  Idempotent."""
-        client = self._client
-        if client is None:
-            return
-        self._client = None
-        self._connected = False
+        with self._lock:
+            client = self._client
+            if client is None:
+                self._connected = False
+                self._client_settings = None
+                return
+            self._client = None
+            self._client_settings = None
+            self._connected = False
+            self._client_loop_started = False
+            self._client_generation += 1
         try:
             client.disconnect()
         except Exception:

app/main.py

@@ -169,9 +169,12 @@ def _run_scheduled_ha_state_publish() -> None:
     session: Session = session_local()
     try:
         runtime_settings = build_runtime_settings(session, get_settings())
-        # Reconnect if MQTT is configured (in DB) but not yet connected.
-        if mqtt_manager.is_configured(runtime_settings) and not mqtt_manager.is_connected:
-            logger.info("_run_scheduled_ha_state_publish: MQTT configured but not connected — attempting connect.")
+        # Ensure the configured MQTT client is installed; the manager avoids
+        # replacing it while Paho is already retrying a transient disconnect.
+        if mqtt_manager.is_configured(runtime_settings):
+            # connect() is idempotent for the same settings and retains a Paho
+            # client while its own loop is retrying a transient disconnect.
+            logger.info("_run_scheduled_ha_state_publish: ensuring MQTT client is active.")
             mqtt_manager.connect(runtime_settings)
         publish_discovery(session)
         publish_states(session)