home-automation/app/services/ha_discovery.py

"""HA MQTT Discovery service — build and publish discovery configs + state.

This module handles:

1. Building HA MQTT Discovery payloads for each ``ExposableEntity``.
2. Publishing discovery configs (retained) for enabled entities.
3. Clearing (empty-payload) discovery configs for disabled entities.
4. Publishing current entity state values.
5. A per-device helper for use in the Modbus poll loop.

Design
------
- **No-op when MQTT / discovery is not configured**: every public function
  checks ``ha_discovery_enabled`` and ``mqtt_manager.is_connected`` before
  doing any work.  Callers never need to guard these conditions.
- **Stable unique_id**: derived from device ``uuid`` + metric ``key``, never
  from mutable fields like ``friendly_name`` or auto-increment DB ids.
- **Discovery topic format**: ``<discovery_prefix>/<component>/<node_id>/<object_id>/config``
  where ``node_id`` is the device uuid (slugified to be safe) and
  ``object_id`` is a uuid-prefixed stable string.  Uses ``ha_discovery_prefix``
  (default ``"homeassistant"``), which must stay under the HA discovery namespace.
- **State topic format**: ``<state_prefix>/<component>/<node_id>/<object_id>/state``
  Uses ``ha_state_topic_prefix`` (default ``"home_automation"``), separate from
  the discovery namespace so HA discovery and state/availability topics live under
  different prefixes.
- **Availability topic**: ``<state_prefix>/modbus/<node_id>/availability`` (shared
  across all entities of the same device; publishes "online"/"offline").  Also uses
  the state prefix, not the discovery prefix.
- **best-effort**: functions catch all exceptions internally so that callers
  (e.g. the Modbus poll loop) never crash due to MQTT publish errors.
"""

from __future__ import annotations

import json
import logging
import re
from datetime import UTC, datetime
from typing import Any

from sqlalchemy.orm import Session
from sqlalchemy import inspect

from app.integrations.expose import ExposableEntity, build_catalog
from app.models.config import AppConfigEntry
from app.integrations.mqtt import mqtt_manager
from app.services.config_page import build_runtime_settings

logger = logging.getLogger(__name__)

_HA_SEGMENT_RE = re.compile(r"[^A-Za-z0-9_-]+")
# These durable, versioned application metadata entries prevent the minute
# publisher from repeatedly emitting already accepted known-bad v1.6.1 thermal
# topics.  A failed publish is not acknowledged and remains retryable across
# restarts.  They are intentionally application metadata, rather than broker
# state.  An empty retained publish to an illegal v1.6.1 topic cannot be used
# as an acknowledgement: HA rejects that topic before it processes its payload.
_LEGACY_THERMAL_CLEANUP_KEY = "HA_DISCOVERY_LEGACY_THERMAL_CLEANUP_V1"
_REGISTRY_REPAIR_KEY = "HA_DISCOVERY_REGISTRY_REPAIR_V2"


# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------


def _should_publish(settings: Any) -> bool:
    """Return True only if MQTT and HA Discovery are both enabled and connected."""
    return bool(
        settings.mqtt_enabled
        and settings.ha_discovery_enabled
        and mqtt_manager.is_connected
    )


def _safe_segment(value: str) -> str:
    """Return a deterministic HA discovery topic segment.

    Home Assistant accepts only ``[A-Za-z0-9_-]+`` for node/object ids.
    Replacing each run of other characters keeps ordinary UUID/key output
    readable while ensuring composite identities cannot produce invalid topics.
    """
    result = _HA_SEGMENT_RE.sub("_", value).strip("_")
    return result or "home_automation"


def _node_id(identity: str) -> str:
    """Stable, strictly valid MQTT node_id from an internal identity."""
    return _safe_segment(identity.replace("-", "_"))


def _object_id(entity: ExposableEntity) -> str:
    """Stable, strictly valid MQTT object_id derived from the entity key."""
    return _safe_segment(entity.key.replace("-", "_"))


def _discovery_topic(entity: ExposableEntity, prefix: str) -> str:
    """Build the HA Discovery config topic for *entity*.

    Format: ``<prefix>/<component>/<node_id>/<object_id>/config``
    """
    node = _node_id(entity.device.internal_identity)
    obj = _object_id(entity)
    return f"{prefix}/{entity.component}/{node}/{obj}/config"


def _state_topic(entity: ExposableEntity, prefix: str) -> str:
    """Build the state publish topic for *entity*.

    Format: ``<prefix>/<component>/<node_id>/<object_id>/state``
    """
    node = _node_id(entity.device.internal_identity)
    obj = _object_id(entity)
    return f"{prefix}/{entity.component}/{node}/{obj}/state"


def _availability_topic(device_uuid: str, prefix: str) -> str:
    """Shared availability topic for all entities of a device.

    Format: ``<prefix>/modbus/<node_id>/availability``
    """
    node = _node_id(device_uuid)
    return f"{prefix}/modbus/{node}/availability"


def _availability_id(entity: ExposableEntity) -> str:
    """Return the identity which owns this entity's liveness topic.

    M8 meters deliberately retain their own UUID as HA node/unique identity,
    while their availability is supplied by a MeterSource UUID.
    """
    return entity.device.availability_id or entity.device.internal_identity


def _unique_id(entity: ExposableEntity) -> str:
    """Stable unique_id — device uuid + metric key (never from mutable fields)."""
    device_uuid = entity.device.internal_identity
    # entity.key is already "modbus.<uuid>.<metric_key>" — use it as the seed
    return f"{device_uuid}_{entity.key.replace('.', '_')}"


def _migration_state(session: Session, key: str, default: str) -> str:
    """Read a versioned discovery-repair acknowledgement from ``app_config``."""
    entry = session.query(AppConfigEntry).filter(AppConfigEntry.key == key).one_or_none()
    return entry.value if entry is not None else default


def _has_repair_store(session: Session) -> bool:
    """Whether this session uses the app schema (not a lightweight unit DB)."""
    return inspect(session.get_bind()).has_table("app_config")


def _set_migration_state(session: Session, key: str, value: str) -> None:
    """Durably acknowledge a completed discovery-repair step.

    The state is operational metadata only; it never alters meters, readings,
    contracts, costs, or expose toggles.
    """
    entry = session.query(AppConfigEntry).filter(AppConfigEntry.key == key).one_or_none()
    if entry is None:
        session.add(AppConfigEntry(key=key, value=value, updated_at=datetime.now(UTC)))
    else:
        entry.value = value
        entry.updated_at = datetime.now(UTC)
    session.commit()


def _migration_json(session: Session, key: str) -> dict[str, Any]:
    """Read a versioned JSON migration ledger, treating corrupt values as pending."""
    raw = _migration_state(session, key, "{}")
    try:
        value = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Ignoring malformed HA discovery migration ledger %s", key)
        return {}
    return value if isinstance(value, dict) else {}


def _set_migration_json(session: Session, key: str, value: dict[str, Any]) -> None:
    _set_migration_state(session, key, json.dumps(value, sort_keys=True))


def _frozen_legacy_inventory(ledger: dict[str, Any]) -> list[str] | None:
    """Return a structurally valid immutable cleanup inventory, if present.

    The ledger predates a schema migration, so it must tolerate both an absent
    entry and old ``topics``-only values.  An inventory becomes immutable only
    once it is a list of non-empty topic strings; anything else is compatibility
    data to be frozen during startup, not state a publisher may reinterpret.
    """
    value = ledger.get("inventory")
    if not isinstance(value, list) or not all(isinstance(topic, str) and topic for topic in value):
        return None
    return list(dict.fromkeys(value))


def _publish_ok(topic: str, payload: str | bytes, *, retain: bool = True) -> bool:
    """Publish best-effort while treating only an explicit ``False`` as failure.

    Production ``MqttManager.publish`` returns ``bool``.  Accepting ``None``
    keeps existing third-party/mock publishers best-effort compatible.
    """
    return mqtt_manager.publish(topic, payload, retain=retain) is not False


def initialize_legacy_thermal_cleanup(session: Session) -> None:
    """Create the legacy-cleanup ledger before expose toggles can be edited.

    A newly installed database has no v1.6.1 retained thermal topics.  Marking
    that case complete during startup prevents a later first-time toggle from
    manufacturing an old, illegal topic.  Conversely, an upgraded database
    freezes its exact enabled legacy topics before the UI can change a toggle.
    """
    if not _has_repair_store(session):
        return

    ledger = _migration_json(session, _LEGACY_THERMAL_CLEANUP_KEY)
    if ledger.get("complete") is True:
        return

    inventory = _frozen_legacy_inventory(ledger)
    if inventory is not None:
        # A previously frozen non-empty inventory must never be expanded or
        # recomputed from mutable toggle/prefix state.  A zero-item inventory
        # is just the fresh-install terminal state written by an older build.
        if not inventory:
            _set_migration_json(
                session,
                _LEGACY_THERMAL_CLEANUP_KEY,
                {"complete": True, "inventory": [], "topics": []},
            )
        return

    # This deliberately propagates.  Startup runs before the expose UI can
    # write a toggle, so swallowing an enumeration or durable-write error would
    # create a window in which the cleanup scope could be changed or lost.
    legacy_entities = _legacy_thermal_entities(session)
    from app.config import get_settings

    settings = build_runtime_settings(session, get_settings())
    inventory = list(dict.fromkeys(
        _legacy_discovery_topic(entity, settings.ha_discovery_prefix)
        for entity in legacy_entities
    ))
    _set_migration_json(
        session,
        _LEGACY_THERMAL_CLEANUP_KEY,
        {
            "complete": not inventory,
            "inventory": inventory,
            "topics": sorted({
                topic for topic in ledger.get("topics", [])
                if isinstance(topic, str) and topic in inventory
            }),
        },
    )


# ---------------------------------------------------------------------------
# Public: build discovery payload
# ---------------------------------------------------------------------------


def build_discovery_payload(
    entity: ExposableEntity,
    discovery_prefix: str,
    state_prefix: str | None = None,
) -> tuple[str, dict[str, Any]]:
    """Build the HA MQTT Discovery config topic and payload dict for *entity*.

    Parameters
    ----------
    entity:
        An ``ExposableEntity`` (from ``build_catalog``).
    discovery_prefix:
        The HA Discovery config topic prefix (e.g. ``"homeassistant"``).
        This determines where HA looks for the discovery config topic.
    state_prefix:
        The prefix used for state and availability topics (e.g.
        ``"home_automation"``).  When omitted or ``None``, falls back to
        ``discovery_prefix`` for backward compatibility (e.g. in tests that
        pass only one prefix).

    Returns
    -------
    tuple[str, dict]
        ``(topic, config_dict)`` — the config topic and the HA Discovery
        payload (not yet JSON-serialised).
    """
    if state_prefix is None:
        state_prefix = discovery_prefix

    avail_topic = _availability_topic(_availability_id(entity), state_prefix)
    state_t = _state_topic(entity, state_prefix)
    topic = _discovery_topic(entity, discovery_prefix)

    config: dict[str, Any] = {
        "unique_id": _unique_id(entity),
        "name": entity.name,
        "state_topic": state_t,
        "device": {
            "identifiers": list(entity.device.identifiers),
            "name": entity.device.name,
        },
    }

    # Only declare an availability topic for devices that actually publish an
    # online/offline heartbeat (e.g. Modbus, via its "online" binary_sensor).
    # Devices without a heartbeat (e.g. energy-cost) omit availability so HA
    # treats their entities as always-available — otherwise HA would mark them
    # ``unavailable`` forever even though their state is being published.
    if entity.device.provides_availability:
        config["availability"] = [{"topic": avail_topic}]
        config["availability_mode"] = "all"

    # Component-specific fields
    if entity.component == "binary_sensor":
        # "online" binary sensor: payload ON/OFF
        config["payload_on"] = "ON"
        config["payload_off"] = "OFF"
        if entity.device_class:
            config["device_class"] = entity.device_class
    else:
        # sensor (and others)
        if entity.device_class:
            config["device_class"] = entity.device_class
        if entity.unit:
            config["unit_of_measurement"] = entity.unit
        if entity.state_class:
            config["state_class"] = entity.state_class

    return topic, config


# ---------------------------------------------------------------------------
# Public: publish discovery
# ---------------------------------------------------------------------------


def publish_discovery(session: Session) -> None:
    """Publish HA Discovery configs for all entities in the catalog.

    - Enabled entities: publish retained JSON config payload.
    - Disabled entities: publish empty (b"") payload to clear any previously
      retained config from HA.

    No-op if MQTT / discovery is not enabled or the client is not connected.
    All MQTT errors are caught internally — this function never raises.
    """
    from app.config import get_settings
    settings = build_runtime_settings(session, get_settings())
    if not _should_publish(settings):
        logger.debug("publish_discovery: skipped (MQTT/Discovery not enabled or not connected)")
        return

    discovery_prefix = settings.ha_discovery_prefix
    state_prefix = settings.ha_state_topic_prefix

    try:
        catalog = build_catalog(session)
    except Exception:
        logger.exception("publish_discovery: failed to build catalog; aborting")
        return

    # Clear the *illegal* v1.6.1 thermal keys once.  Startup freezes the exact
    # inventory while the legacy toggle state still describes v1.6.1.  Thus a
    # later UI toggle/prefix/meter change cannot erase or expand this repair.
    # ``topics`` is only durable success progress; ``inventory`` is immutable.
    # Startup freezes compatibility ledgers before the UI can mutate the
    # toggle/prefix inputs.  The publisher only consumes that frozen snapshot.
    repaired_keys: set[str] = set()
    if _has_repair_store(session):
        ledger = _migration_json(session, _LEGACY_THERMAL_CLEANUP_KEY)
        if not ledger.get("complete"):
            inventory = _frozen_legacy_inventory(ledger)
            if inventory is None:
                logger.error(
                    "publish_discovery: legacy cleanup ledger was not frozen at startup; refusing cleanup"
                )

            if inventory is not None:
                completed = {
                    topic for topic in ledger.get("topics", [])
                    if isinstance(topic, str) and topic in inventory
                }
                for topic in (topic for topic in inventory if topic not in completed):
                    try:
                        if _publish_ok(topic, b""):
                            completed.add(topic)
                            # Commit each accepted illegal-topic tombstone before
                            # attempting another one: a crash must not replay it.
                            _set_migration_json(
                                session,
                                _LEGACY_THERMAL_CLEANUP_KEY,
                                {
                                    "complete": False,
                                    "inventory": inventory,
                                    "topics": sorted(completed),
                                },
                            )
                        else:
                            logger.warning("publish_discovery: broker rejected v1.6.1 cleanup for %s", topic)
                    except Exception:
                        logger.exception("publish_discovery: unable to clear v1.6.1 topic %s", topic)
                if set(inventory).issubset(completed):
                    _set_migration_json(
                        session,
                        _LEGACY_THERMAL_CLEANUP_KEY,
                        {"complete": True, "inventory": inventory, "topics": sorted(completed)},
                    )

        # Current-format stale topics are legal and deliberately remain retryable:
        # they cover post-repair meter swaps and include all 14 thermal metrics.
        try:
            stale_entities = _stale_m8_entities(session)
        except Exception:
            logger.exception("publish_discovery: unable to enumerate stale M8 identities")
            stale_entities = []
        for old_entity in stale_entities:
            try:
                if not _publish_ok(_discovery_topic(old_entity, discovery_prefix), b""):
                    logger.warning("publish_discovery: broker rejected stale cleanup for %r", old_entity.key)
            except Exception:
                logger.exception("publish_discovery: unable to clear stale identity %r", old_entity.key)

        # v1.6.1 used the same legal topics/unique_ids for Meter, Source, Modbus
        # and electricity-cost entities, but attached their registry entries to
        # wrongly merged HA devices.  HA does not move those entries when only a
        # discovery ``device`` block changes.  One durable unload -> re-add cycle
        # lets HA rebuild the existing unique_id entries against the new singleton
        # identifiers, preserving entity keys, toggles and user customizations.
        repaired_keys = _run_registry_repair(session, catalog, discovery_prefix, state_prefix, settings)

    for entry in catalog:
        entity = entry.entity
        if entity.key in repaired_keys:
            continue
        try:
            topic, config = build_discovery_payload(entity, discovery_prefix, state_prefix)
            if entry.enabled:
                payload = json.dumps(config)
                _publish_ok(topic, payload)
                logger.debug(
                    "publish_discovery: published config for %r → %s", entity.key, topic
                )
            else:
                # Clear retained config for disabled entities.
                _publish_ok(topic, b"")
                logger.debug(
                    "publish_discovery: cleared config for disabled entity %r → %s",
                    entity.key,
                    topic,
                )
        except Exception:
            logger.exception(
                "publish_discovery: error processing entity %r; continuing", entity.key
            )


def _is_registry_repair_entity(entity: ExposableEntity) -> bool:
    """Whether a legal v1.6.1 config needs an unload/re-add device split."""
    return entity.key.startswith(("modbus.", "source.", "meter.", "energy.", "thermal_cost."))


def _ha_registry_bindings(settings: Any, unique_ids: set[str]) -> dict[str, set[str]] | None:
    """Read HA's registry, returning ``None`` when it is not observable."""
    from app.integrations.homeassistant import (
        HomeAssistantClient,
        HomeAssistantConfigError,
        HomeAssistantRequestError,
    )

    # Runtime settings from old callers/tests may not expose outbound HA
    # fields.  Treat absent/non-string credentials as intentionally optional.
    if not isinstance(getattr(settings, "home_assistant_base_url", None), str) or not isinstance(
        getattr(settings, "home_assistant_auth_token", None), str
    ):
        return None
    client = HomeAssistantClient(settings)
    if not client.is_configured():
        return None
    try:
        return client.discovery_registry_bindings(unique_ids)
    except (HomeAssistantConfigError, HomeAssistantRequestError, TypeError, ValueError):
        logger.warning("HA registry repair remains pending: HA registry is unavailable")
        return None


def _run_registry_repair(
    session: Session,
    catalog: list[Any],
    discovery_prefix: str,
    state_prefix: str,
    settings: Any,
) -> set[str]:
    """Repair each v1.6.1 entity only after HA confirms every phase.

    MQTT acknowledges broker receipt, not Home Assistant processing.  The
    ledger therefore holds each target at ``unloaded`` until HA's entity
    registry no longer contains its stable unique_id, then at ``republished``
    until HA reports the expected singleton device identifier.  A partial
    catalog simply adds targets on a later run; it cannot complete others.
    When HA's registry is unavailable, ordinary discovery remains untouched.
    """
    targets = [entry for entry in catalog if _is_registry_repair_entity(entry.entity)]
    if not targets:
        return set()
    ledger = _migration_json(session, _REGISTRY_REPAIR_KEY)
    pending_targets = [
        entry for entry in targets if ledger.get(_unique_id(entry.entity), {}).get("phase") != "complete"
    ]
    if not pending_targets:
        return set()
    unique_ids = {_unique_id(entry.entity) for entry in pending_targets}
    bindings = _ha_registry_bindings(settings, unique_ids)
    if bindings is None:
        return set()
    blocked: set[str] = set()
    for entry in pending_targets:
        entity = entry.entity
        unique_id = _unique_id(entity)
        expected = set(entity.device.identifiers)
        phase = ledger.get(unique_id, {}).get("phase", "pending")
        observed = bindings.get(unique_id)
        if phase == "unloaded":
            if observed is not None:
                # HA was disconnected or otherwise missed the retained
                # tombstone. Keep it retained until HA itself confirms delete.
                try:
                    _publish_ok(_discovery_topic(entity, discovery_prefix), b"")
                except Exception:
                    logger.exception("publish_discovery: unable to repeat registry unload for %r", entity.key)
                blocked.add(entity.key)
                continue
            if not entry.enabled:
                _set_registry_phase(session, ledger, unique_id, "complete", expected)
                continue
            _publish_registry_config(session, ledger, entry, discovery_prefix, state_prefix, expected, blocked)
        elif phase == "republished":
            if observed == expected:
                _set_registry_phase(session, ledger, unique_id, "complete", expected)
            elif observed is None and entry.enabled:
                _publish_registry_config(session, ledger, entry, discovery_prefix, state_prefix, expected, blocked)
            else:
                # A stale/wrong device binding must pass through deletion again.
                try:
                    if _publish_ok(_discovery_topic(entity, discovery_prefix), b""):
                        _set_registry_phase(session, ledger, unique_id, "unloaded", expected)
                except Exception:
                    logger.exception("publish_discovery: unable to unload wrong registry entity %r", entity.key)
                blocked.add(entity.key)
        elif phase != "complete":
            if observed == expected or (observed is None and not entry.enabled):
                _set_registry_phase(session, ledger, unique_id, "complete", expected)
            elif observed is None:
                _publish_registry_config(session, ledger, entry, discovery_prefix, state_prefix, expected, blocked)
            else:
                try:
                    if _publish_ok(_discovery_topic(entity, discovery_prefix), b""):
                        _set_registry_phase(session, ledger, unique_id, "unloaded", expected)
                        blocked.add(entity.key)
                    else:
                        logger.warning("publish_discovery: broker rejected registry unload for %r", entity.key)
                except Exception:
                    logger.exception("publish_discovery: unable to unload registry entity %r", entity.key)
    return blocked


def _set_registry_phase(
    session: Session, ledger: dict[str, Any], unique_id: str, phase: str, expected: set[str]
) -> None:
    ledger[unique_id] = {"phase": phase, "identifier": sorted(expected)}
    _set_migration_json(session, _REGISTRY_REPAIR_KEY, ledger)


def _publish_registry_config(
    session: Session, ledger: dict[str, Any], entry: Any, discovery_prefix: str,
    state_prefix: str, expected: set[str], blocked: set[str],
) -> None:
    entity = entry.entity
    topic, config = build_discovery_payload(entity, discovery_prefix, state_prefix)
    try:
        if _publish_ok(topic, json.dumps(config)):
            _set_registry_phase(session, ledger, _unique_id(entity), "republished", expected)
        else:
            logger.warning("publish_discovery: broker rejected registry re-add for %r", entity.key)
    except Exception:
        logger.exception("publish_discovery: unable to re-add registry entity %r", entity.key)
    blocked.add(entity.key)


def _legacy_discovery_topic(entity: ExposableEntity, prefix: str) -> str:
    """Return the exact v1.6.1 config topic for a known stale M8 entity.

    This intentionally preserves the old hyphen-only conversion because the
    point is to remove that exact retained broker key once, never to publish a
    wildcard or manufacture a new invalid topic.
    """
    node = entity.device.internal_identity.replace("-", "_")
    obj = entity.key.replace(".", "_").replace("-", "_")
    return f"{prefix}/{entity.component}/{node}/{obj}/config"


def _overlapping_thermal_pairs(session: Session) -> list[tuple[Any, Any]]:
    """Return only heating/water Meter epochs that could have coexisted."""
    from app.models.energy import Meter
    from sqlalchemy import select

    meters = session.execute(select(Meter).where(
        Meter.commodity.in_(("electricity", "heating", "hot_water"))
    )).scalars().all()
    heatings = [meter for meter in meters if meter.commodity == "heating"]
    waters = [meter for meter in meters if meter.commodity == "hot_water"]
    pairs: list[tuple[Any, Any]] = []
    for heating in heatings:
        for water in waters:
            # A thermal identity can only have been published when both Meter
            # epochs were current at the same instant.  Do not form a Cartesian
            # product of historical records: that would tombstone identities
            # which have never existed in HA.
            heating_start, water_start = heating.started_at, water.started_at
            heating_end, water_end = heating.ended_at, water.ended_at
            if (heating_end is not None and water_start >= heating_end) or (
                water_end is not None and heating_start >= water_end
            ):
                continue
            pairs.append((heating, water))
    return pairs


def _thermal_cleanup_entities(
    pairs: list[tuple[Any, Any]], *, include_hot_water_total: bool
) -> list[ExposableEntity]:
    """Build exact synthetic config entries for known thermal identities."""
    from app.integrations.expose import DeviceInfo

    metrics = ["heating", "hot_water_heating", "water", "water_tax", "fixed", "all_in"]
    if include_hot_water_total:
        metrics.insert(2, "hot_water_total")
    entities: list[ExposableEntity] = []
    for heating, water in pairs:
        identity = ".".join(sorted((heating.uuid, water.uuid)))
        info = DeviceInfo(
            identifiers=(f"home-automation:thermal-cost:{identity}",),
            name="obsolete",
            identity=identity,
        )
        for metric in metrics:
            for suffix in ("total", "today"):
                entities.append(ExposableEntity(
                    key=f"thermal_cost.{identity}.{metric}_{suffix}", component="sensor", device=info,
                    device_class=None, unit="", name="obsolete",
                ))
    return entities


def _legacy_thermal_entities(session: Session) -> list[ExposableEntity]:
    """Return only v1.6.1 illegal topics that could retain a non-empty config.

    v1.6.1 published an illegal thermal config only while its expose toggle was
    enabled.  A disabled toggle published its own tombstone, so querying the
    durable toggle state prevents a fresh install from manufacturing warnings
    for twelve never-used illegal topics.
    """
    from app.models.expose import ExposedEntityToggle

    entities = _thermal_cleanup_entities(_overlapping_thermal_pairs(session), include_hot_water_total=False)
    keys = [entity.key for entity in entities]
    enabled_keys = {
        row.key for row in session.query(ExposedEntityToggle).filter(
            ExposedEntityToggle.key.in_(keys), ExposedEntityToggle.enabled.is_(True)
        )
    }
    return [entity for entity in entities if entity.key in enabled_keys]


def _stale_m8_entities(session: Session) -> list[ExposableEntity]:
    """Return *safe-format* configs that became stale after a later epoch swap.

    Unlike the v1.6.1 cleanup, this intentionally excludes the active thermal
    pair and includes the post-repair ``hot_water_total`` metric (14 configs
    per stale pair).  All generated topics use ``_discovery_topic``.
    """
    from app.integrations.expose import DeviceInfo
    from app.models.energy import Meter
    from sqlalchemy import select

    meters = session.execute(select(Meter).where(
        Meter.commodity.in_(("electricity", "heating", "hot_water"))
    )).scalars().all()
    entities: list[ExposableEntity] = []
    for meter in (meter for meter in meters if meter.ended_at is not None):
        info = DeviceInfo(
            identifiers=(f"home-automation:meter:{meter.uuid}",), name=meter.label, identity=meter.uuid
        )
        for suffix in ("total", "today"):
            entities.append(ExposableEntity(
                key=f"meter.{meter.uuid}.{suffix}", component="sensor", device=info,
                device_class=None, unit="", name="obsolete",
            ))
    stale_pairs = [
        (heating, water)
        for heating, water in _overlapping_thermal_pairs(session)
        if heating.ended_at is not None or water.ended_at is not None
    ]
    entities.extend(_thermal_cleanup_entities(stale_pairs, include_hot_water_total=True))
    return entities


# ---------------------------------------------------------------------------
# Public: publish states
# ---------------------------------------------------------------------------


def publish_states(session: Session) -> None:
    """Publish current state values for all enabled entities.

    Calls each entity's ``value_getter`` to obtain the current value and
    publishes it to the entity's state topic.  Also publishes availability
    (online/offline) for each device.

    No-op if MQTT / discovery is not enabled or the client is not connected.
    All errors are caught internally — this function never raises.
    """
    from app.config import get_settings
    settings = build_runtime_settings(session, get_settings())
    if not _should_publish(settings):
        logger.debug("publish_states: skipped (MQTT/Discovery not enabled or not connected)")
        return

    state_prefix = settings.ha_state_topic_prefix

    try:
        catalog = build_catalog(session)
    except Exception:
        logger.exception("publish_states: failed to build catalog; aborting")
        return

    # Collect enabled entries and publish
    for entry in catalog:
        if not entry.enabled:
            continue
        entity = entry.entity
        try:
            _publish_entity_state(entity, state_prefix, session)
        except Exception:
            logger.exception(
                "publish_states: error publishing state for %r; continuing", entity.key
            )


def _publish_entity_state(
    entity: ExposableEntity,
    prefix: str,
    session: Session,
) -> None:
    """Publish one entity's current state value to its state topic.

    Also publishes the availability topic for ``binary_sensor`` "online" entities.
    """
    state_t = _state_topic(entity, prefix)
    # Source-backed entities can have a different liveness identity from their
    # HA device identity.  Publish it before the state; a None value below is
    # intentionally not converted to a synthetic zero.
    if entity.device.provides_availability and entity.device.availability_getter is not None:
        try:
            available = bool(entity.device.availability_getter(session))
            mqtt_manager.publish(
                _availability_topic(_availability_id(entity), prefix),
                "online" if available else "offline",
                retain=False,
            )
        except Exception:
            logger.exception("availability_getter raised for entity %r", entity.key)

    if entity.component == "binary_sensor" and "online" in entity.key:
        # The online sensor represents device availability.
        # Ask the value_getter for the current ON/OFF string, then also publish
        # the shared availability topic (online/offline).
        raw_value = None
        if entity.value_getter is not None:
            try:
                raw_value = entity.value_getter(session)
            except Exception:
                logger.exception("value_getter raised for online entity %r", entity.key)
        # Default to offline when no reading is available.
        online = (raw_value == "ON")
        avail_payload = "online" if online else "offline"
        avail_topic = _availability_topic(_availability_id(entity), prefix)
        mqtt_manager.publish(avail_topic, avail_payload, retain=False)
        # The state of the binary_sensor itself
        state_payload = "ON" if online else "OFF"
        mqtt_manager.publish(state_t, state_payload, retain=False)
    else:
        # Regular sensor: call value_getter(session) if available.
        value = None
        if entity.value_getter is not None:
            try:
                value = entity.value_getter(session)
            except Exception:
                logger.exception("value_getter raised for entity %r", entity.key)

        if value is None:
            logger.debug("publish_states: no value for %r — skipping state publish", entity.key)
            return

        mqtt_manager.publish(state_t, str(value), retain=False)


# ---------------------------------------------------------------------------
# Public: per-device state push (called by modbus_poll after each poll)
# ---------------------------------------------------------------------------


def publish_device_state(session: Session, device: Any) -> None:
    """Publish state + availability for all enabled entities of *device*.

    Called by ``modbus_poll.poll_device`` after a successful (or failed) poll.
    Publishes:
    - The current availability topic ("online" / "offline") for the device.
    - The state topic for each **enabled** entity of the device.

    No-op if MQTT / discovery is not enabled or the client is not connected.
    All errors are caught internally — never raises.
    """
    from app.config import get_settings
    settings = build_runtime_settings(session, get_settings())
    if not _should_publish(settings):
        return

    state_prefix = settings.ha_state_topic_prefix
    device_uuid = device.uuid
    online = bool(device.last_poll_ok)

    # Publish availability topic first.
    try:
        avail_topic = _availability_topic(device_uuid, state_prefix)
        avail_payload = "online" if online else "offline"
        mqtt_manager.publish(avail_topic, avail_payload, retain=False)
    except Exception:
        logger.exception(
            "publish_device_state: failed to publish availability for device uuid=%s", device_uuid
        )

    if not online:
        # Device is offline — no point pushing stale state values.
        return

    # Publish state for each enabled entity of this device.
    try:
        catalog = build_catalog(session)
    except Exception:
        logger.exception(
            "publish_device_state: failed to build catalog for device uuid=%s", device_uuid
        )
        return

    for entry in catalog:
        if not entry.enabled:
            continue
        entity = entry.entity
        # Only process entities belonging to this device.
        if entity.device.internal_identity != device_uuid:
            continue
        # Skip the online binary_sensor itself (availability handled above).
        if entity.component == "binary_sensor" and "online" in entity.key:
            # Publish the binary_sensor state too.
            try:
                state_t = _state_topic(entity, state_prefix)
                mqtt_manager.publish(state_t, "ON", retain=False)
            except Exception:
                logger.exception(
                    "publish_device_state: error publishing online sensor state for %r",
                    entity.key,
                )
            continue

        # Regular sensor — call value_getter(session).
        try:
            value = None
            if entity.value_getter is not None:
                value = entity.value_getter(session)
            if value is None:
                continue
            state_t = _state_topic(entity, state_prefix)
            mqtt_manager.publish(state_t, str(value), retain=False)
        except Exception:
            logger.exception(
                "publish_device_state: error publishing state for entity %r", entity.key
            )


def clear_device_discovery(session: Session, device_uuid: str) -> None:
    """Best-effort: send empty retained payloads to all HA Discovery config topics
    for the given device, effectively removing the device's entities from HA.

    This must be called **before** the device rows are deleted from the DB, so
    that ``build_catalog`` can still enumerate the device's entities.

    Behaviour
    ---------
    - No-op if MQTT / HA Discovery is not enabled or the MQTT client is not
      connected.
    - All exceptions are caught internally; this function never raises.
      The caller proceeds with the DB deletion regardless of MQTT outcome.

    Parameters
    ----------
    session:
        Active SQLAlchemy session (device must still exist in DB at call time).
    device_uuid:
        UUID string of the device being deleted.
    """
    from app.config import get_settings
    settings = build_runtime_settings(session, get_settings())
    if not _should_publish(settings):
        logger.debug(
            "clear_device_discovery: skipped (MQTT/Discovery not enabled or not connected)"
        )
        return

    discovery_prefix = settings.ha_discovery_prefix
    state_prefix = settings.ha_state_topic_prefix

    try:
        catalog = build_catalog(session)
    except Exception:
        logger.exception(
            "clear_device_discovery: failed to build catalog for device uuid=%s; skipping HA cleanup",
            device_uuid,
        )
        return

    cleared = 0
    for entry in catalog:
        entity = entry.entity
        # Only clear entities belonging to this device.
        if entity.device.internal_identity != device_uuid:
            continue
        try:
            topic, _config = build_discovery_payload(entity, discovery_prefix, state_prefix)
            mqtt_manager.publish(topic, b"", retain=True)
            cleared += 1
            logger.debug(
                "clear_device_discovery: cleared config topic for entity %r → %s",
                entity.key,
                topic,
            )
        except Exception:
            logger.exception(
                "clear_device_discovery: error clearing entity %r; continuing", entity.key
            )

    logger.info(
        "clear_device_discovery: cleared %d HA discovery topic(s) for device uuid=%s",
        cleared,
        device_uuid,
    )


def publish_device_offline(session: Session, device_uuid: str) -> None:
    """Publish an "offline" availability payload for *device_uuid*.

    Called by ``modbus_poll.poll_device`` when a poll fails, to immediately
    reflect the device going offline in HA (before the next full state sweep).

    No-op if MQTT / discovery is not enabled or the client is not connected.
    Never raises.
    """
    from app.config import get_settings
    settings = build_runtime_settings(session, get_settings())
    if not _should_publish(settings):
        return

    try:
        state_prefix = settings.ha_state_topic_prefix
        avail_topic = _availability_topic(device_uuid, state_prefix)
        mqtt_manager.publish(avail_topic, "offline", retain=False)
    except Exception:
        logger.exception(
            "publish_device_offline: failed for device uuid=%s", device_uuid
        )