M8-T09: add WarmteLink ingest state machine

This commit is contained in:
2026-08-23 21:22:06 +02:00
parent 25a08c47a4
commit afe653bafa
3 changed files with 983 additions and 1 deletions
+362
View File
@@ -0,0 +1,362 @@
"""Privacy-preserving WarmteLink frame admission and minute sampling.
The serial worker added later owns I/O. This module deliberately only accepts
already parsed :class:`P1Telegram` instances (or a parser callable at its
small convenience entry point), so rejected telegram bytes never enter the
database or an exception message.
"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from decimal import Decimal
import re
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.db import get_session_local
from app.integrations.p1 import IntegrityStatus, P1Channel, P1Telegram, parse_telegram
from app.models.meter_source import MeterSource, WarmteLinkReading
from app.services.meter_sources import upsert_discovered_channel
@dataclass(frozen=True)
class _ChannelSample:
key: str
label: str
value: Decimal
unit: str
device_type: str | None
fingerprint: str | None
identity: tuple[object, ...]
@dataclass(frozen=True)
class _FrameSnapshot:
recorded_at: datetime
fingerprint: str | None
samples: tuple[_ChannelSample, ...]
class WarmteLinkIngestor:
"""Keep unverifiable candidates isolated by source for one worker lifetime."""
def __init__(
self,
*,
clock: Callable[[], datetime] | None = None,
) -> None:
self._clock = clock or (lambda: datetime.now(UTC))
self._previous: dict[int, _FrameSnapshot] = {}
def ingest(self, session: Session, *, source_id: int, telegram: P1Telegram) -> bool:
"""Apply one parsed telegram in the caller's transaction.
Returns whether the frame was admitted. Callers that own a session
must commit on success; :func:`handle_frame` is the failure-contained
worker-facing entry point.
"""
source = session.get(MeterSource, source_id)
if source is None:
raise ValueError("WarmteLink source was not found")
if source.kind != "warmtelink_serial":
raise ValueError("Source is not a WarmteLink serial source")
try:
integrity = _integrity_status(telegram.integrity)
except Exception:
self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame could not be normalized")
return False
if integrity is IntegrityStatus.INVALID:
self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame checksum is invalid")
return False
try:
snapshot = _snapshot(telegram)
fingerprints = _final_fingerprints(snapshot)
except Exception:
# A parser DTO remains an untrusted boundary. Do not let an
# unnormalizable DTO bridge two otherwise matching candidates.
self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame could not be normalized")
return False
if fingerprints is None:
self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame fingerprint is invalid")
return False
if integrity is IntegrityStatus.UNVERIFIABLE:
previous = self._previous.get(source_id)
if previous is None:
self._previous[source_id] = snapshot
self._diagnose(source, "Awaiting a second matching unverifiable WarmteLink frame")
return False
reason = _continuity_problem(previous, snapshot)
if reason is not None:
self._previous[source_id] = snapshot
self._diagnose(source, reason)
return False
try:
self._admit(session, source, snapshot, integrity.value, fingerprints)
except Exception:
# ``ingest`` owns flushes and may be used directly by tests or
# future callers. A failed write must never become a speculative
# predecessor for this source.
self._previous.pop(source_id, None)
raise
if integrity is IntegrityStatus.UNVERIFIABLE:
# Advance the sliding predecessor only once every database write
# for this frame has succeeded. ``handle_frame`` also clears it
# if the caller's later commit fails.
self._previous[source_id] = snapshot
else:
# A verified frame has no need for a speculative predecessor.
self._previous.pop(source_id, None)
return True
def handle_frame(
self,
source_id: int,
frame: bytes,
*,
session_factory: Callable[[], Session] = get_session_local,
parser: Callable[[bytes], P1Telegram] = parse_telegram,
) -> bool:
"""Parse and persist one frame, containing both parse and DB failures.
A failed write is rolled back before a fresh transaction records only
a generic source error. Consequently no partial latest/history update
survives and the following frame may recover normally.
"""
try:
telegram = parser(frame)
except Exception:
self._previous.pop(source_id, None)
self._record_error(session_factory, source_id, "WarmteLink frame could not be parsed")
return False
try:
with session_factory() as session:
admitted = self.ingest(session, source_id=source_id, telegram=telegram)
session.commit()
return admitted
except Exception:
self._previous.pop(source_id, None)
self._record_error(session_factory, source_id, "WarmteLink ingest failed")
return False
def _admit(
self,
session: Session,
source: MeterSource,
snapshot: _FrameSnapshot,
quality: str,
fingerprints: tuple[str, ...],
) -> None:
received_at = _utc(self._clock())
for sample, fingerprint in zip(snapshot.samples, fingerprints, strict=True):
channel = upsert_discovered_channel(
session,
source_id=source.id,
channel_key=sample.key,
label=sample.label,
unit=sample.unit,
suggested_commodity=_suggestion(sample.unit),
device_type=sample.device_type,
fingerprint=fingerprint,
latest_value=sample.value,
latest_at=snapshot.recorded_at,
latest_quality=quality,
)
session.flush()
bucket = snapshot.recorded_at.replace(second=0, microsecond=0)
exists = session.scalar(
select(WarmteLinkReading.id)
.where(
WarmteLinkReading.channel_id == channel.id,
WarmteLinkReading.recorded_at >= bucket,
WarmteLinkReading.recorded_at < bucket + timedelta(minutes=1),
)
.limit(1)
)
if exists is None:
session.add(
WarmteLinkReading(
channel_id=channel.id,
recorded_at=snapshot.recorded_at,
received_at=received_at,
value=sample.value,
unit=sample.unit,
quality=quality,
equipment_fingerprint=fingerprint,
)
)
source.status = "online"
source.last_seen_at = received_at
source.last_error = None
source.updated_at = received_at
def _diagnose(self, source: MeterSource, reason: str) -> None:
now = _utc(self._clock())
source.status = "error"
source.last_error = reason
source.updated_at = now
def _record_error(
self, session_factory: Callable[[], Session], source_id: int, message: str) -> None:
try:
with session_factory() as session:
source = session.get(MeterSource, source_id)
if source is not None:
self._diagnose(source, message)
session.commit()
except Exception:
# Error reporting itself must not kill another source worker.
return
def _snapshot(telegram: P1Telegram) -> _FrameSnapshot:
recorded_at = _parse_timestamp(telegram.timestamp)
samples = tuple(_sample(channel) for channel in telegram.channels)
if not samples:
raise ValueError("WarmteLink telegram contains no cumulative channels")
return _FrameSnapshot(recorded_at, telegram.equipment_fingerprint, samples)
def _sample(channel: P1Channel) -> _ChannelSample:
profile = _canonical_channel_profile(channel.number)
if channel.device_type != profile.device_type:
raise ValueError("WarmteLink channel device type is not canonical")
if len(channel.readings) != 1:
raise ValueError("WarmteLink channel has no unambiguous cumulative reading")
reading = channel.readings[0]
if reading.code != profile.reading_code or reading.value is None or reading.unit != profile.raw_unit:
raise ValueError("WarmteLink cumulative reading is incomplete")
# Parser annotations are not a trust boundary: fake or future parser DTOs
# must not put a float (including NaN) or a non-finite Decimal into the
# per-source unverifiable candidate state. Do not coerce here: accepting
# another numeric type would make the persistence and continuity paths
# disagree about the cumulative-value contract.
if not isinstance(reading.value, Decimal) or not reading.value.is_finite():
raise ValueError("WarmteLink cumulative reading is not a finite Decimal")
fingerprint = channel.equipment_fingerprint
return _ChannelSample(
key=profile.key,
label=profile.label,
value=reading.value,
unit=profile.unit,
device_type=profile.device_type,
fingerprint=fingerprint,
identity=(channel.number, profile.device_type, fingerprint, profile.reading_code, profile.unit),
)
@dataclass(frozen=True)
class _CanonicalChannelProfile:
key: str
label: str
device_type: str
reading_code: str
raw_unit: str
unit: str
_CANONICAL_CHANNELS = {
1: _CanonicalChannelProfile(
key="channel-1",
label="WarmteLink channel 1",
device_type="006",
reading_code="0-1:24.2.1",
raw_unit="m3",
unit="",
),
2: _CanonicalChannelProfile(
key="channel-2",
label="WarmteLink channel 2",
device_type="012",
reading_code="0-2:24.2.1",
raw_unit="GJ",
unit="GJ",
),
}
def _canonical_channel_profile(number: object) -> _CanonicalChannelProfile:
# Do not format or coerce parser supplied channel numbers: that could turn
# an arbitrary object into an identity key before it is rejected.
if type(number) is not int:
raise ValueError("WarmteLink channel number is not canonical")
try:
return _CANONICAL_CHANNELS[number]
except KeyError as exc:
raise ValueError("WarmteLink channel is not supported by the profile") from exc
def _integrity_status(value: object) -> IntegrityStatus:
"""Require a real parser integrity enum, never a look-alike value object."""
if not isinstance(value, IntegrityStatus):
raise ValueError("WarmteLink integrity status is not canonical")
return value
def _continuity_problem(previous: _FrameSnapshot, current: _FrameSnapshot) -> str | None:
if current.recorded_at <= previous.recorded_at:
return "Unverifiable WarmteLink timestamp is not strictly increasing"
if current.recorded_at - previous.recorded_at != timedelta(seconds=10):
return "Unverifiable WarmteLink frame cadence is not 10 seconds"
if current.fingerprint != previous.fingerprint:
return "WarmteLink equipment metadata changed"
if tuple(sample.identity for sample in current.samples) != tuple(sample.identity for sample in previous.samples):
return "WarmteLink channel metadata changed or channel set changed"
old_values = {sample.key: sample.value for sample in previous.samples}
if any(sample.value < old_values[sample.key] for sample in current.samples):
return "WarmteLink cumulative value decreased"
return None
_FINGERPRINT_PATTERN = re.compile(r"[0-9a-f]{64}")
def _final_fingerprints(snapshot: _FrameSnapshot) -> tuple[str, ...] | None:
"""Return only canonical SHA-256 hexdigests safe to persist.
A parser DTO is an untrusted boundary: even a field named ``fingerprint``
can contain a raw equipment identifier. Validate the complete DTO before
choosing persisted values: the top-level value and every channel value
must independently be canonical hashes. This deliberately does not use
a top-level fallback for an absent or malformed channel fingerprint.
"""
values = (snapshot.fingerprint, *(sample.fingerprint for sample in snapshot.samples))
if any(not _is_canonical_fingerprint(value) for value in values):
return None
return tuple(sample.fingerprint for sample in snapshot.samples if sample.fingerprint is not None)
def _is_canonical_fingerprint(value: str | None) -> bool:
return value is not None and _FINGERPRINT_PATTERN.fullmatch(value) is not None
def _parse_timestamp(value: str | None) -> datetime:
if value is None or len(value) != 13 or value[-1] not in {"S", "W"} or not value[:-1].isdigit():
raise ValueError("WarmteLink timestamp is unavailable")
naive = datetime.strptime(value[:-1], "%y%m%d%H%M%S")
# The telegram's S/W marker is evidence of the intended Amsterdam offset.
offset = 2 if value[-1] == "S" else 1
return (naive.replace(tzinfo=UTC) - timedelta(hours=offset)).replace(tzinfo=UTC)
def _suggestion(unit: str) -> str | None:
return {"GJ": "heating", "": "hot_water"}.get(unit)
def _canonical_unit(unit: str) -> str:
"""Map the P1 spelling of cubic metres to the source-profile unit."""
return "" if unit == "m3" else unit
def _utc(value: datetime) -> datetime:
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+1 -1
View File
@@ -615,7 +615,7 @@ T01T06 先把现有 DSMR 安全迁到统一 source/bindingT07T11 再接
### M8-T09 — WarmteLink 质量接纳、发现与分钟采样 [structural]
- **Status**: `todo`
- **Status**: `done`
- **Depends**: M8-T08
- **Context**: 把 parser 输出变成可审计的 latest/history;这里锁住最关键的 unverifiable 接纳策略。
+620
View File
@@ -0,0 +1,620 @@
"""Behaviour tests for the WarmteLink per-source admission state machine."""
from __future__ import annotations
from dataclasses import replace
from datetime import UTC, datetime
from decimal import Decimal
from pathlib import Path
from types import SimpleNamespace
import pytest
from alembic import command
from alembic.config import Config
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session, sessionmaker
from app.integrations.p1 import IntegrityStatus, ObisField, P1Channel, P1Telegram, dsmr_crc16, parse_telegram
from app.models.meter_source import MeterSource, MeterSourceBinding, MeterSourceChannel, WarmteLinkReading
from app.services.warmtelink_ingest import WarmteLinkIngestor
def _config(url: str) -> Config:
config = Config("alembic_app.ini")
config.set_main_option("sqlalchemy.url", url)
return config
@pytest.fixture()
def database(tmp_path: Path):
url = f"sqlite:///{tmp_path / 'warmtelink_ingest.db'}"
command.upgrade(_config(url), "head")
engine = create_engine(url, connect_args={"check_same_thread": False})
factory = sessionmaker(bind=engine, autoflush=False, class_=Session)
with factory() as session:
for name in ("one", "two"):
now = datetime(2026, 8, 22, tzinfo=UTC)
session.add(
MeterSource(
name=name,
kind="warmtelink_serial",
enabled=True,
config={"path": f"/dev/{name}", "baudrate": 115200, "data_bits": 7, "parity": "N", "stop_bits": 1},
created_at=now,
updated_at=now,
)
)
session.commit()
ids = list(session.scalars(select(MeterSource.id).order_by(MeterSource.id)))
yield engine, factory, ids[-2:]
engine.dispose()
def _telegram(
*,
second: int,
minute: int = 0,
integrity: IntegrityStatus = IntegrityStatus.UNVERIFIABLE,
water: str = "5.900",
heat: str = "0.017",
channels: tuple[int, ...] = (1, 2),
device_type: str = "006",
fingerprint: str | None = "a" * 64,
channel_fingerprint: str | None = None,
channel_fingerprints: dict[int, str | None] | None = None,
) -> P1Telegram:
fields: list[ObisField] = []
parsed_channels: list[P1Channel] = []
for number in channels:
value, unit = (water, "m3") if number == 1 else (heat, "GJ")
code = f"0-{number}:24.2.1"
field = ObisField(code, (value + "*" + unit,), Decimal(value), unit)
fields.append(field)
parsed_channels.append(
P1Channel(
number,
device_type if number == 1 else "012",
(
channel_fingerprints[number]
if channel_fingerprints is not None
else fingerprint if channel_fingerprint is None else channel_fingerprint
),
(field,),
)
)
return P1Telegram(
100,
integrity,
"fixture",
f"26082212{minute:02d}{second:02d}S",
fingerprint,
tuple(fields),
tuple(parsed_channels),
)
def _counts(factory) -> tuple[int, int]:
with factory() as session:
return (
# Revision 16 supplies one DSMR channel; this task must not touch it.
int(session.scalar(select(func.count(MeterSourceChannel.id))) or 0) - 1,
int(session.scalar(select(func.count(WarmteLinkReading.id))) or 0),
)
def _all_counts(factory) -> tuple[int, int, int]:
channels, readings = _counts(factory)
with factory() as session:
bindings = int(session.scalar(select(func.count(MeterSourceBinding.id))) or 0)
return channels, readings, bindings
def _fixed_clock() -> datetime:
return datetime(2026, 8, 22, 10, tzinfo=UTC)
def test_unverifiable_requires_two_continuous_frames_and_preserves_true_metadata(database):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
session.commit()
assert _counts(factory) == (0, 0)
with factory() as session:
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=20, water="5.902"))
session.commit()
assert _counts(factory) == (2, 2)
with factory() as session:
channels = list(
session.scalars(
select(MeterSourceChannel)
.where(MeterSourceChannel.source_id == source_id)
.order_by(MeterSourceChannel.channel_key)
)
)
assert [(item.channel_key, item.unit, item.device_type, item.latest_quality) for item in channels] == [
("channel-1", "", "006", "unverifiable"),
("channel-2", "GJ", "012", "unverifiable"),
]
assert all(item.fingerprint == "a" * 64 for item in channels)
assert channels[0].latest_value == Decimal("5.902")
def test_unverifiable_cadence_requires_exactly_ten_seconds_and_restarts_after_gaps(database):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, water="5.901"))
session.commit()
assert _counts(factory) == (2, 2)
restarted = WarmteLinkIngestor()
with factory() as session:
assert not restarted.ingest(session, source_id=source_id, telegram=_telegram(second=0, water="5.902"))
# 20 seconds misses one expected frame; it is only a new candidate.
assert not restarted.ingest(session, source_id=source_id, telegram=_telegram(second=20, water="5.903"))
assert restarted.ingest(session, source_id=source_id, telegram=_telegram(second=30, water="5.904"))
# A larger gap likewise needs its own new matching successor.
assert not restarted.ingest(session, source_id=source_id, telegram=_telegram(second=0, minute=2, water="5.905"))
assert restarted.ingest(session, source_id=source_id, telegram=_telegram(second=10, minute=2, water="5.906"))
session.commit()
assert _counts(factory) == (2, 4)
@pytest.mark.parametrize(
"first, second",
[
(_telegram(second=10), _telegram(second=9)),
(_telegram(second=0), _telegram(second=30)),
(_telegram(second=10), _telegram(second=20, water="5.899")),
(_telegram(second=10), _telegram(second=20, device_type="007")),
(_telegram(second=10), _telegram(second=20, channels=(1,))),
(_telegram(second=10, channels=(1,)), _telegram(second=20)),
],
)
def test_unverifiable_discontinuities_restart_two_frame_confirmation(database, first, second):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=first)
assert not ingest.ingest(session, source_id=source_id, telegram=second)
session.commit()
assert _counts(factory) == (0, 0)
with factory() as session:
source = session.get(MeterSource, source_id)
assert source is not None and source.status == "error" and source.last_error is not None
def test_discontinuity_candidate_needs_a_new_matching_successor(database):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, channels=(1,)))
# The restored channel set is a fresh candidate, not an admission.
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=20))
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=30, water="5.901"))
session.commit()
assert _counts(factory) == (2, 2)
@pytest.mark.parametrize(
"first, discontinuous, successor",
[
(_telegram(second=0), _telegram(second=10, water="5.899"), _telegram(second=20, water="5.900")),
(_telegram(second=0), _telegram(second=10, fingerprint="b" * 64), _telegram(second=20, fingerprint="b" * 64)),
(_telegram(second=0), _telegram(second=10, channels=(1,)), _telegram(second=20, channels=(1,))),
(_telegram(second=0, channels=(1,)), _telegram(second=10), _telegram(second=20)),
],
)
def test_each_metadata_or_channel_discontinuity_requires_its_own_two_frames(
database, first, discontinuous, successor
):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=first)
# The discontinuous frame becomes a new first candidate, never an admission.
assert not ingest.ingest(session, source_id=source_id, telegram=discontinuous)
assert ingest.ingest(session, source_id=source_id, telegram=successor)
session.commit()
def test_parser_failure_clears_only_its_source_candidate_and_requires_two_new_frames(database):
_engine, factory, (one, two) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
session.commit()
def parse_failure(_frame: bytes) -> P1Telegram:
raise ValueError("raw telegram must never be reported")
assert not ingest.handle_frame(one, b"private", session_factory=factory, parser=parse_failure)
with factory() as session:
# One's candidate was cleared; two's independent candidate survives.
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=two, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=one, telegram=_telegram(second=20, water="5.902"))
session.commit()
assert _counts(factory) == (4, 4)
def _normalization_failure(kind: str) -> P1Telegram:
telegram = _telegram(second=10)
if kind == "no-channels":
return replace(telegram, channels=())
first, *rest = telegram.channels
if kind == "empty-readings":
return replace(telegram, channels=(replace(first, readings=()), *rest))
if kind == "ambiguous-readings":
return replace(telegram, channels=(replace(first, readings=(first.readings[0],) * 2), *rest))
if kind == "incomplete-reading":
return replace(
telegram,
channels=(replace(first, readings=(replace(first.readings[0], value=None),)), *rest),
)
invalid_values: dict[str, object] = {
"decimal-nan": Decimal("NaN"),
"decimal-positive-infinity": Decimal("Infinity"),
"decimal-negative-infinity": Decimal("-Infinity"),
"float": 5.9,
"string": "private-invalid-cumulative-value",
}
if kind in invalid_values:
return replace(
telegram,
channels=(
replace(first, readings=(replace(first.readings[0], value=invalid_values[kind]),)),
*rest,
),
)
if kind == "invalid-timestamp":
return replace(telegram, timestamp="invalid-time")
raise AssertionError(f"unexpected normalization failure kind: {kind}")
@pytest.mark.parametrize(
"kind",
[
"no-channels",
"empty-readings",
"ambiguous-readings",
"incomplete-reading",
"decimal-nan",
"decimal-positive-infinity",
"decimal-negative-infinity",
"float",
"string",
"invalid-timestamp",
],
)
def test_dto_normalization_failure_clears_only_its_source_candidate_and_requires_two_new_frames(
database, kind
):
_engine, factory, (one, two) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
session.commit()
failed = _normalization_failure(kind)
assert not ingest.handle_frame(one, b"private", session_factory=factory, parser=lambda _raw: failed)
# The rejected DTO creates no discovery/latest/history rows, and source two
# retains its independent first candidate.
assert _counts(factory) == (0, 0)
with factory() as session:
source = session.get(MeterSource, one)
assert source is not None and source.last_error == "WarmteLink frame could not be normalized"
assert "private" not in (source.last_error or "")
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=two, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=one, telegram=_telegram(second=20, water="5.902"))
session.commit()
assert _counts(factory) == (4, 4)
def test_valid_invalid_sampling_replay_restart_and_source_isolation(database):
_engine, factory, (one, two) = database
ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session:
assert ingest.ingest(session, source_id=one, telegram=_telegram(second=0, integrity=IntegrityStatus.VALID))
assert ingest.ingest(session, source_id=one, telegram=_telegram(second=10, integrity=IntegrityStatus.VALID, water="5.901"))
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=20, integrity=IntegrityStatus.INVALID))
# Source two cannot consume source one's candidate.
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=30))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=40))
session.commit()
assert _counts(factory) == (2, 2) # candidates are never discovered; first minute only once
# A new process sees the minute bucket in DB and does not create duplicates.
restarted = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session:
assert restarted.ingest(session, source_id=one, telegram=_telegram(second=50, integrity=IntegrityStatus.VALID))
session.commit()
assert _counts(factory) == (2, 2)
def test_history_keeps_first_utc_minute_sample_and_creates_next_minute(database):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session:
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0, integrity=IntegrityStatus.VALID))
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, integrity=IntegrityStatus.VALID, water="5.901"))
assert ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=0, minute=1, integrity=IntegrityStatus.VALID, water="5.902"),
)
session.commit()
with factory() as session:
readings = list(session.scalars(select(WarmteLinkReading).order_by(WarmteLinkReading.recorded_at)))
assert len(readings) == 4
assert readings[0].recorded_at.second == 0
assert readings[2].recorded_at.minute == 1 and readings[2].recorded_at.second == 0
channel = session.scalar(
select(MeterSourceChannel).where(
MeterSourceChannel.source_id == source_id,
MeterSourceChannel.channel_key == "channel-1",
)
)
assert channel is not None and channel.latest_value == Decimal("5.902")
def test_handle_frame_rolls_back_database_error_marks_source_and_recovers(database, monkeypatch):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor()
telegram = _telegram(second=0, integrity=IntegrityStatus.VALID)
import app.services.warmtelink_ingest as module
original = module.upsert_discovered_channel
def broken(*_args, **_kwargs):
raise RuntimeError("database unavailable")
monkeypatch.setattr(module, "upsert_discovered_channel", broken)
assert not ingest.handle_frame(source_id, b"private telegram", session_factory=factory, parser=lambda _raw: telegram)
assert _counts(factory) == (0, 0)
with factory() as session:
source = session.get(MeterSource, source_id)
assert source is not None and source.status == "error"
assert "private" not in (source.last_error or "")
monkeypatch.setattr(module, "upsert_discovered_channel", original)
assert ingest.handle_frame(source_id, b"ignored", session_factory=factory, parser=lambda _raw: telegram)
assert _counts(factory) == (2, 2)
def test_second_channel_failure_rolls_back_first_channel_latest_and_history(database, monkeypatch):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor()
import app.services.warmtelink_ingest as module
original = module.upsert_discovered_channel
def broken_on_second(*args, **kwargs):
if kwargs["channel_key"] == "channel-2":
raise RuntimeError("second channel failed")
return original(*args, **kwargs)
monkeypatch.setattr(module, "upsert_discovered_channel", broken_on_second)
assert not ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: _telegram(second=0, integrity=IntegrityStatus.VALID),
)
assert _counts(factory) == (0, 0)
monkeypatch.setattr(module, "upsert_discovered_channel", original)
assert ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: _telegram(second=10, integrity=IntegrityStatus.VALID),
)
assert _counts(factory) == (2, 2)
def test_unverifiable_database_failure_clears_only_its_source_sliding_candidate(database, monkeypatch):
_engine, factory, (one, two) = database
ingest = WarmteLinkIngestor()
import app.services.warmtelink_ingest as module
with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
session.commit()
original = module.upsert_discovered_channel
monkeypatch.setattr(module, "upsert_discovered_channel", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError()))
assert not ingest.handle_frame(
one, b"private", session_factory=factory, parser=lambda _raw: _telegram(second=10, water="5.901")
)
monkeypatch.setattr(module, "upsert_discovered_channel", original)
assert _all_counts(factory) == (0, 0, 0)
with factory() as session:
# One needs a new pair after its failed write; two retains its own candidate.
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=20, water="5.902"))
assert ingest.ingest(session, source_id=two, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=one, telegram=_telegram(second=30, water="5.903"))
session.commit()
assert _all_counts(factory) == (4, 4, 0)
def _valid_crc_frame(*, device_type: str = "006") -> bytes:
body = (
b"/WARMTE\r\n"
b"0-0:1.0.0(260822120000S)\r\n"
b"0-0:96.1.1(WARMTE-REDACTED)\r\n"
+ f"0-1:24.1.0({device_type})\r\n".encode()
+ b"0-1:96.1.0(KAM-REDACTED)\r\n"
+ b"0-1:24.2.1(260822120000S)(5.900*m3)\r\n"
+ b"0-2:24.1.0(012)\r\n"
+ b"0-2:96.1.0(KAM-REDACTED)\r\n"
+ b"0-2:24.2.1(260822120000S)(0.017*GJ)\r\n"
)
payload = body + b"!"
return payload + f"{dsmr_crc16(payload):04X}".encode() + b"\r\n"
def test_non_enum_integrity_and_fake_metadata_reject_whole_frame_per_source(database):
_engine, factory, (one, two) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
session.commit()
forged_quality = replace(_telegram(second=10, integrity=IntegrityStatus.VALID), integrity=SimpleNamespace(value="valid"))
assert not ingest.handle_frame(one, b"private", session_factory=factory, parser=lambda _raw: forged_quality)
mixed = replace(_telegram(second=10), channels=(replace(_telegram(second=10).channels[0], device_type="raw-device"),))
assert not ingest.handle_frame(one, b"private", session_factory=factory, parser=lambda _raw: mixed)
assert _all_counts(factory) == (0, 0, 0)
with factory() as session:
source = session.get(MeterSource, one)
assert source is not None and source.last_error == "WarmteLink frame could not be normalized"
# Both malformed frames clear one, while two's candidate is untouched.
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=20, water="5.901"))
assert ingest.ingest(session, source_id=two, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=one, telegram=_telegram(second=30, water="5.902"))
session.commit()
assert _all_counts(factory) == (4, 4, 0)
def test_valid_crc_parser_frame_with_raw_device_type_is_rejected_without_leakage(database):
_engine, factory, (one, two) = database
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
session.commit()
raw_identifier = "serial-number-which-must-not-persist"
assert parse_telegram(_valid_crc_frame(device_type=raw_identifier)).integrity is IntegrityStatus.VALID
assert not ingest.handle_frame(one, _valid_crc_frame(device_type=raw_identifier), session_factory=factory)
assert _all_counts(factory) == (0, 0, 0)
with factory() as session:
source = session.get(MeterSource, one)
assert source is not None and source.last_error == "WarmteLink frame could not be normalized"
assert raw_identifier not in (source.last_error or "")
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=two, telegram=_telegram(second=10, water="5.901"))
assert ingest.ingest(session, source_id=one, telegram=_telegram(second=20, water="5.902"))
session.commit()
assert _all_counts(factory) == (4, 4, 0)
@pytest.mark.parametrize(
"fingerprint, channel_fingerprint",
[
(None, None),
("", ""),
("a" * 63, "a" * 63),
("g" * 64, "g" * 64),
("serial-number-which-must-not-persist", "serial-number-which-must-not-persist"),
],
)
def test_invalid_fingerprint_fails_closed_without_persisting_identifiers(
database, fingerprint, channel_fingerprint
):
_engine, factory, (source_id, _) = database
raw_identifier = "serial-number-which-must-not-persist"
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0, integrity=IntegrityStatus.INVALID))
assert not ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(
second=10,
integrity=IntegrityStatus.VALID,
fingerprint=fingerprint,
channel_fingerprint=channel_fingerprint,
),
)
session.commit()
assert _counts(factory) == (0, 0)
with factory() as session:
source = session.get(MeterSource, source_id)
assert source is not None and source.last_error == "WarmteLink frame fingerprint is invalid"
assert raw_identifier not in (source.last_error or "")
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=20, integrity=IntegrityStatus.VALID))
session.commit()
rendered = " ".join(str(row) for row in session.execute(select(WarmteLinkReading)).all())
assert raw_identifier not in rendered
assert all("raw" not in column.name for column in WarmteLinkReading.__table__.columns)
@pytest.mark.parametrize(
"invalid_fingerprint",
[None, "", "a" * 63, "serial-number-which-must-not-persist"],
ids=["none", "empty", "wrong-length", "raw-like"],
)
@pytest.mark.parametrize("invalid_channel", [1, 2])
def test_invalid_channel_fingerprint_rejects_whole_frame_and_clears_candidate(
database, invalid_fingerprint, invalid_channel
):
_engine, factory, (source_id, _) = database
raw_identifier = "serial-number-which-must-not-persist"
valid = "a" * 64
ingest = WarmteLinkIngestor()
with factory() as session:
# Establish a candidate, then prove an invalid channel clears only it.
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
assert not ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(
second=10,
fingerprint=valid,
channel_fingerprints={1: valid, 2: valid} | {invalid_channel: invalid_fingerprint},
),
)
session.commit()
assert _counts(factory) == (0, 0)
with factory() as session:
source = session.get(MeterSource, source_id)
assert source is not None and source.last_error == "WarmteLink frame fingerprint is invalid"
assert raw_identifier not in (source.last_error or "")
# The first legal frame is a fresh candidate; only its successor admits.
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=20, water="5.901"))
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=30, water="5.902"))
session.commit()
assert _counts(factory) == (2, 2)
@pytest.mark.parametrize(
"invalid_fingerprint",
[None, "", "a" * 63, "serial-number-which-must-not-persist"],
ids=["none", "empty", "wrong-length", "raw-like"],
)
def test_invalid_top_fingerprint_rejects_whole_frame_and_clears_candidate(database, invalid_fingerprint):
_engine, factory, (source_id, _) = database
raw_identifier = "serial-number-which-must-not-persist"
valid = "a" * 64
ingest = WarmteLinkIngestor()
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
assert not ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(
second=10,
fingerprint=invalid_fingerprint,
channel_fingerprints={1: valid, 2: valid},
),
)
session.commit()
assert _counts(factory) == (0, 0)
with factory() as session:
source = session.get(MeterSource, source_id)
assert source is not None and source.last_error == "WarmteLink frame fingerprint is invalid"
assert raw_identifier not in (source.last_error or "")
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=20, water="5.901"))
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=30, water="5.902"))
session.commit()
assert _counts(factory) == (2, 2)