M8-R04: interpret WarmteLink timestamps as Amsterdam wall time

This commit is contained in:
2026-08-24 06:45:31 +02:00
parent 18083822ea
commit 72ac7e7300
2 changed files with 373 additions and 40 deletions
+62 -23
View File
@@ -13,6 +13,7 @@ from dataclasses import dataclass
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from decimal import Decimal from decimal import Decimal
import re import re
from zoneinfo import ZoneInfo
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -52,7 +53,14 @@ class WarmteLinkIngestor:
self._clock = clock or (lambda: datetime.now(UTC)) self._clock = clock or (lambda: datetime.now(UTC))
self._previous: dict[int, _FrameSnapshot] = {} self._previous: dict[int, _FrameSnapshot] = {}
def ingest(self, session: Session, *, source_id: int, telegram: P1Telegram) -> bool: def ingest(
self,
session: Session,
*,
source_id: int,
telegram: P1Telegram,
received_at: datetime | None = None,
) -> bool:
"""Apply one parsed telegram in the caller's transaction. """Apply one parsed telegram in the caller's transaction.
Returns whether the frame was admitted. Callers that own a session Returns whether the frame was admitted. Callers that own a session
@@ -65,44 +73,47 @@ class WarmteLinkIngestor:
if source.kind != "warmtelink_serial": if source.kind != "warmtelink_serial":
raise ValueError("Source is not a WarmteLink serial source") raise ValueError("Source is not a WarmteLink serial source")
received_at = _utc(self._clock()) if received_at is None else _utc(received_at)
try: try:
integrity = _integrity_status(telegram.integrity) integrity = _integrity_status(telegram.integrity)
except Exception: except Exception:
self._previous.pop(source_id, None) self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame could not be normalized") self._diagnose(source, "WarmteLink frame could not be normalized", now=received_at)
return False return False
if integrity is IntegrityStatus.INVALID: if integrity is IntegrityStatus.INVALID:
self._previous.pop(source_id, None) self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame checksum is invalid") self._diagnose(source, "WarmteLink frame checksum is invalid", now=received_at)
return False return False
try: try:
snapshot = _snapshot(telegram) snapshot = _snapshot(telegram, received_at)
fingerprints = _final_fingerprints(snapshot) fingerprints = _final_fingerprints(snapshot)
except Exception: except Exception:
# A parser DTO remains an untrusted boundary. Do not let an # A parser DTO remains an untrusted boundary. Do not let an
# unnormalizable DTO bridge two otherwise matching candidates. # unnormalizable DTO bridge two otherwise matching candidates.
self._previous.pop(source_id, None) self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame could not be normalized") self._diagnose(source, "WarmteLink frame could not be normalized", now=received_at)
return False return False
if fingerprints is None: if fingerprints is None:
self._previous.pop(source_id, None) self._previous.pop(source_id, None)
self._diagnose(source, "WarmteLink frame fingerprint is invalid") self._diagnose(source, "WarmteLink frame fingerprint is invalid", now=received_at)
return False return False
if integrity is IntegrityStatus.UNVERIFIABLE: if integrity is IntegrityStatus.UNVERIFIABLE:
previous = self._previous.get(source_id) previous = self._previous.get(source_id)
if previous is None: if previous is None:
self._previous[source_id] = snapshot self._previous[source_id] = snapshot
self._diagnose(source, "Awaiting a second matching unverifiable WarmteLink frame") self._diagnose(
source, "Awaiting a second matching unverifiable WarmteLink frame", now=received_at
)
return False return False
reason = _continuity_problem(previous, snapshot) reason = _continuity_problem(previous, snapshot)
if reason is not None: if reason is not None:
self._previous[source_id] = snapshot self._previous[source_id] = snapshot
self._diagnose(source, reason) self._diagnose(source, reason, now=received_at)
return False return False
try: try:
self._admit(session, source, snapshot, integrity.value, fingerprints) self._admit(session, source, snapshot, integrity.value, fingerprints, received_at)
except Exception: except Exception:
# ``ingest`` owns flushes and may be used directly by tests or # ``ingest`` owns flushes and may be used directly by tests or
# future callers. A failed write must never become a speculative # future callers. A failed write must never become a speculative
@@ -133,20 +144,25 @@ class WarmteLinkIngestor:
a generic source error. Consequently no partial latest/history update a generic source error. Consequently no partial latest/history update
survives and the following frame may recover normally. survives and the following frame may recover normally.
""" """
received_at = _utc(self._clock())
try: try:
telegram = parser(frame) telegram = parser(frame)
except Exception: except Exception:
self._previous.pop(source_id, None) self._previous.pop(source_id, None)
self._record_error(session_factory, source_id, "WarmteLink frame could not be parsed") self._record_error(
session_factory, source_id, "WarmteLink frame could not be parsed", now=received_at
)
return False return False
try: try:
with session_factory() as session: with session_factory() as session:
admitted = self.ingest(session, source_id=source_id, telegram=telegram) admitted = self.ingest(
session, source_id=source_id, telegram=telegram, received_at=received_at
)
session.commit() session.commit()
return admitted return admitted
except Exception: except Exception:
self._previous.pop(source_id, None) self._previous.pop(source_id, None)
self._record_error(session_factory, source_id, "WarmteLink ingest failed") self._record_error(session_factory, source_id, "WarmteLink ingest failed", now=received_at)
return False return False
def _admit( def _admit(
@@ -156,8 +172,8 @@ class WarmteLinkIngestor:
snapshot: _FrameSnapshot, snapshot: _FrameSnapshot,
quality: str, quality: str,
fingerprints: tuple[str, ...], fingerprints: tuple[str, ...],
received_at: datetime,
) -> None: ) -> None:
received_at = _utc(self._clock())
for sample, fingerprint in zip(snapshot.samples, fingerprints, strict=True): for sample, fingerprint in zip(snapshot.samples, fingerprints, strict=True):
channel = upsert_discovered_channel( channel = upsert_discovered_channel(
session, session,
@@ -200,27 +216,33 @@ class WarmteLinkIngestor:
source.last_error = None source.last_error = None
source.updated_at = received_at source.updated_at = received_at
def _diagnose(self, source: MeterSource, reason: str) -> None: def _diagnose(self, source: MeterSource, reason: str, *, now: datetime | None = None) -> None:
now = _utc(self._clock()) now = _utc(self._clock()) if now is None else now
source.status = "error" source.status = "error"
source.last_error = reason source.last_error = reason
source.updated_at = now source.updated_at = now
def _record_error( def _record_error(
self, session_factory: Callable[[], Session], source_id: int, message: str) -> None: self,
session_factory: Callable[[], Session],
source_id: int,
message: str,
*,
now: datetime,
) -> None:
try: try:
with session_factory() as session: with session_factory() as session:
source = session.get(MeterSource, source_id) source = session.get(MeterSource, source_id)
if source is not None: if source is not None:
self._diagnose(source, message) self._diagnose(source, message, now=now)
session.commit() session.commit()
except Exception: except Exception:
# Error reporting itself must not kill another source worker. # Error reporting itself must not kill another source worker.
return return
def _snapshot(telegram: P1Telegram) -> _FrameSnapshot: def _snapshot(telegram: P1Telegram, received_at: datetime) -> _FrameSnapshot:
recorded_at = _parse_timestamp(telegram.timestamp) recorded_at = _parse_timestamp(telegram.timestamp, received_at)
samples = tuple(_sample(channel) for channel in telegram.channels) samples = tuple(_sample(channel) for channel in telegram.channels)
if not samples: if not samples:
raise ValueError("WarmteLink telegram contains no cumulative channels") raise ValueError("WarmteLink telegram contains no cumulative channels")
@@ -340,13 +362,30 @@ def _is_canonical_fingerprint(value: str | None) -> bool:
return value is not None and _FINGERPRINT_PATTERN.fullmatch(value) is not None return value is not None and _FINGERPRINT_PATTERN.fullmatch(value) is not None
def _parse_timestamp(value: str | None) -> datetime: _AMSTERDAM = ZoneInfo("Europe/Amsterdam")
_MAX_CLOCK_SKEW = timedelta(minutes=5)
def _parse_timestamp(value: str | None, received_at: datetime) -> datetime:
if value is None or len(value) != 13 or value[-1] not in {"S", "W"} or not value[:-1].isdigit(): if value is None or len(value) != 13 or value[-1] not in {"S", "W"} or not value[:-1].isdigit():
raise ValueError("WarmteLink timestamp is unavailable") raise ValueError("WarmteLink timestamp is unavailable")
naive = datetime.strptime(value[:-1], "%y%m%d%H%M%S") naive = datetime.strptime(value[:-1], "%y%m%d%H%M%S")
# The telegram's S/W marker is evidence of the intended Amsterdam offset. # The S/W marker is only advisory: deployed devices have emitted W while
offset = 2 if value[-1] == "S" else 1 # on CEST. Validate both folds by a UTC round trip, which rejects spring
return (naive.replace(tzinfo=UTC) - timedelta(hours=offset)).replace(tzinfo=UTC) # gaps and leaves one (ordinary) or two (fall-back) real instants.
candidates: list[datetime] = []
for fold in (0, 1):
candidate = naive.replace(tzinfo=_AMSTERDAM, fold=fold).astimezone(UTC)
local = candidate.astimezone(_AMSTERDAM)
if local.replace(tzinfo=None) == naive and local.fold == fold and candidate not in candidates:
candidates.append(candidate)
if not candidates:
raise ValueError("WarmteLink timestamp is unavailable")
received_at = _utc(received_at)
recorded_at = min(candidates, key=lambda candidate: abs(candidate - received_at))
if abs(recorded_at - received_at) > _MAX_CLOCK_SKEW:
raise ValueError("WarmteLink timestamp is unavailable")
return recorded_at
def _suggestion(unit: str) -> str | None: def _suggestion(unit: str) -> str | None:
+311 -17
View File
@@ -3,7 +3,7 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import replace from dataclasses import replace
from datetime import UTC, datetime from datetime import UTC, datetime, timedelta
from decimal import Decimal from decimal import Decimal
from pathlib import Path from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
@@ -62,6 +62,7 @@ def _telegram(
fingerprint: str | None = "a" * 64, fingerprint: str | None = "a" * 64,
channel_fingerprint: str | None = None, channel_fingerprint: str | None = None,
channel_fingerprints: dict[int, str | None] | None = None, channel_fingerprints: dict[int, str | None] | None = None,
timestamp: str | None = None,
) -> P1Telegram: ) -> P1Telegram:
fields: list[ObisField] = [] fields: list[ObisField] = []
parsed_channels: list[P1Channel] = [] parsed_channels: list[P1Channel] = []
@@ -86,7 +87,7 @@ def _telegram(
100, 100,
integrity, integrity,
"fixture", "fixture",
f"26082212{minute:02d}{second:02d}S", timestamp or f"26082212{minute:02d}{second:02d}S",
fingerprint, fingerprint,
tuple(fields), tuple(fields),
tuple(parsed_channels), tuple(parsed_channels),
@@ -113,6 +114,260 @@ def _fixed_clock() -> datetime:
return datetime(2026, 8, 22, 10, tzinfo=UTC) return datetime(2026, 8, 22, 10, tzinfo=UTC)
def _clock_at(value: datetime):
return lambda: value
class _AdvancingClock:
def __init__(self, initial: datetime) -> None:
self._next = initial
self.calls = 0
def __call__(self) -> datetime:
value = self._next
self._next += timedelta(minutes=1)
self.calls += 1
return value
def _as_utc(value: datetime) -> datetime:
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
def test_summer_wall_clock_ignores_incorrect_w_marker(database):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 8, 23, 21, 40, tzinfo=UTC)
ingest = WarmteLinkIngestor(clock=_clock_at(received_at))
with factory() as session:
assert ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=0, integrity=IntegrityStatus.VALID, timestamp="260823234000W"),
)
session.commit()
with factory() as session:
reading = session.scalar(select(WarmteLinkReading).order_by(WarmteLinkReading.id))
assert reading is not None
assert _as_utc(reading.recorded_at) == datetime(2026, 8, 23, 21, 40, tzinfo=UTC)
def test_winter_wall_clock_uses_amsterdam_standard_time_even_with_s_marker(database):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 1, 23, 22, 40, tzinfo=UTC)
ingest = WarmteLinkIngestor(clock=_clock_at(received_at))
with factory() as session:
assert ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=0, integrity=IntegrityStatus.VALID, timestamp="260123234000S"),
)
session.commit()
with factory() as session:
reading = session.scalar(select(WarmteLinkReading).order_by(WarmteLinkReading.id))
assert reading is not None
assert _as_utc(reading.recorded_at) == datetime(2026, 1, 23, 22, 40, tzinfo=UTC)
@pytest.mark.parametrize(
("received_at", "expected"),
[
(datetime(2026, 10, 25, 0, 30, tzinfo=UTC), datetime(2026, 10, 25, 0, 30, tzinfo=UTC)),
(datetime(2026, 10, 25, 1, 30, tzinfo=UTC), datetime(2026, 10, 25, 1, 30, tzinfo=UTC)),
],
)
def test_fall_back_fold_is_selected_from_received_at(database, received_at, expected):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor(clock=_clock_at(received_at))
with factory() as session:
assert ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=0, integrity=IntegrityStatus.VALID, timestamp="261025023000W"),
)
session.commit()
with factory() as session:
reading = session.scalar(select(WarmteLinkReading).order_by(WarmteLinkReading.id))
assert reading is not None and _as_utc(reading.recorded_at) == expected
@pytest.mark.parametrize("timestamp", ["260329023000S", "260823234000X"])
def test_nonexistent_or_invalid_marker_timestamp_fails_closed_with_generic_diagnostic(database, timestamp):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor(clock=_clock_at(datetime(2026, 3, 29, 1, 30, tzinfo=UTC)))
with factory() as session:
assert not ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=0, integrity=IntegrityStatus.VALID, timestamp=timestamp),
)
session.commit()
source = session.get(MeterSource, source_id)
assert source is not None and source.last_error == "WarmteLink frame could not be normalized"
assert _counts(factory) == (0, 0)
def test_timestamp_beyond_clock_skew_fails_closed_with_generic_diagnostic(database):
_engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor(clock=_clock_at(datetime(2026, 8, 23, 20, tzinfo=UTC)))
with factory() as session:
assert not ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=0, integrity=IntegrityStatus.VALID, timestamp="260823234000W"),
)
session.commit()
source = session.get(MeterSource, source_id)
assert source is not None and source.last_error == "WarmteLink frame could not be normalized"
assert _counts(factory) == (0, 0)
@pytest.mark.parametrize("timestamp", ["260823234000X", "260823234600W"])
def test_post_clock_rejections_reuse_received_at_for_diagnostic(database, timestamp):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 8, 23, 21, 40, tzinfo=UTC)
clock = _AdvancingClock(received_at)
ingest = WarmteLinkIngestor(clock=clock)
with factory() as session:
assert not ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=0, integrity=IntegrityStatus.VALID, timestamp=timestamp),
)
session.commit()
source = session.get(MeterSource, source_id)
assert source is not None
assert _as_utc(source.updated_at) == received_at
assert clock.calls == 1
def test_first_unverifiable_candidate_reuses_received_at_for_diagnostic(database):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 8, 22, 10, tzinfo=UTC)
clock = _AdvancingClock(received_at)
ingest = WarmteLinkIngestor(clock=clock)
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
session.commit()
source = session.get(MeterSource, source_id)
assert source is not None
assert _as_utc(source.updated_at) == received_at
assert clock.calls == 1
def test_accepted_frame_uses_received_at_for_all_source_timestamps(database):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 8, 22, 10, tzinfo=UTC)
clock = _AdvancingClock(received_at)
ingest = WarmteLinkIngestor(clock=clock)
with factory() as session:
assert ingest.ingest(
session, source_id=source_id, telegram=_telegram(second=0, integrity=IntegrityStatus.VALID)
)
session.commit()
with factory() as session:
source = session.get(MeterSource, source_id)
reading = session.scalar(select(WarmteLinkReading).order_by(WarmteLinkReading.id))
assert source is not None and reading is not None
assert _as_utc(reading.received_at) == received_at
assert _as_utc(source.last_seen_at) == received_at
assert _as_utc(source.updated_at) == received_at
assert clock.calls == 1
@pytest.mark.parametrize(
"path",
["parser", "integrity", "normalization", "fingerprint", "candidate", "continuity", "accepted"],
)
def test_handle_frame_samples_the_receive_clock_once_for_each_admission_path(database, path):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 8, 22, 10, tzinfo=UTC)
clock = _AdvancingClock(received_at)
ingest = WarmteLinkIngestor(clock=clock)
if path == "parser":
assert not ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: (_ for _ in ()).throw(ValueError()),
)
expected_at = received_at
elif path == "integrity":
assert not ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: _telegram(second=0, integrity=IntegrityStatus.INVALID),
)
expected_at = received_at
elif path == "normalization":
assert not ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: _telegram(second=0, timestamp="invalid-time"),
)
expected_at = received_at
elif path == "fingerprint":
assert not ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: _telegram(second=0, integrity=IntegrityStatus.VALID, fingerprint="invalid"),
)
expected_at = received_at
elif path == "candidate":
assert not ingest.handle_frame(
source_id, b"private", session_factory=factory, parser=lambda _raw: _telegram(second=0)
)
expected_at = received_at
elif path == "continuity":
assert not ingest.handle_frame(
source_id, b"private", session_factory=factory, parser=lambda _raw: _telegram(second=0)
)
assert not ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: _telegram(second=20, water="5.901"),
)
expected_at = received_at + timedelta(minutes=1)
else:
assert ingest.handle_frame(
source_id,
b"private",
session_factory=factory,
parser=lambda _raw: _telegram(second=0, integrity=IntegrityStatus.VALID),
)
expected_at = received_at
with factory() as session:
source = session.get(MeterSource, source_id)
assert source is not None
assert _as_utc(source.updated_at) == expected_at
assert clock.calls == (2 if path == "continuity" else 1)
def test_unverifiable_summer_w_frames_keep_utc_continuity(database):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 8, 23, 21, 40, 10, tzinfo=UTC)
ingest = WarmteLinkIngestor(clock=_clock_at(received_at))
with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0, timestamp="260823234000W"))
assert ingest.ingest(
session,
source_id=source_id,
telegram=_telegram(second=10, water="5.901", timestamp="260823234010W"),
)
session.commit()
with factory() as session:
readings = list(session.scalars(select(WarmteLinkReading).order_by(WarmteLinkReading.id)))
assert [_as_utc(reading.recorded_at) for reading in readings] == [
datetime(2026, 8, 23, 21, 40, 10, tzinfo=UTC),
datetime(2026, 8, 23, 21, 40, 10, tzinfo=UTC),
]
def test_unverifiable_requires_two_continuous_frames_and_preserves_true_metadata(database): def test_unverifiable_requires_two_continuous_frames_and_preserves_true_metadata(database):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor(clock=_fixed_clock) ingest = WarmteLinkIngestor(clock=_fixed_clock)
@@ -143,14 +398,14 @@ def test_unverifiable_requires_two_continuous_frames_and_preserves_true_metadata
def test_unverifiable_cadence_requires_exactly_ten_seconds_and_restarts_after_gaps(database): def test_unverifiable_cadence_requires_exactly_ten_seconds_and_restarts_after_gaps(database):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, water="5.901")) assert ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, water="5.901"))
session.commit() session.commit()
assert _counts(factory) == (2, 2) assert _counts(factory) == (2, 2)
restarted = WarmteLinkIngestor() restarted = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not restarted.ingest(session, source_id=source_id, telegram=_telegram(second=0, water="5.902")) assert not restarted.ingest(session, source_id=source_id, telegram=_telegram(second=0, water="5.902"))
# 20 seconds misses one expected frame; it is only a new candidate. # 20 seconds misses one expected frame; it is only a new candidate.
@@ -176,7 +431,7 @@ def test_unverifiable_cadence_requires_exactly_ten_seconds_and_restarts_after_ga
) )
def test_unverifiable_discontinuities_restart_two_frame_confirmation(database, first, second): def test_unverifiable_discontinuities_restart_two_frame_confirmation(database, first, second):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=first) assert not ingest.ingest(session, source_id=source_id, telegram=first)
assert not ingest.ingest(session, source_id=source_id, telegram=second) assert not ingest.ingest(session, source_id=source_id, telegram=second)
@@ -189,7 +444,7 @@ def test_unverifiable_discontinuities_restart_two_frame_confirmation(database, f
def test_discontinuity_candidate_needs_a_new_matching_successor(database): def test_discontinuity_candidate_needs_a_new_matching_successor(database):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, channels=(1,))) assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=10, channels=(1,)))
@@ -213,7 +468,7 @@ def test_each_metadata_or_channel_discontinuity_requires_its_own_two_frames(
database, first, discontinuous, successor database, first, discontinuous, successor
): ):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=first) assert not ingest.ingest(session, source_id=source_id, telegram=first)
# The discontinuous frame becomes a new first candidate, never an admission. # The discontinuous frame becomes a new first candidate, never an admission.
@@ -224,7 +479,7 @@ def test_each_metadata_or_channel_discontinuity_requires_its_own_two_frames(
def test_parser_failure_clears_only_its_source_candidate_and_requires_two_new_frames(database): def test_parser_failure_clears_only_its_source_candidate_and_requires_two_new_frames(database):
_engine, factory, (one, two) = database _engine, factory, (one, two) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
@@ -296,7 +551,7 @@ def test_dto_normalization_failure_clears_only_its_source_candidate_and_requires
database, kind database, kind
): ):
_engine, factory, (one, two) = database _engine, factory, (one, two) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
@@ -366,7 +621,9 @@ def test_history_keeps_first_utc_minute_sample_and_creates_next_minute(database)
def test_handle_frame_rolls_back_database_error_marks_source_and_recovers(database, monkeypatch): def test_handle_frame_rolls_back_database_error_marks_source_and_recovers(database, monkeypatch):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor() received_at = datetime(2026, 8, 22, 10, tzinfo=UTC)
clock = _AdvancingClock(received_at)
ingest = WarmteLinkIngestor(clock=clock)
telegram = _telegram(second=0, integrity=IntegrityStatus.VALID) telegram = _telegram(second=0, integrity=IntegrityStatus.VALID)
import app.services.warmtelink_ingest as module import app.services.warmtelink_ingest as module
@@ -382,14 +639,51 @@ def test_handle_frame_rolls_back_database_error_marks_source_and_recovers(databa
source = session.get(MeterSource, source_id) source = session.get(MeterSource, source_id)
assert source is not None and source.status == "error" assert source is not None and source.status == "error"
assert "private" not in (source.last_error or "") assert "private" not in (source.last_error or "")
assert _as_utc(source.updated_at) == received_at
assert clock.calls == 1
monkeypatch.setattr(module, "upsert_discovered_channel", original) monkeypatch.setattr(module, "upsert_discovered_channel", original)
assert ingest.handle_frame(source_id, b"ignored", session_factory=factory, parser=lambda _raw: telegram) assert ingest.handle_frame(source_id, b"ignored", session_factory=factory, parser=lambda _raw: telegram)
assert _counts(factory) == (2, 2) assert _counts(factory) == (2, 2)
def test_handle_frame_commit_failure_reuses_its_receive_clock_for_error_diagnostic(database, monkeypatch):
_engine, factory, (source_id, _) = database
received_at = datetime(2026, 8, 22, 10, tzinfo=UTC)
clock = _AdvancingClock(received_at)
ingest = WarmteLinkIngestor(clock=clock)
original_commit = Session.commit
sessions_created = 0
def session_factory():
nonlocal sessions_created
session = factory()
session.info["fail_commit"] = sessions_created == 0
sessions_created += 1
return session
def fail_initial_commit(self):
if self.info.get("fail_commit"):
raise RuntimeError("database commit failed")
return original_commit(self)
monkeypatch.setattr(Session, "commit", fail_initial_commit)
assert not ingest.handle_frame(
source_id,
b"private",
session_factory=session_factory,
parser=lambda _raw: _telegram(second=0, integrity=IntegrityStatus.VALID),
)
with factory() as session:
source = session.get(MeterSource, source_id)
assert source is not None
assert source.status == "error"
assert _as_utc(source.updated_at) == received_at
assert clock.calls == 1
def test_second_channel_failure_rolls_back_first_channel_latest_and_history(database, monkeypatch): def test_second_channel_failure_rolls_back_first_channel_latest_and_history(database, monkeypatch):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
import app.services.warmtelink_ingest as module import app.services.warmtelink_ingest as module
original = module.upsert_discovered_channel original = module.upsert_discovered_channel
@@ -419,7 +713,7 @@ def test_second_channel_failure_rolls_back_first_channel_latest_and_history(data
def test_unverifiable_database_failure_clears_only_its_source_sliding_candidate(database, monkeypatch): def test_unverifiable_database_failure_clears_only_its_source_sliding_candidate(database, monkeypatch):
_engine, factory, (one, two) = database _engine, factory, (one, two) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
import app.services.warmtelink_ingest as module import app.services.warmtelink_ingest as module
with factory() as session: with factory() as session:
@@ -462,7 +756,7 @@ def _valid_crc_frame(*, device_type: str = "006") -> bytes:
def test_non_enum_integrity_and_fake_metadata_reject_whole_frame_per_source(database): def test_non_enum_integrity_and_fake_metadata_reject_whole_frame_per_source(database):
_engine, factory, (one, two) = database _engine, factory, (one, two) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
@@ -487,7 +781,7 @@ def test_non_enum_integrity_and_fake_metadata_reject_whole_frame_per_source(data
def test_valid_crc_parser_frame_with_raw_device_type_is_rejected_without_leakage(database): def test_valid_crc_parser_frame_with_raw_device_type_is_rejected_without_leakage(database):
_engine, factory, (one, two) = database _engine, factory, (one, two) = database
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=one, telegram=_telegram(second=0))
assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=two, telegram=_telegram(second=0))
@@ -523,7 +817,7 @@ def test_invalid_fingerprint_fails_closed_without_persisting_identifiers(
): ):
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
raw_identifier = "serial-number-which-must-not-persist" raw_identifier = "serial-number-which-must-not-persist"
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0, integrity=IntegrityStatus.INVALID)) assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0, integrity=IntegrityStatus.INVALID))
assert not ingest.ingest( assert not ingest.ingest(
@@ -561,7 +855,7 @@ def test_invalid_channel_fingerprint_rejects_whole_frame_and_clears_candidate(
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
raw_identifier = "serial-number-which-must-not-persist" raw_identifier = "serial-number-which-must-not-persist"
valid = "a" * 64 valid = "a" * 64
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
# Establish a candidate, then prove an invalid channel clears only it. # Establish a candidate, then prove an invalid channel clears only it.
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
@@ -596,7 +890,7 @@ def test_invalid_top_fingerprint_rejects_whole_frame_and_clears_candidate(databa
_engine, factory, (source_id, _) = database _engine, factory, (source_id, _) = database
raw_identifier = "serial-number-which-must-not-persist" raw_identifier = "serial-number-which-must-not-persist"
valid = "a" * 64 valid = "a" * 64
ingest = WarmteLinkIngestor() ingest = WarmteLinkIngestor(clock=_fixed_clock)
with factory() as session: with factory() as session:
assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0)) assert not ingest.ingest(session, source_id=source_id, telegram=_telegram(second=0))
assert not ingest.ingest( assert not ingest.ingest(