xavierk/Fenris
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases 11 Wiki Activity

fix: rebuild trustworthy local-day evidence (#99)

Browse Source
This commit is contained in:
xavierk
2026-09-28 13:09:00 +05:30
parent 7e392c4ea2
commit 017a562566
15 changed files with 1305 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fenris/local_day.py
+563 -13
View File
@@ -1,9 +1,10 @@
"""Local-day activity derivation from measured sample intervals.
Computes durable local-day read/write summaries using recorded local
midnight boundaries. UTC hour/day aggregates remain separate inputs to
endurance projections. The collector owns this derivation, preserving the
read-only TUI boundary (ADR 0010).
midnight boundaries. Collection derives new measured intervals; migration
and repair rebuild legacy summaries from surviving evidence. UTC hour/day
aggregates remain separate inputs to endurance projections, and readers stay
read-only (ADR 0010).
Key contracts:
- A compatible interval wholly inside a local day contributes its volume once
@@ -28,6 +29,8 @@ Retention policy (issue #93):
import sqlite3
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta, timezone
from itertools import pairwise
from typing import Any
from zoneinfo import ZoneInfo
@@ -50,6 +53,55 @@ class LocalDaySummary:
activity_precision: str = "measured"
@dataclass(frozen=True)
class CoarseLocalDayActivity:
"""Rebuilt byte evidence and its coverage limits for one local day."""
bytes_written: int
bytes_read: int
activity_seconds: int
activity_intervals: int
incomplete: bool
segment_totals: dict[int, "LocalDaySegmentTotals"]
@dataclass
class LocalDaySegmentTotals:
"""Activity totals attributed to one controller segment."""
bytes_written: int = 0
bytes_read: int = 0
activity_seconds: int = 0
activity_intervals: int = 0
def add(
self,
*,
bytes_written: int,
bytes_read: int,
activity_seconds: int,
activity_intervals: int = 1,
) -> None:
"""Accumulate one measured interval or coarse hour."""
self.bytes_written += bytes_written
self.bytes_read += bytes_read
self.activity_seconds += activity_seconds
self.activity_intervals += activity_intervals
@dataclass(frozen=True)
class ReconstructedLocalDayInterval:
"""One counter delta with source and timezone provenance."""
start: datetime
end: datetime
bytes_written: int
bytes_read: int
activity_seconds: int
segment_id: int
start_sample_id: int
end_sample_id: int
start_tz: str | None
end_tz: str | None
@dataclass(frozen=True)
class LocalDayHistoryEntry:
"""A local-day summary with evidence-limit metadata for history readout.
@@ -119,6 +171,8 @@ def local_day_boundaries(local_date: str, tz_name: str) -> tuple[datetime, datet
start_utc = start_local.astimezone(timezone.utc)
end_utc = end_local.astimezone(timezone.utc)
offset = start_local.utcoffset()
if offset is None:
raise ValueError("Local midnight has no UTC offset")
offset_seconds = int(offset.total_seconds())
sign = "+" if offset_seconds >= 0 else "-"
offset_seconds = abs(offset_seconds)
@@ -277,6 +331,472 @@ def _same_monitoring_period(
return False
def repair_legacy_local_day_evidence(conn: sqlite3.Connection) -> int:
"""Rebuild untrusted local-day totals from surviving evidence.
Sample intervals take precedence over UTC-hour summaries. An interval is
usable only when its counters, controller segment, monitoring period, and
recorded local-day boundaries all agree. Old byte totals remain stored but
hidden when no such evidence survives.
"""
tables = {
row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
)
}
if "local_days" not in tables:
return 0
recorded_days = conn.execute(
"SELECT id, local_date, tz_name, utc_start, utc_end, "
" activity_incomplete, activity_precision "
"FROM local_days ORDER BY utc_start, id"
).fetchall()
local_days = [
row[:6] for row in recorded_days
if row[6] in ("legacy", "unavailable")
]
if not local_days:
return 0
required_sample_columns = {"id", "ts", "bytes_written", "bytes_read", "segment_id"}
periods = []
segment_boundaries = []
segment_windows = {}
if {"monitoring_periods", "controller_segments"}.issubset(tables):
periods = [
(_utc_datetime(start), _utc_datetime(end) if end else None)
for start, end in conn.execute(
"SELECT started_at, ended_at FROM monitoring_periods"
)
]
segment_boundaries = [
(segment_id, _utc_datetime(opened_at))
for segment_id, opened_at in conn.execute(
"SELECT id, opened_at FROM controller_segments ORDER BY opened_at, id"
)
]
segment_windows = {
segment_id: (
opened_at,
segment_boundaries[index + 1][1]
if index + 1 < len(segment_boundaries) else None,
)
for index, (segment_id, opened_at) in enumerate(segment_boundaries)
}
sample_columns = (
{row[1] for row in conn.execute("PRAGMA table_info(samples)")}
if "samples" in tables else set()
)
intervals: list[ReconstructedLocalDayInterval] = []
samples = []
if required_sample_columns.issubset(sample_columns):
sample_select = (
"SELECT id, ts, bytes_written, bytes_read, segment_id, "
+ ("local_tz " if "local_tz" in sample_columns else "NULL AS local_tz ")
+ "FROM samples ORDER BY ts, id"
)
samples = [
{
"id": row[0], "ts": _utc_datetime(row[1]),
"bytes_written": row[2], "bytes_read": row[3],
"segment_id": row[4], "local_tz": row[5],
}
for row in conn.execute(sample_select)
]
for previous, current in pairwise(samples):
start = previous["ts"]
end = current["ts"]
segment_id = current["segment_id"]
segment_window = segment_windows.get(segment_id)
if (
end <= start
or segment_id is None
or previous["segment_id"] != segment_id
or segment_window is None
or segment_window[0] > start
or (segment_window[1] is not None and end >= segment_window[1])
or bool(previous["local_tz"]) != bool(current["local_tz"])
or previous["bytes_written"] is None
or current["bytes_written"] is None
or previous["bytes_read"] is None
or current["bytes_read"] is None
):
continue
bytes_written = current["bytes_written"] - previous["bytes_written"]
bytes_read = current["bytes_read"] - previous["bytes_read"]
if bytes_written < 0 or bytes_read < 0:
continue
if not any(
period_start <= start and (period_end is None or end <= period_end)
for period_start, period_end in periods
):
continue
intervals.append(ReconstructedLocalDayInterval(
start=start,
end=end,
bytes_written=bytes_written,
bytes_read=bytes_read,
activity_seconds=int((end - start).total_seconds()),
segment_id=segment_id,
start_sample_id=previous["id"],
end_sample_id=current["id"],
start_tz=previous["local_tz"],
end_tz=current["local_tz"],
))
_record_reconstructed_shared_intervals(conn, intervals, recorded_days)
day_bounds = [
{
"id": row[0],
"local_date": row[1],
"tz_name": row[2],
"start": _utc_datetime(row[3]),
"end": _utc_datetime(row[4]),
"precision": row[6],
}
for row in recorded_days
]
rebuilding_ids = {row[0] for row in local_days}
matching_by_day: dict[int, list[ReconstructedLocalDayInterval]] = {
local_day_id: [] for local_day_id in rebuilding_ids
}
for interval in intervals:
candidates = [
day for day in day_bounds
if day["start"] <= interval.start
and interval.end <= day["end"]
and all(
zone is None or zone == day["tz_name"]
for zone in (interval.start_tz, interval.end_tz)
)
]
if len(candidates) == 1 and candidates[0]["id"] in rebuilding_ids:
matching_by_day[candidates[0]["id"]].append(interval)
rebuilt = 0
for row in local_days:
(
local_day_id, local_date, tz_name, utc_start, utc_end,
prior_incomplete,
) = row
start = _utc_datetime(utc_start)
end = _utc_datetime(utc_end)
matching = matching_by_day[local_day_id]
if not matching:
has_samples = any(
start <= sample["ts"] < end for sample in samples
)
coarse = (
None if has_samples
else _coarse_local_day_activity(
conn, local_day_id, start, end, periods, segment_boundaries,
)
)
if coarse is not None:
incomplete = bool(prior_incomplete) or coarse.incomplete
conn.execute(
"UPDATE local_days SET bytes_written = ?, bytes_read = ?, "
"activity_seconds = ?, activity_intervals = ?, "
"activity_incomplete = ?, activity_precision = 'coarse', "
"last_sample_id = NULL WHERE id = ?",
(coarse.bytes_written, coarse.bytes_read,
coarse.activity_seconds, coarse.activity_intervals,
incomplete, local_day_id),
)
_replace_local_day_segment_totals(
conn, local_day_id, coarse.segment_totals,
)
rebuilt += 1
continue
conn.execute(
"UPDATE local_days SET activity_precision = 'unavailable', "
"activity_incomplete = CASE WHEN EXISTS ("
" SELECT 1 FROM local_day_unallocated_evidence "
" WHERE (start_local_date = ? AND start_tz_name = ?) "
" OR (end_local_date = ? AND end_tz_name = ?)"
") THEN 1 ELSE 0 END WHERE id = ?",
(local_date, tz_name, local_date, tz_name, local_day_id),
)
continue
totals: dict[int, LocalDaySegmentTotals] = {}
bytes_written = 0
bytes_read = 0
activity_seconds = 0
for interval in matching:
segment_total = totals.setdefault(
interval.segment_id, LocalDaySegmentTotals(),
)
segment_total.add(
bytes_written=interval.bytes_written,
bytes_read=interval.bytes_read,
activity_seconds=interval.activity_seconds,
)
bytes_written += interval.bytes_written
bytes_read += interval.bytes_read
activity_seconds += interval.activity_seconds
day_seconds = int((end - start).total_seconds())
incomplete = bool(prior_incomplete) or activity_seconds < day_seconds
last_sample_id = max(interval.end_sample_id for interval in matching)
_replace_local_day_segment_totals(conn, local_day_id, totals)
conn.execute(
"UPDATE local_days SET bytes_written = ?, bytes_read = ?, "
"activity_seconds = ?, activity_intervals = ?, "
"activity_incomplete = ?, activity_precision = 'measured', "
"last_sample_id = ? WHERE id = ?",
(bytes_written, bytes_read, activity_seconds, len(matching),
incomplete, last_sample_id, local_day_id),
)
rebuilt += 1
return rebuilt
def _replace_local_day_segment_totals(
conn: sqlite3.Connection,
local_day_id: int,
totals: dict[int, LocalDaySegmentTotals],
) -> None:
"""Replace segment provenance rows for one rebuilt local day."""
conn.execute(
"DELETE FROM local_day_segment_totals WHERE local_day_id = ?",
(local_day_id,),
)
conn.executemany(
"INSERT INTO local_day_segment_totals "
"(local_day_id, segment_id, bytes_written, bytes_read, "
" activity_seconds, activity_intervals) "
"VALUES (?, ?, ?, ?, ?, ?)",
(
(
local_day_id,
segment_id,
values.bytes_written,
values.bytes_read,
values.activity_seconds,
values.activity_intervals,
)
for segment_id, values in totals.items()
),
)
def _record_reconstructed_shared_intervals(
conn: sqlite3.Connection,
intervals: list[ReconstructedLocalDayInterval],
recorded_days: list[tuple],
) -> None:
"""Retain compatible counter deltas that cross stored local boundaries."""
day_rows = [
{
"id": row[0], "local_date": row[1], "tz_name": row[2],
"utc_start": _utc_datetime(row[3]),
"utc_end": _utc_datetime(row[4]),
"precision": row[6],
}
for row in recorded_days
]
def find_day(instant: datetime, zone: str | None, *, end: bool):
candidates = [
row for row in day_rows
if (row["utc_start"] < instant <= row["utc_end"] if end
else row["utc_start"] <= instant < row["utc_end"])
and (zone is None or row["tz_name"] == zone)
]
return candidates[0] if len(candidates) == 1 else None
for interval in intervals:
start_day = find_day(interval.start, interval.start_tz, end=False)
end_day = find_day(interval.end, interval.end_tz, end=True)
if start_day is None or end_day is None or start_day["id"] == end_day["id"]:
continue
if start_day["precision"] not in ("legacy", "unavailable") and (
end_day["precision"] not in ("legacy", "unavailable")
):
continue
if start_day["tz_name"] == end_day["tz_name"]:
zone_days = sorted(
(row for row in day_rows if row["tz_name"] == start_day["tz_name"]),
key=lambda row: row["utc_start"],
)
start_index = next(
(i for i, row in enumerate(zone_days) if row["id"] == start_day["id"]),
None,
)
end_index = next(
(i for i, row in enumerate(zone_days) if row["id"] == end_day["id"]),
None,
)
if start_index is None or end_index is None or end_index <= start_index:
continue
chain = zone_days[start_index:end_index + 1]
if any(
left["utc_end"] != right["utc_start"]
for left, right in pairwise(chain)
):
continue
reason = "local_midnight"
else:
reason = "timezone_change"
conn.execute(
"INSERT OR IGNORE INTO local_day_unallocated_evidence "
"(start_sample_id, end_sample_id, start_local_date, end_local_date, "
" start_tz_name, end_tz_name, started_at, ended_at, bytes_written, "
" bytes_read, reason, segment_id) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
interval.start_sample_id, interval.end_sample_id,
start_day["local_date"], end_day["local_date"],
start_day["tz_name"], end_day["tz_name"],
interval.start.isoformat(), interval.end.isoformat(),
interval.bytes_written, interval.bytes_read, reason,
interval.segment_id,
),
)
conn.execute(
"UPDATE local_days SET activity_incomplete = 1 "
"WHERE id IN (?, ?)",
(start_day["id"], end_day["id"]),
)
def _coarse_local_day_activity(
conn: sqlite3.Connection,
local_day_id: int,
day_start: datetime,
day_end: datetime,
periods: list[tuple[datetime, datetime | None]],
segments: list[tuple[int, datetime]],
) -> CoarseLocalDayActivity | None:
"""Return UTC hours unique to these recorded local-day bounds."""
if not periods or not segments or conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='hour_observations'"
).fetchone() is None:
return None
hour = timedelta(hours=1)
full_hours: dict[datetime, tuple] = {}
boundary_hour_found = False
other_day_bounds = [
(_utc_datetime(row[0]), _utc_datetime(row[1]))
for row in conn.execute(
"SELECT utc_start, utc_end FROM local_days WHERE id != ?",
(local_day_id,),
)
]
first_hour = day_start.replace(minute=0, second=0, microsecond=0)
for row in conn.execute(
"SELECT hour, bytes_written_delta, bytes_read_delta, sample_count, "
" active_seconds, idle_seconds, powered_off_seconds "
"FROM hour_observations WHERE hour >= ? AND hour < ? ORDER BY hour",
(first_hour.isoformat(), day_end.isoformat()),
):
hour_start = _utc_datetime(row[0])
hour_end = hour_start + hour
if hour_start >= day_end or hour_end <= day_start:
continue
if day_start <= hour_start and hour_end <= day_end:
if any(
other_start < hour_end and hour_start < other_end
for other_start, other_end in other_day_bounds
):
# Another historical local-day row claims this whole or
# partial UTC hour too. Keep the byte delta unattributed.
boundary_hour_found = True
continue
full_hours[hour_start] = row
else:
boundary_hour_found = True
expected_hours = []
expected_start = day_start.replace(minute=0, second=0, microsecond=0)
if expected_start < day_start:
expected_start += hour
while expected_start + hour <= day_end:
expected_hours.append(expected_start)
expected_start += hour
def hour_context(hour_start: datetime) -> int | None:
hour_end = hour_start + hour
period_matches = any(
period_start <= hour_start
and (period_end is None or hour_end <= period_end)
for period_start, period_end in periods
)
segment_matches = []
for index, (segment_id, opened_at) in enumerate(segments):
next_segment = segments[index + 1][1] if index + 1 < len(segments) else None
if opened_at <= hour_start and (next_segment is None or hour_end <= next_segment):
segment_matches.append(segment_id)
if not period_matches or len(segment_matches) != 1:
return None
return segment_matches[0]
totals: dict[int, LocalDaySegmentTotals] = {}
bytes_written = 0
bytes_read = 0
activity_seconds = 0
activity_intervals = 0
context_complete = True
for hour_start, row in full_hours.items():
segment_id = hour_context(hour_start)
if segment_id is None:
context_complete = False
continue
hour_seconds = (row[4] or 0) + (row[5] or 0) + (row[6] or 0)
activity_seconds += hour_seconds
if (row[3] or 0) <= 0 and (row[1] or 0) == 0 and (row[2] or 0) == 0:
continue
segment_total = totals.setdefault(segment_id, LocalDaySegmentTotals())
segment_total.add(
bytes_written=row[1] or 0,
bytes_read=row[2] or 0,
activity_seconds=hour_seconds,
)
bytes_written += row[1] or 0
bytes_read += row[2] or 0
activity_intervals += 1
if not activity_intervals:
return None
all_hours_present = all(
hour_start in full_hours for hour_start in expected_hours
)
has_unallocated_bytes = False
if conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='day_aggregates'"
).fetchone() is not None:
last_utc_day = (day_end - timedelta(microseconds=1)).date()
has_unallocated_bytes = conn.execute(
"SELECT 1 FROM day_aggregates WHERE day >= ? AND day <= ? "
"AND (COALESCE(unattributed_bytes_written, 0) > 0 "
" OR COALESCE(unattributed_bytes_read, 0) > 0) LIMIT 1",
(day_start.date().isoformat(), last_utc_day.isoformat()),
).fetchone() is not None
complete = (
all_hours_present
and activity_seconds >= int((day_end - day_start).total_seconds())
and not boundary_hour_found
and not has_unallocated_bytes
and context_complete
)
return CoarseLocalDayActivity(
bytes_written=bytes_written,
bytes_read=bytes_read,
activity_seconds=activity_seconds,
activity_intervals=activity_intervals,
incomplete=not complete,
segment_totals=totals,
)
def _upsert_activity_day(
conn: sqlite3.Connection,
local_date: str,
@@ -298,17 +818,20 @@ def _upsert_activity_day(
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'measured', ?) "
"ON CONFLICT(local_date, tz_name) DO UPDATE SET "
"bytes_written = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" AND excluded.activity_intervals > 0 "
" THEN excluded.bytes_written ELSE local_days.bytes_written + excluded.bytes_written END, "
"bytes_read = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" AND excluded.activity_intervals > 0 "
" THEN excluded.bytes_read ELSE local_days.bytes_read + excluded.bytes_read END, "
"activity_seconds = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" AND excluded.activity_intervals > 0 "
" THEN excluded.activity_seconds ELSE local_days.activity_seconds + excluded.activity_seconds END, "
"activity_intervals = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" AND excluded.activity_intervals > 0 "
" THEN excluded.activity_intervals ELSE local_days.activity_intervals + excluded.activity_intervals END, "
"activity_incomplete = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" THEN excluded.activity_incomplete "
" ELSE MAX(local_days.activity_incomplete, excluded.activity_incomplete) END, "
"activity_precision = 'measured', "
"activity_incomplete = MAX(local_days.activity_incomplete, excluded.activity_incomplete), "
"activity_precision = CASE WHEN excluded.activity_intervals > 0 "
" THEN 'measured' ELSE local_days.activity_precision END, "
"last_sample_id = COALESCE(excluded.last_sample_id, local_days.last_sample_id)",
(
local_date, tz_name, offset, start.isoformat(), end.isoformat(),
@@ -320,8 +843,8 @@ def _upsert_activity_day(
def record_local_activity_interval(
conn: sqlite3.Connection,
previous: dict,
current: dict,
previous: dict[str, Any],
current: dict[str, Any],
*,
start_sample_id: int,
end_sample_id: int,
@@ -354,7 +877,12 @@ def record_local_activity_interval(
prior_written = previous.get("bytes_written")
read = current.get("bytes_read")
prior_read = previous.get("bytes_read")
if None in (written, prior_written, read, prior_read):
if (
written is None
or prior_written is None
or read is None
or prior_read is None
):
return None
bytes_written = written - prior_written
bytes_read = read - prior_read
@@ -379,7 +907,7 @@ def record_local_activity_interval(
reason = "local_midnight"
else:
already_accounted = conn.execute(
"SELECT last_sample_id FROM local_days "
"SELECT last_sample_id, activity_precision FROM local_days "
"WHERE local_date = ? AND tz_name = ?",
(end_date, current_tz),
).fetchone()
@@ -389,6 +917,26 @@ def record_local_activity_interval(
and already_accounted[0] >= end_sample_id
):
return "known"
replacing_coarse = (
already_accounted is not None
and already_accounted[1] == "coarse"
)
if replacing_coarse:
local_day_id = conn.execute(
"SELECT id FROM local_days WHERE local_date = ? AND tz_name = ?",
(end_date, current_tz),
).fetchone()[0]
conn.execute(
"DELETE FROM local_day_segment_totals WHERE local_day_id = ?",
(local_day_id,),
)
conn.execute(
"UPDATE local_days SET bytes_written = 0, bytes_read = 0, "
"activity_seconds = 0, activity_intervals = 0, "
"activity_precision = 'unavailable', activity_incomplete = 1, "
"last_sample_id = NULL WHERE id = ?",
(local_day_id,),
)
seconds = int((end - start).total_seconds())
_upsert_activity_day(
conn,
@@ -398,6 +946,7 @@ def record_local_activity_interval(
bytes_read=bytes_read,
seconds=seconds,
interval_count=1,
incomplete=replacing_coarse,
last_sample_id=end_sample_id,
)
local_day_id = conn.execute(
@@ -567,7 +1116,8 @@ def query_local_day_summary(
interval_count = activity_intervals or 0
has_unallocated = evidence_count > 0
has_known = activity_precision == "measured" and interval_count > 0
trusted_precision = activity_precision in ("measured", "coarse")
has_known = trusted_precision and interval_count > 0
if not has_known and not has_unallocated:
activity_state = "incomplete" if bool(activity_incomplete) else "unavailable"
else:
@@ -595,7 +1145,7 @@ def query_local_day_summary(
else:
activity_state = "complete"
exact = activity_precision == "measured" and has_known
exact = trusted_precision and has_known
return {
"local_date": recorded_date,
"tz_name": recorded_tz,