xavierk/Fenris
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases 11 Wiki Activity

fix: conserve local-day activity evidence

Browse Source
This commit is contained in:
xavierk
2026-09-28 10:21:55 +05:30
parent 8a5ef05188
commit 7e392c4ea2
11 changed files with 1162 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fenris/collector.py
+26 -4
View File
@@ -239,8 +239,8 @@ def write_sample(
percentage_used, available_spare, media_errors, power_on_hours,
power_cycles, unsafe_shutdowns, temperature_c,
data_units_written, data_units_read, bytes_written, bytes_read,
critical_warning, segment_id
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
critical_warning, segment_id, local_tz
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
sample["ts"],
@@ -263,6 +263,7 @@ def write_sample(
sample["bytes_read"],
sample["critical_warning"],
segment_id,
sample.get("local_tz"),
),
)
@@ -272,6 +273,7 @@ def write_sample(
"identity_key": identity_key,
"identity_degraded": identity_degraded,
"segment_id": segment_id,
"sample_id": cursor.lastrowid,
}
@@ -291,12 +293,12 @@ def _publish_observation(
) -> None:
"""Publish one staged sample and all dependent evidence in caller transaction."""
observed_at = _observation_time(sample)
sample["local_tz"] = tz_name
validate_sample_invariants(sample, conn)
ensure_period_open(conn, observed_at)
seg_info = write_sample(sample, identity, conn, observed_at)
cursor = conn.execute("SELECT id FROM samples ORDER BY id DESC LIMIT 1")
current_id = cursor.fetchone()[0]
current_id = seg_info["sample_id"]
prev = find_previous_sample(conn, seg_info.get("segment_id"), current_id)
if prev is not None:
current = {
@@ -308,8 +310,28 @@ def _publish_observation(
"temperature_c": sample["temperature_c"],
"data_units_written": sample["data_units_written"],
"data_units_read": sample["data_units_read"],
"local_tz": tz_name,
}
derive_hours_from_interval(conn, prev, current)
from .local_day import record_local_activity_interval
record_local_activity_interval(
conn,
prev,
current,
start_sample_id=prev["id"],
end_sample_id=current_id,
segment_id=seg_info.get("segment_id"),
)
else:
previous_any_segment = find_previous_sample(conn, None, current_id)
if previous_any_segment is not None:
from .local_day import mark_local_activity_gap
mark_local_activity_gap(
conn,
sample["ts"],
tz_name,
previous_any_segment,
)
from .day_aggregate import derive_all_days, persist_day_aggregate
for aggregate in derive_all_days(conn):
src/fenris/derive.py
+3 -3
View File
@@ -29,7 +29,7 @@ def find_previous_sample(
if segment_id is not None:
cursor = conn.execute(
"SELECT id, ts, bytes_written, bytes_read, power_on_hours, "
" temperature_c, data_units_written, data_units_read "
" temperature_c, data_units_written, data_units_read, local_tz "
"FROM samples WHERE id < ? AND segment_id = ? "
"ORDER BY id DESC LIMIT 1",
(current_sample_id, segment_id),
@@ -37,7 +37,7 @@ def find_previous_sample(
else:
cursor = conn.execute(
"SELECT id, ts, bytes_written, bytes_read, power_on_hours, "
" temperature_c, data_units_written, data_units_read "
" temperature_c, data_units_written, data_units_read, local_tz "
"FROM samples WHERE id < ? "
"ORDER BY id DESC LIMIT 1",
(current_sample_id,),
@@ -49,7 +49,7 @@ def find_previous_sample(
"id": row[0], "ts": row[1], "bytes_written": row[2],
"bytes_read": row[3], "power_on_hours": row[4],
"temperature_c": row[5], "data_units_written": row[6],
"data_units_read": row[7],
"data_units_read": row[7], "local_tz": row[8],
}
src/fenris/local_day.py
+461 -132
View File
@@ -1,22 +1,21 @@
"""Local-day activity derivation from UTC hour observations.
"""Local-day activity derivation from measured sample intervals.
Computes durable local-day read/write summaries using the actual local
midnight boundaries, retaining UTC hour/day aggregates for endurance
projections. The collector owns this derivation, preserving the
existing read-only TUI boundary (ADR 0010).
Computes durable local-day read/write summaries using recorded local
midnight boundaries. UTC hour/day aggregates remain separate inputs to
endurance projections. The collector owns this derivation, preserving the
read-only TUI boundary (ADR 0010).
Key contracts:
- An interval wholly attributable to a local day contributes its volume once
- Midnight-spanning intervals are retained once as shared/unallocated evidence
- A compatible interval wholly inside a local day contributes its volume once
- Midnight-spanning and incompatible intervals persist once as unallocated evidence
- UTC hour/day aggregates are never modified or deleted
- Migration cannot manufacture local precision from historical UTC data
Retention policy (issue #93):
- Raw three-minute samples are pruned after 14 days (spec §3.4, ST-5).
- Hour observations and day aggregates are retained indefinitely.
- Local-day summaries are persisted at collection time and retained
indefinitely. They survive raw-sample pruning because they depend on
hour observations, not on raw samples.
- Local-day summaries and unallocated boundaries are persisted at collection
time and retained indefinitely. They survive raw-sample pruning.
- After detail expires, aged local summaries remain queryable with their
recorded timezone and UTC boundaries. The evidence-availability flag
on each history entry indicates whether the underlying raw detail is
@@ -28,7 +27,8 @@ Retention policy (issue #93):
"""
import sqlite3
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from datetime import date, datetime, time, timedelta, timezone
from zoneinfo import ZoneInfo
@dataclass(frozen=True)
@@ -39,11 +39,15 @@ class LocalDaySummary:
tz_offset: str # e.g. "+05:30"
utc_start: str # ISO 8601 UTC: the local midnight that starts this day
utc_end: str # ISO 8601 UTC: the local midnight that ends this day
bytes_written: int
bytes_read: int
bytes_written: int | None
bytes_read: int | None
coverage: float # known / (known + unknown) in UTC hours
sample_count: int
complete: bool # day's UTC range fully covered by hour observations
activity_seconds: int = 0
activity_intervals: int = 1
activity_incomplete: bool = False
activity_precision: str = "measured"
@dataclass(frozen=True)
@@ -60,13 +64,20 @@ class LocalDayHistoryEntry:
tz_offset: str
utc_start: str
utc_end: str
bytes_written: int
bytes_read: int
bytes_written: int | None
bytes_read: int | None
coverage: float
sample_count: int
complete: bool
detail_available: bool # True if raw samples for this day are within 14-day retention
derived_from_surviving: bool # True if derived from hour observations, not raw samples
shared_bytes_written: int = 0
shared_bytes_read: int = 0
unallocated_bytes_written: int = 0
unallocated_bytes_read: int = 0
shared_evidence_count: int = 0
unallocated_evidence_count: int = 0
activity_state: str = "unavailable"
@classmethod
def from_summary(
@@ -89,21 +100,32 @@ class LocalDayHistoryEntry:
complete=summary["complete"],
detail_available=detail_available,
derived_from_surviving=derived_from_surviving,
shared_bytes_written=summary.get("shared_bytes_written", 0),
shared_bytes_read=summary.get("shared_bytes_read", 0),
unallocated_bytes_written=summary.get("unallocated_bytes_written", 0),
unallocated_bytes_read=summary.get("unallocated_bytes_read", 0),
shared_evidence_count=summary.get("shared_evidence_count", 0),
unallocated_evidence_count=summary.get("unallocated_evidence_count", 0),
activity_state=summary.get("activity_state", "unavailable"),
)
def _local_midnight_utc(dt: datetime, tz_name: str) -> datetime:
"""Compute the UTC time of the local midnight that contains *dt*.
Returns the most recent local midnight in UTC. For example, if *dt*
is 2026-09-01T17:35:00+05:30 (i.e. 12:05 UTC) this returns
2026-08-31T18:30:00+00:00 (2026-09-01 00:00 in +05:30).
"""
from zoneinfo import ZoneInfo
local_tz = ZoneInfo(tz_name)
local_dt = dt.astimezone(local_tz)
local_midnight = local_dt.replace(hour=0, minute=0, second=0, microsecond=0)
return local_midnight.astimezone(timezone.utc)
def local_day_boundaries(local_date: str, tz_name: str) -> tuple[datetime, datetime, str]:
"""Return recorded UTC boundaries and midnight offset for one local date."""
day = date.fromisoformat(local_date)
zone = ZoneInfo(tz_name)
start_local = datetime.combine(day, time.min, tzinfo=zone)
end_local = datetime.combine(day + timedelta(days=1), time.min, tzinfo=zone)
start_utc = start_local.astimezone(timezone.utc)
end_utc = end_local.astimezone(timezone.utc)
offset = start_local.utcoffset()
offset_seconds = int(offset.total_seconds())
sign = "+" if offset_seconds >= 0 else "-"
offset_seconds = abs(offset_seconds)
offset_text = "%s%02d:%02d" % (
sign, offset_seconds // 3600, (offset_seconds % 3600) // 60
)
return start_utc, end_utc, offset_text
def derive_local_day_summary(
@@ -111,108 +133,78 @@ def derive_local_day_summary(
tz_name: str,
clock_now: datetime,
) -> LocalDaySummary | None:
"""Derive a local-day summary from UTC hour observations.
"""Build local-day coverage while preserving measured interval totals.
Computes the UTC boundaries of the current local day, queries the
UTC hours overlapping that range, and aggregates read/write volumes.
Midnight-spanning hours are retained once as shared evidence.
Returns None if no hours exist for the local day.
Hour rows support the established usage-habit coverage contract. They
cannot allocate a read/write delta to a half-hour local boundary, so
local activity totals come only from measured sample intervals.
"""
from zoneinfo import ZoneInfo
from .tz_util import get_tz_offset_str
local_tz = ZoneInfo(tz_name)
local_dt = clock_now.astimezone(local_tz)
local_date = local_dt.strftime("%Y-%m-%d")
tz_offset_str = get_tz_offset_str(clock_now, tz_name)
# UTC boundaries of this local day
utc_start = _local_midnight_utc(clock_now, tz_name)
next_local = local_dt + timedelta(days=1)
utc_end = next_local.replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc)
local_dt = clock_now.astimezone(ZoneInfo(tz_name))
local_date = local_dt.date().isoformat()
utc_start, utc_end, tz_offset_str = local_day_boundaries(local_date, tz_name)
utc_start_iso = utc_start.isoformat()
utc_end_iso = utc_end.isoformat()
# Query UTC hours overlapping the local day
cursor = conn.execute(
"SELECT hour, bytes_written_delta, bytes_read_delta, sample_count, "
" active_seconds, idle_seconds, powered_off_seconds, unknown_seconds "
"FROM hour_observations "
"WHERE hour >= ? AND hour < ? "
"ORDER BY hour",
rows = conn.execute(
"SELECT hour, sample_count, active_seconds, idle_seconds, "
" powered_off_seconds, unknown_seconds "
"FROM hour_observations WHERE hour >= ? AND hour < ? ORDER BY hour",
(utc_start_iso, utc_end_iso),
)
rows = cursor.fetchall()
).fetchall()
# Check for a midnight-spanning UTC hour before utc_start.
# When the local midnight falls inside a UTC hour (e.g. UTC+5:30
# where local midnight is 18:30 UTC), the hour 18:00 straddles the
# boundary. The main query (hour >= utc_start) excludes it because
# 18:00 < 18:30, so we must include it separately. This does NOT
# double-count: the hour falls outside the query range by
# construction (issue #93).
# A UTC hour can straddle local midnight. Count its existence for the
# coverage summary, but never use its bytes as local-day activity.
midnight_hour = utc_start.replace(minute=0, second=0, microsecond=0)
midnight_hour_iso = midnight_hour.strftime("%Y-%m-%dT%H:00:00+00:00")
prev_row = conn.execute(
"SELECT hour, bytes_written_delta, bytes_read_delta, sample_count "
"FROM hour_observations WHERE hour = ?",
previous_hour = conn.execute(
"SELECT hour, sample_count FROM hour_observations WHERE hour = ?",
(midnight_hour_iso,),
).fetchone()
total_bw = 0
total_br = 0
total_samples = 0
known_seconds = 0
unknown_seconds = 0
hour_count = 0
for row in rows:
total_bw += row[1] or 0
total_br += row[2] or 0
total_samples += row[3] or 0
known_seconds += (row[4] or 0) + (row[5] or 0) + (row[6] or 0)
unknown_seconds += row[7] or 0
total_samples += row[1] or 0
known_seconds += (row[2] or 0) + (row[3] or 0) + (row[4] or 0)
hour_count += 1
if previous_hour is not None and all(row[0] != previous_hour[0] for row in rows):
total_samples += previous_hour[1] or 0
hour_count += 1
# Include midnight-spanning hour if it exists and is not already
# in the main query results (it won't be, since hour < utc_start).
if prev_row is not None:
# Verify this hour is NOT already counted in the main query
prev_hour_iso = prev_row[0]
already_counted = any(r[0] == prev_hour_iso for r in rows)
if not already_counted:
total_bw += prev_row[1] or 0
total_br += prev_row[2] or 0
total_samples += prev_row[3] or 0
hour_count += 1
total_evidenced = known_seconds + unknown_seconds
# Coverage is known seconds as a share of the full local day,
# not just the hours present — gaps reduce coverage
day_seconds = int((utc_end - utc_start).total_seconds())
coverage = known_seconds / day_seconds if day_seconds > 0 else 0.0
# Day is complete only if known (usable) seconds cover the full local day.
# Unknown seconds represent gaps without usable observation evidence.
complete = known_seconds >= day_seconds and hour_count > 0
# Return None if no hours exist for this local day
if hour_count == 0:
return None
day_seconds = int((utc_end - utc_start).total_seconds())
coverage = known_seconds / day_seconds if day_seconds > 0 else 0.0
complete = known_seconds >= day_seconds and hour_count > 0
existing = conn.execute(
"SELECT bytes_written, bytes_read, activity_seconds, activity_intervals, "
" activity_incomplete, activity_precision "
"FROM local_days WHERE local_date = ? AND tz_name = ?",
(local_date, tz_name),
).fetchone()
if existing is None:
activity = (0, 0, 0, 0, False, "unavailable")
else:
activity = existing
return LocalDaySummary(
local_date=local_date,
tz_name=tz_name,
tz_offset=tz_offset_str,
utc_start=utc_start_iso,
utc_end=utc_end_iso,
bytes_written=total_bw,
bytes_read=total_br,
bytes_written=activity[0] or 0,
bytes_read=activity[1] or 0,
coverage=coverage,
sample_count=total_samples,
complete=complete,
activity_seconds=activity[2] or 0,
activity_intervals=activity[3] or 0,
activity_incomplete=bool(activity[4]),
activity_precision=activity[5],
)
@@ -230,12 +222,15 @@ def persist_local_day(
conn.execute(
"INSERT INTO local_days "
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
" bytes_written, bytes_read, coverage, sample_count, complete) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
" bytes_written, bytes_read, coverage, sample_count, complete, "
" activity_seconds, activity_intervals, activity_incomplete, activity_precision) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(summary.local_date, summary.tz_name, summary.tz_offset,
summary.utc_start, summary.utc_end,
summary.bytes_written, summary.bytes_read,
summary.coverage, summary.sample_count, summary.complete),
summary.coverage, summary.sample_count, summary.complete,
summary.activity_seconds, summary.activity_intervals,
summary.activity_incomplete, summary.activity_precision),
)
created = True
else:
@@ -243,11 +238,15 @@ def persist_local_day(
"UPDATE local_days "
"SET tz_offset = ?, utc_start = ?, utc_end = ?, "
" bytes_written = ?, bytes_read = ?, "
" coverage = ?, sample_count = ?, complete = ? "
" coverage = ?, sample_count = ?, complete = ?, "
" activity_seconds = ?, activity_intervals = ?, "
" activity_incomplete = ?, activity_precision = ? "
"WHERE id = ?",
(summary.tz_offset, summary.utc_start, summary.utc_end,
summary.bytes_written, summary.bytes_read,
summary.coverage, summary.sample_count, summary.complete,
summary.activity_seconds, summary.activity_intervals,
summary.activity_incomplete, summary.activity_precision,
existing[0]),
)
created = False
@@ -255,34 +254,371 @@ def persist_local_day(
return created
def _utc_datetime(value: str | datetime) -> datetime:
parsed = value if isinstance(value, datetime) else datetime.fromisoformat(value)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def _same_monitoring_period(
conn: sqlite3.Connection,
start: datetime,
end: datetime,
) -> bool:
"""Return whether one recorded monitoring period contains the interval."""
for started_at, ended_at in conn.execute(
"SELECT started_at, ended_at FROM monitoring_periods"
):
period_start = _utc_datetime(started_at)
period_end = _utc_datetime(ended_at) if ended_at else None
if period_start <= start and (period_end is None or end <= period_end):
return True
return False
def _upsert_activity_day(
conn: sqlite3.Connection,
local_date: str,
tz_name: str,
*,
bytes_written: int = 0,
bytes_read: int = 0,
seconds: int = 0,
interval_count: int = 0,
incomplete: bool = False,
last_sample_id: int | None = None,
) -> None:
start, end, offset = local_day_boundaries(local_date, tz_name)
conn.execute(
"INSERT INTO local_days "
"(local_date, tz_name, tz_offset, utc_start, utc_end, bytes_written, "
" bytes_read, activity_seconds, activity_intervals, activity_incomplete, "
" activity_precision, last_sample_id) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'measured', ?) "
"ON CONFLICT(local_date, tz_name) DO UPDATE SET "
"bytes_written = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" THEN excluded.bytes_written ELSE local_days.bytes_written + excluded.bytes_written END, "
"bytes_read = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" THEN excluded.bytes_read ELSE local_days.bytes_read + excluded.bytes_read END, "
"activity_seconds = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" THEN excluded.activity_seconds ELSE local_days.activity_seconds + excluded.activity_seconds END, "
"activity_intervals = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" THEN excluded.activity_intervals ELSE local_days.activity_intervals + excluded.activity_intervals END, "
"activity_incomplete = CASE WHEN local_days.activity_precision IN ('legacy', 'unavailable') "
" THEN excluded.activity_incomplete "
" ELSE MAX(local_days.activity_incomplete, excluded.activity_incomplete) END, "
"activity_precision = 'measured', "
"last_sample_id = COALESCE(excluded.last_sample_id, local_days.last_sample_id)",
(
local_date, tz_name, offset, start.isoformat(), end.isoformat(),
bytes_written, bytes_read, seconds, interval_count, incomplete,
last_sample_id,
),
)
def record_local_activity_interval(
conn: sqlite3.Connection,
previous: dict,
current: dict,
*,
start_sample_id: int,
end_sample_id: int,
segment_id: int | None,
) -> str | None:
"""Persist one measured interval as known local activity or once-only evidence.
Counter differences are allocated only when both sample timezones match,
both samples share one monitoring period, and the interval stays inside one
recorded local date. Every other valid difference remains one unallocated row.
"""
start = _utc_datetime(previous["ts"])
end = _utc_datetime(current["ts"])
if end <= start:
return None
previous_tz = previous.get("local_tz")
current_tz = current.get("local_tz")
if not current_tz:
return None
from_zone = ZoneInfo(previous_tz) if previous_tz else None
to_zone = ZoneInfo(current_tz)
start_date = start.astimezone(from_zone).date().isoformat() if from_zone else None
end_date = end.astimezone(to_zone).date().isoformat()
if start_date is None:
start_date = end_date
written = current.get("bytes_written")
prior_written = previous.get("bytes_written")
read = current.get("bytes_read")
prior_read = previous.get("bytes_read")
if None in (written, prior_written, read, prior_read):
return None
bytes_written = written - prior_written
bytes_read = read - prior_read
if bytes_written < 0 or bytes_read < 0:
for local_date, zone_name in (
(start_date, previous_tz), (end_date, current_tz),
):
if zone_name is not None:
_upsert_activity_day(
conn, local_date, zone_name, incomplete=True,
)
return "counter_discontinuity"
elif previous_tz is None:
reason = "legacy_timezone_unknown"
elif previous_tz != current_tz:
reason = "timezone_change"
elif segment_id is None:
reason = "segment_unknown"
elif not _same_monitoring_period(conn, start, end):
reason = "monitoring_period"
elif start_date != end_date:
reason = "local_midnight"
else:
already_accounted = conn.execute(
"SELECT last_sample_id FROM local_days "
"WHERE local_date = ? AND tz_name = ?",
(end_date, current_tz),
).fetchone()
if (
already_accounted is not None
and already_accounted[0] is not None
and already_accounted[0] >= end_sample_id
):
return "known"
seconds = int((end - start).total_seconds())
_upsert_activity_day(
conn,
end_date,
current_tz,
bytes_written=bytes_written,
bytes_read=bytes_read,
seconds=seconds,
interval_count=1,
last_sample_id=end_sample_id,
)
local_day_id = conn.execute(
"SELECT id FROM local_days WHERE local_date = ? AND tz_name = ?",
(end_date, current_tz),
).fetchone()[0]
conn.execute(
"INSERT INTO local_day_segment_totals "
"(local_day_id, segment_id, bytes_written, bytes_read, "
" activity_seconds, activity_intervals) "
"VALUES (?, ?, ?, ?, ?, 1) "
"ON CONFLICT(local_day_id, segment_id) DO UPDATE SET "
"bytes_written = local_day_segment_totals.bytes_written + excluded.bytes_written, "
"bytes_read = local_day_segment_totals.bytes_read + excluded.bytes_read, "
"activity_seconds = local_day_segment_totals.activity_seconds + excluded.activity_seconds, "
"activity_intervals = local_day_segment_totals.activity_intervals + 1",
(local_day_id, segment_id, bytes_written, bytes_read, seconds),
)
return "known"
conn.execute(
"INSERT OR IGNORE INTO local_day_unallocated_evidence "
"(start_sample_id, end_sample_id, start_local_date, end_local_date, "
" start_tz_name, end_tz_name, started_at, ended_at, bytes_written, "
" bytes_read, reason, segment_id) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
start_sample_id, end_sample_id, start_date, end_date,
previous_tz, current_tz, start.isoformat(), end.isoformat(),
bytes_written, bytes_read, reason, segment_id,
),
)
affected_days: set[tuple[str, str]] = set()
if previous_tz is not None:
affected_days.add((start_date, previous_tz))
affected_days.add((end_date, current_tz))
if previous_tz is not None and previous_tz == current_tz and start_date < end_date:
day = date.fromisoformat(start_date) + timedelta(days=1)
last = date.fromisoformat(end_date)
while day < last:
affected_days.add((day.isoformat(), current_tz))
day += timedelta(days=1)
for local_date, zone_name in affected_days:
_upsert_activity_day(
conn, local_date, zone_name, incomplete=True,
)
return reason
def mark_local_activity_gap(
conn: sqlite3.Connection,
current_ts: str,
current_tz: str,
previous: dict | None = None,
) -> None:
"""Mark dates around a controller boundary as missing local activity."""
current = _utc_datetime(current_ts)
current_date = current.astimezone(ZoneInfo(current_tz)).date().isoformat()
_upsert_activity_day(conn, current_date, current_tz, incomplete=True)
if previous is None or not previous.get("local_tz"):
return
previous_at = _utc_datetime(previous["ts"])
previous_tz = previous["local_tz"]
previous_date = previous_at.astimezone(ZoneInfo(previous_tz)).date().isoformat()
_upsert_activity_day(conn, previous_date, previous_tz, incomplete=True)
def query_local_day_summary(
conn: sqlite3.Connection,
local_date: str,
tz_name: str | None = None,
clock_now: datetime | None = None,
) -> dict | None:
"""Query a stored local-day summary by date.
Returns a dict with the summary fields, or None if no entry exists.
"""
tz_filter = " AND tz_name = ?" if tz_name else ""
params = (local_date, tz_name) if tz_name else (local_date,)
row = conn.execute(
"SELECT local_date, tz_name, tz_offset, utc_start, utc_end, "
" bytes_written, bytes_read, coverage, sample_count, complete "
"FROM local_days WHERE local_date = ? "
"ORDER BY id DESC LIMIT 1",
(local_date,),
" bytes_written, bytes_read, coverage, sample_count, complete, "
" activity_seconds, activity_intervals, activity_incomplete, "
" activity_precision "
"FROM local_days WHERE local_date = ?" + tz_filter +
" ORDER BY id DESC LIMIT 1",
params,
).fetchone()
if row is None:
return None
(
recorded_date,
recorded_tz,
tz_offset,
utc_start,
utc_end,
bytes_written,
bytes_read,
coverage,
sample_count,
complete,
activity_seconds,
activity_intervals,
activity_incomplete,
activity_precision,
) = row
evidence = conn.execute(
"SELECT COUNT(*), "
" COALESCE(SUM(CASE WHEN reason = 'local_midnight' THEN 1 ELSE 0 END), 0), "
" COALESCE(SUM(CASE WHEN reason != 'local_midnight' THEN 1 ELSE 0 END), 0), "
" COALESCE(SUM(CASE WHEN reason = 'local_midnight' "
" THEN bytes_written ELSE 0 END), 0), "
" COALESCE(SUM(CASE WHEN reason = 'local_midnight' "
" THEN bytes_read ELSE 0 END), 0), "
" COALESCE(SUM(CASE WHEN reason != 'local_midnight' "
" THEN bytes_written ELSE 0 END), 0), "
" COALESCE(SUM(CASE WHEN reason != 'local_midnight' "
" THEN bytes_read ELSE 0 END), 0) "
"FROM local_day_unallocated_evidence "
"WHERE (start_local_date = ? AND start_tz_name = ?) "
" OR (end_local_date = ? AND end_tz_name = ?) "
" OR (start_tz_name = ? AND end_tz_name = ? "
" AND start_local_date < ? AND end_local_date > ?)",
(
recorded_date, recorded_tz, recorded_date, recorded_tz,
recorded_tz, recorded_tz, recorded_date, recorded_date,
),
).fetchone()
(
evidence_count,
shared_evidence_count,
unallocated_evidence_count,
shared_bytes_written,
shared_bytes_read,
unallocated_bytes_written,
unallocated_bytes_read,
) = evidence
local_day_id = conn.execute(
"SELECT id FROM local_days WHERE local_date = ? AND tz_name = ?",
(recorded_date, recorded_tz),
).fetchone()[0]
segments = [
{
"segment_id": segment_id,
"bytes_written": segment_written,
"bytes_read": segment_read,
"activity_seconds": segment_seconds,
"activity_intervals": segment_intervals,
}
for (
segment_id,
segment_written,
segment_read,
segment_seconds,
segment_intervals,
) in conn.execute(
"SELECT segment_id, bytes_written, bytes_read, activity_seconds, "
" activity_intervals "
"FROM local_day_segment_totals WHERE local_day_id = ? "
"ORDER BY segment_id",
(local_day_id,),
)
]
interval_count = activity_intervals or 0
has_unallocated = evidence_count > 0
has_known = activity_precision == "measured" and interval_count > 0
if not has_known and not has_unallocated:
activity_state = "incomplete" if bool(activity_incomplete) else "unavailable"
else:
now = clock_now or datetime.now(timezone.utc)
is_current_day = (
now.astimezone(ZoneInfo(recorded_tz)).date().isoformat()
== recorded_date
)
day_seconds = int(
(_utc_datetime(utc_end) - _utc_datetime(utc_start)).total_seconds()
)
interval_seconds = activity_seconds or 0
full_activity_day = interval_seconds >= day_seconds or (
interval_count == 0 and bool(complete)
)
incomplete = (
bool(activity_incomplete) or has_unallocated or not full_activity_day
)
if is_current_day:
activity_state = "so_far"
elif incomplete:
activity_state = "incomplete"
elif has_known and (bytes_written or 0) == 0 and (bytes_read or 0) == 0:
activity_state = "zero"
else:
activity_state = "complete"
exact = activity_precision == "measured" and has_known
return {
"local_date": row[0],
"tz_name": row[1],
"tz_offset": row[2],
"utc_start": row[3],
"utc_end": row[4],
"bytes_written": row[5],
"bytes_read": row[6],
"coverage": row[7],
"sample_count": row[8],
"complete": bool(row[9]),
"local_date": recorded_date,
"tz_name": recorded_tz,
"tz_offset": tz_offset,
"utc_start": utc_start,
"utc_end": utc_end,
"bytes_written": bytes_written if exact else None,
"bytes_read": bytes_read if exact else None,
"shared_evidence_count": shared_evidence_count,
"unallocated_evidence_count": unallocated_evidence_count,
"shared_bytes_written": shared_bytes_written,
"shared_bytes_read": shared_bytes_read,
"unallocated_bytes_written": unallocated_bytes_written,
"unallocated_bytes_read": unallocated_bytes_read,
"coverage": coverage,
"sample_count": sample_count,
"complete": bool(complete),
"activity_seconds": activity_seconds or 0,
"activity_intervals": interval_count,
"activity_incomplete": bool(activity_incomplete),
"activity_precision": activity_precision,
"activity_state": activity_state,
"segments": segments,
}
@@ -296,7 +632,7 @@ def query_current_local_day(
local_tz = ZoneInfo(tz_name)
local_dt = clock_now.astimezone(local_tz)
local_date = local_dt.strftime("%Y-%m-%d")
return query_local_day_summary(conn, local_date)
return query_local_day_summary(conn, local_date, tz_name, clock_now)
def _is_detail_available(
@@ -347,27 +683,20 @@ def query_local_day_history(
"""
rows = conn.execute(
"SELECT local_date, tz_name, tz_offset, utc_start, utc_end, "
" bytes_written, bytes_read, coverage, sample_count, complete "
" bytes_written, bytes_read, coverage, sample_count, complete, "
" activity_seconds, activity_intervals, activity_incomplete, "
" activity_precision "
"FROM local_days "
"WHERE local_date >= ? AND local_date <= ? "
"ORDER BY local_date",
"ORDER BY local_date, id",
(start_date, end_date),
).fetchall()
entries = []
for row in rows:
summary = {
"local_date": row[0],
"tz_name": row[1],
"tz_offset": row[2],
"utc_start": row[3],
"utc_end": row[4],
"bytes_written": row[5],
"bytes_read": row[6],
"coverage": row[7],
"sample_count": row[8],
"complete": row[9],
}
summary = query_local_day_summary(conn, row[0], row[1], now)
if summary is None:
continue
detail_available = _is_detail_available(conn, row[3], row[4], now)
entries.append(
LocalDayHistoryEntry.from_summary(
src/fenris/store.py
+90 -2
View File
@@ -11,7 +11,7 @@ from typing import Optional
# Schema version - increment on each migration
SCHEMA_VERSION = 4
SCHEMA_VERSION = 5
# Packaged default placement (spec §8.3). The config may override it, but a
@@ -106,7 +106,8 @@ def _create_schema(conn: sqlite3.Connection):
bytes_written INTEGER,
bytes_read INTEGER,
critical_warning INTEGER,
segment_id INTEGER
segment_id INTEGER,
local_tz TEXT
)
""")
@@ -208,10 +209,18 @@ def _create_schema(conn: sqlite3.Connection):
coverage REAL DEFAULT 0.0,
sample_count INTEGER DEFAULT 0,
complete BOOLEAN DEFAULT 0,
activity_seconds INTEGER NOT NULL DEFAULT 0,
activity_intervals INTEGER NOT NULL DEFAULT 0,
activity_incomplete BOOLEAN NOT NULL DEFAULT 0,
activity_precision TEXT NOT NULL DEFAULT 'measured',
last_sample_id INTEGER,
UNIQUE(local_date, tz_name)
)
""")
_create_local_day_shared_evidence(conn)
_create_local_day_segment_totals(conn)
# Metadata table for store state (e.g., legacy import marker)
conn.execute("""
CREATE TABLE IF NOT EXISTS store_metadata (
@@ -234,6 +243,52 @@ def _create_pending_publications(conn: sqlite3.Connection) -> None:
""")
def _create_local_day_shared_evidence(conn: sqlite3.Connection) -> None:
"""Create once-only local activity evidence that cannot be day-allocated."""
conn.execute("""
CREATE TABLE IF NOT EXISTS local_day_unallocated_evidence (
id INTEGER PRIMARY KEY AUTOINCREMENT,
start_sample_id INTEGER NOT NULL,
end_sample_id INTEGER NOT NULL,
start_local_date TEXT NOT NULL,
end_local_date TEXT NOT NULL,
start_tz_name TEXT,
end_tz_name TEXT,
started_at TEXT NOT NULL,
ended_at TEXT NOT NULL,
bytes_written INTEGER NOT NULL DEFAULT 0,
bytes_read INTEGER NOT NULL DEFAULT 0,
reason TEXT NOT NULL,
segment_id INTEGER,
UNIQUE(start_sample_id, end_sample_id)
)
""")
conn.execute(
"CREATE INDEX IF NOT EXISTS local_day_evidence_start "
"ON local_day_unallocated_evidence(start_local_date, start_tz_name)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS local_day_evidence_end "
"ON local_day_unallocated_evidence(end_local_date, end_tz_name)"
)
def _create_local_day_segment_totals(conn: sqlite3.Connection) -> None:
"""Retain the controller-segment provenance behind known day totals."""
conn.execute("""
CREATE TABLE IF NOT EXISTS local_day_segment_totals (
id INTEGER PRIMARY KEY AUTOINCREMENT,
local_day_id INTEGER NOT NULL,
segment_id INTEGER NOT NULL,
bytes_written INTEGER NOT NULL DEFAULT 0,
bytes_read INTEGER NOT NULL DEFAULT 0,
activity_seconds INTEGER NOT NULL DEFAULT 0,
activity_intervals INTEGER NOT NULL DEFAULT 0,
UNIQUE(local_day_id, segment_id)
)
""")
def _apply_migrations(conn: sqlite3.Connection, current_version: int):
"""Apply forward-only migrations from current_version to SCHEMA_VERSION.
@@ -293,6 +348,39 @@ def _apply_migrations(conn: sqlite3.Connection, current_version: int):
_create_pending_publications(conn)
current_version = 4
# Migration 4→5: retain precise measured local-day activity and once-only
# unallocated intervals. Existing local-day totals came from UTC-hour
# aggregates, so preserve their rows but mark their local precision legacy.
if current_version < 5:
tables = {row[0] for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()}
if "samples" in tables:
sample_cols = {row[1] for row in conn.execute(
"PRAGMA table_info(samples)"
).fetchall()}
if "local_tz" not in sample_cols:
conn.execute("ALTER TABLE samples ADD COLUMN local_tz TEXT")
if "local_days" in tables:
local_cols = {row[1] for row in conn.execute(
"PRAGMA table_info(local_days)"
).fetchall()}
for column, declaration in (
("activity_seconds", "INTEGER NOT NULL DEFAULT 0"),
("activity_intervals", "INTEGER NOT NULL DEFAULT 0"),
("activity_incomplete", "BOOLEAN NOT NULL DEFAULT 0"),
("activity_precision", "TEXT NOT NULL DEFAULT 'legacy'"),
("last_sample_id", "INTEGER"),
):
if column not in local_cols:
conn.execute(
"ALTER TABLE local_days ADD COLUMN %s %s" %
(column, declaration)
)
_create_local_day_shared_evidence(conn)
_create_local_day_segment_totals(conn)
current_version = 5
def migrate_to_latest(store_path: Path) -> int:
"""Apply forward-only migrations to bring the store to SCHEMA_VERSION.
src/fenris/tui.py
+43 -16
View File
@@ -1709,7 +1709,9 @@ class FenrisTuiApp(App):
)
tz_name = detect_system_tz()
if self._browse_date is not None:
local = query_local_day_summary(conn, self._browse_date)
local = query_local_day_summary(
conn, self._browse_date, clock_now=self._clock_now
)
else:
local = query_current_local_day(conn, self._clock_now, tz_name)
except Exception:
@@ -1720,7 +1722,10 @@ class FenrisTuiApp(App):
if local is None:
date = self._browse_date or self._clock_now.astimezone().date().isoformat()
widget.update("%s · local-day evidence unavailable\nW unavailable · R unavailable" % date)
widget.update(
"%s · local-day evidence unavailable\nW unavailable · R unavailable"
% date
)
widget.display = True
main_grid.remove_class("local-day")
return
@@ -1728,23 +1733,45 @@ class FenrisTuiApp(App):
main_grid.add_class("local-day")
widget.styles.display = "block"
tz_display = "%s %s" % (local["tz_name"], local["tz_offset"])
state = local["activity_state"]
state_labels = {
"so_far": "totals so far",
"incomplete": "incomplete",
"complete": "complete",
"zero": "measured zero",
"unavailable": "local activity unavailable",
}
label = state_labels.get(state, "local activity unavailable")
bw = local["bytes_written"]
br = local["bytes_read"]
partial = "" if local["complete"] else " · totals so far"
tz_display = "%s %s" % (local["tz_name"], local["tz_offset"])
text = (
"[bold]%s[/bold] · %s%s\n"
" W %.3f GB · R %.3f GB · %.0f%% coverage"
% (
local["local_date"],
tz_display,
partial,
bw / 1e9,
br / 1e9,
local["coverage"] * 100,
if bw is None or br is None:
totals = "W unavailable · R unavailable"
else:
totals = "W %.3f GB known · R %.3f GB known" % (
bw / 1e9, br / 1e9
)
)
lines = [
"[bold]%s[/bold] · %s · %s" % (
local["local_date"], tz_display, label,
),
totals,
]
if local["shared_evidence_count"]:
lines.append(
"shared at midnight W %.3f GB · R %.3f GB" % (
local["shared_bytes_written"] / 1e9,
local["shared_bytes_read"] / 1e9,
)
)
if local["unallocated_evidence_count"]:
lines.append(
"unallocated W %.3f GB · R %.3f GB" % (
local["unallocated_bytes_written"] / 1e9,
local["unallocated_bytes_read"] / 1e9,
)
)
text = "\n".join(lines)
widget.update(text)
def _on_graph_drill(self, day: str) -> None: