xavierk/Fenris
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases 11 Wiki Activity

Publish consistent measured drive activity (#89)

Browse Source 
Repair the collection-to-display path so each successful acquisition
publishes correct read/write deltas through the observation store and
visible dashboard.

Fixes:
- derive.py: accumulate bytes_read_delta on same-hour hour_observation
  merge (was silently dropped)
- collector.py: rebuild day_aggregates from hour observations after
  each collection run (previously only populated for cross-hour intervals)
- day_aggregate.py: add persist_day_aggregate upsert helper
- tui.py: query and display both read and write deltas in daily and
  hourly readouts, constrained summaries, and graph data queries

Tests:
- Add 14 integration tests (test_measured_activity.py) exercising the
  full collector→store→reader→display path with real fixtures and
  injected time
- Update constrained-layout assertion to match new W/R format

Closes #89
This commit is contained in:
xavierk
2026-09-17 17:43:31 +05:30
parent 54cd56e4ec
commit 366c2f55b7
6 changed files with 693 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fenris/collector.py
+7 -1
View File
@@ -325,7 +325,7 @@ def run_collection(
# Find the sample we just wrote
cursor = conn.execute("SELECT id FROM samples ORDER BY id DESC LIMIT 1")
current_id = cursor.fetchone()[0]
prev = find_previous_sample(conn, seg_info.get("segment_id"), current_id)
if prev is not None:
# Build current sample dict for derivation
@@ -340,6 +340,12 @@ def run_collection(
"data_units_read": sample["data_units_read"],
}
derive_hours_from_interval(conn, prev, current)
# Rebuild day aggregates from hour observations
from .day_aggregate import derive_all_days, persist_day_aggregate
for agg in derive_all_days(conn):
persist_day_aggregate(conn, agg)
conn.commit()
except Exception:
# Derivation failure must not prevent sample persistence (issue #73 AC6)
pass
src/fenris/day_aggregate.py
+38
View File
@@ -163,3 +163,41 @@ def derive_all_days(conn: sqlite3.Connection) -> list[DayAggregate]:
if agg is not None:
results.append(agg)
return results
def persist_day_aggregate(conn: sqlite3.Connection, agg: DayAggregate) -> None:
"""Upsert a derived day aggregate into the day_aggregates table.
Merges attributed bytes from hour observations with any existing
unattributed cross-hour evidence already stored for this day.
Caller must manage transactions and commits.
"""
existing = conn.execute(
"SELECT id FROM day_aggregates WHERE day = ?",
(agg.day,),
).fetchone()
if existing is None:
conn.execute(
"INSERT INTO day_aggregates "
"(day, active_seconds, idle_seconds, powered_off_seconds, unknown_seconds, "
" bytes_written_delta, bytes_read_delta, sample_count, coverage) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(agg.day, agg.seconds_active, agg.seconds_idle,
agg.seconds_powered_off, agg.seconds_unknown,
agg.bytes_written_delta, agg.bytes_read_delta,
agg.sample_count, agg.coverage),
)
else:
conn.execute(
"UPDATE day_aggregates "
"SET active_seconds = ?, idle_seconds = ?, powered_off_seconds = ?, "
" unknown_seconds = ?, bytes_written_delta = ?, bytes_read_delta = ?, "
" sample_count = ?, coverage = ? "
"WHERE id = ?",
(agg.seconds_active, agg.seconds_idle,
agg.seconds_powered_off, agg.seconds_unknown,
agg.bytes_written_delta, agg.bytes_read_delta,
agg.sample_count, agg.coverage,
existing[0]),
)
src/fenris/derive.py
+8 -4
View File
@@ -175,7 +175,8 @@ def _upsert_hour_observation(
"""Insert or update an hour observation."""
# Check if hour exists
existing = conn.execute(
"SELECT id, bytes_written_delta, sample_count FROM hour_observations WHERE hour = ?",
"SELECT id, bytes_written_delta, bytes_read_delta, sample_count "
"FROM hour_observations WHERE hour = ?",
(hour_key,),
).fetchone()
@@ -198,10 +199,13 @@ def _upsert_hour_observation(
else:
# Merge: accumulate bytes and sample count
new_bw = existing[1] + bw_delta
new_samples = existing[2] + sample_count
new_br = existing[2] + br_delta
new_samples = existing[3] + sample_count
conn.execute(
"UPDATE hour_observations SET bytes_written_delta = ?, sample_count = ? WHERE id = ?",
(new_bw, new_samples, existing[0]),
"UPDATE hour_observations "
"SET bytes_written_delta = ?, bytes_read_delta = ?, sample_count = ? "
"WHERE id = ?",
(new_bw, new_br, new_samples, existing[0]),
)
conn.commit()
src/fenris/tui.py
+88 -43
View File
@@ -285,15 +285,16 @@ class DailyBarGraph(Widget):
return
# Summarise visible days as text
total_bytes = sum(d.get("total_bytes", 0) for d in self._day_data)
days_with_data = sum(1 for d in self._day_data if d.get("total_bytes", 0) > 0)
total_written = sum(d.get("total_written", d.get("total_bytes", 0)) for d in self._day_data)
total_read = sum(d.get("total_read", 0) for d in self._day_data)
days_with_data = sum(1 for d in self._day_data if d.get("total_bytes", 0) > 0 or d.get("total_read", 0) > 0)
n = len(self._day_data)
first = self._day_data[0].get("local_label", "")
last = self._day_data[-1].get("local_label", "")
self.query_one("#bar-render").update(
"[dim]Graph needs ≥80×24[/dim]\n"
" %d days · %d with writes · %.3f GB total\n"
" %s → %s" % (n, days_with_data, total_bytes / 1e9, first, last)
" %d days · %d with activity · W %.3f GB R %.3f GB\n"
" %s → %s" % (n, days_with_data, total_written / 1e9, total_read / 1e9, first, last)
)
self.query_one("#bar-legend").update("")
@@ -301,30 +302,41 @@ class DailyBarGraph(Widget):
if 0 <= self.selected_index < len(self._day_data):
day = self._day_data[self.selected_index]
self.query_one("#bar-readout").update(
"[bold]%s[/bold] · %.3f GB"
% (day.get("local_label", ""), day.get("total_bytes", 0) / 1e9)
"[bold]%s[/bold] \u00b7 W %.3f GB \u00b7 R %.3f GB"
% (
day.get("local_label", ""),
day.get("total_written", day.get("total_bytes", 0)) / 1e9,
day.get("total_read", 0) / 1e9,
)
)
else:
self.query_one("#bar-readout").update("[dim]No selectable day[/dim]")
def _show_constrained_hourly_summary(self) -> None:
"""Textual fallback for hourly view when terminal is too small."""
total_bytes = sum(h.get("bytes_written", 0) for h in self._hour_data)
total_written = sum(h.get("bytes_written", 0) for h in self._hour_data)
total_read = sum(h.get("bytes_read", 0) for h in self._hour_data)
hours_with_data = sum(
1 for h in self._hour_data if h.get("bytes_written", 0) > 0
1 for h in self._hour_data
if h.get("bytes_written", 0) > 0 or h.get("bytes_read", 0) > 0
)
n = len(self._hour_data)
self.query_one("#bar-render").update(
"[dim]Graph needs ≥80×24[/dim]\n"
" %d hours · %d with writes · %.3f GB total" % (n, hours_with_data, total_bytes / 1e9)
" %d hours · %d with activity · W %.3f GB R %.3f GB"
% (n, hours_with_data, total_written / 1e9, total_read / 1e9)
)
self.query_one("#bar-legend").update("")
if 0 <= self._hourly_selected < len(self._hour_data):
h = self._hour_data[self._hourly_selected]
self.query_one("#bar-readout").update(
"[bold]%s[/bold] \u00b7 %.3f GB"
% (h.get("local_label", ""), h.get("bytes_written", 0) / 1e9)
"[bold]%s[/bold] \u00b7 W %.3f GB \u00b7 R %.3f GB"
% (
h.get("local_label", ""),
h.get("bytes_written", 0) / 1e9,
h.get("bytes_read", 0) / 1e9,
)
)
else:
self.query_one("#bar-readout").update("[dim]\u2190 \u2192 Select hour[/dim]")
@@ -461,31 +473,42 @@ class DailyBarGraph(Widget):
return
day = self._day_data[self.selected_index]
total = day.get("total_bytes", 0)
allocated = day.get("allocated_bytes", 0)
unallocated = day.get("unallocated_bytes", 0)
total_written = day.get("total_written", day.get("total_bytes", 0))
total_read = day.get("total_read", 0)
allocated_w = day.get("allocated_bytes", 0)
unallocated_w = day.get("unallocated_bytes", 0)
allocated_r = day.get("allocated_read", 0)
unallocated_r = day.get("unallocated_read", 0)
coverage = day.get("coverage", 0)
hours = day.get("evidenced_hours", 0)
total_gb = total / 1e9
state = " · partial" if day.get("is_partial") else (
" · gap" if day.get("is_gap") else ""
)
parts = [
"[bold]%s UTC[/bold] \u00b7 %.3f GB total \u00b7 %d hours \u00b7 %.0f%% coverage%s"
"[bold]%s UTC[/bold] \u00b7 W %.3f GB \u00b7 R %.3f GB \u00b7 %d hours \u00b7 %.0f%% coverage%s"
% (
day.get("local_label", day.get("day", "")),
total_gb,
total_written / 1e9,
total_read / 1e9,
hours,
coverage * 100,
state,
),
]
if unallocated > 0:
parts.append(
" Allocated %.3f GB \u00b7 unallocated %.3f GB"
% (allocated / 1e9, unallocated / 1e9)
alloc_parts = []
if unallocated_w > 0:
alloc_parts.append(
"W alloc %.3f GB \u00b7 unalloc %.3f GB"
% (allocated_w / 1e9, unallocated_w / 1e9)
)
if unallocated_r > 0:
alloc_parts.append(
"R alloc %.3f GB \u00b7 unalloc %.3f GB"
% (allocated_r / 1e9, unallocated_r / 1e9)
)
if alloc_parts:
parts.append(" " + " \u00b7 ".join(alloc_parts))
self.query_one("#bar-readout").update("\n".join(parts))
# -- Hourly drill-down rendering --
@@ -580,10 +603,11 @@ class DailyBarGraph(Widget):
self._drill_unallocated_bytes / 1e9,
)
self.query_one("#bar-readout").update(
"[bold]%s:00 UTC[/bold] \u00b7 %.3f GB \u00b7 %d%% coverage%s%s"
"[bold]%s:00 UTC[/bold] \u00b7 W %.3f GB \u00b7 R %.3f GB \u00b7 %d%% coverage%s%s"
% (
h.get("local_label", h.get("hour", "")),
h.get("bytes_written", 0) / 1e9,
h.get("bytes_read", 0) / 1e9,
h.get("coverage", 0) * 100,
state,
note,
@@ -708,7 +732,8 @@ def _query_daily_graph_data(
cursor = conn.execute(
"SELECT day, bytes_written_delta, unattributed_bytes_written, "
"coverage, sample_count, active_seconds, idle_seconds, "
"powered_off_seconds, unknown_seconds "
"powered_off_seconds, unknown_seconds, "
"bytes_read_delta, unattributed_bytes_read "
"FROM day_aggregates ORDER BY day"
)
rows = cursor.fetchall()
@@ -717,18 +742,21 @@ def _query_daily_graph_data(
for row in rows:
day = row[0]
bw_delta = row[1] or 0
unattributed = row[2] or 0
unattributed_w = row[2] or 0
coverage = row[3] or 0.0
sample_count = row[4] or 0
active = row[5] or 0
idle = row[6] or 0
powered_off = row[7] or 0
unknown = row[8] or 0
br_delta = row[9] or 0
unattributed_r = row[10] or 0
total_bytes = bw_delta + unattributed
total_written = bw_delta + unattributed_w
total_read = br_delta + unattributed_r
evidenced_hours = (active + idle + powered_off) // 3600
is_zero = total_bytes == 0
is_zero = total_written == 0 and total_read == 0
is_gap = (
sample_count == 0
and (active + idle + powered_off) == 0
@@ -739,9 +767,13 @@ def _query_daily_graph_data(
by_day[day] = {
"day": day,
"local_label": day,
"total_bytes": total_bytes,
"total_bytes": total_written,
"total_written": total_written,
"total_read": total_read,
"allocated_bytes": bw_delta,
"unallocated_bytes": unattributed,
"unallocated_bytes": unattributed_w,
"allocated_read": br_delta,
"unallocated_read": unattributed_r,
"coverage": coverage,
"evidenced_hours": evidenced_hours,
"sample_count": sample_count,
@@ -761,8 +793,12 @@ def _query_daily_graph_data(
"day": day,
"local_label": day,
"total_bytes": 0,
"total_written": 0,
"total_read": 0,
"allocated_bytes": 0,
"unallocated_bytes": 0,
"allocated_read": 0,
"unallocated_read": 0,
"coverage": 0.0,
"evidenced_hours": 0,
"sample_count": 0,
@@ -785,10 +821,10 @@ def _query_hourly_graph_data(
) -> List[Dict[str, Any]]:
"""Query hour observations for a specific day.
Returns one dict per hour with bytes written, coverage, and flags.
Returns one dict per hour with bytes written/read, coverage, and flags.
"""
cursor = conn.execute(
"SELECT hour, bytes_written_delta, coverage, sample_count, "
"SELECT hour, bytes_written_delta, bytes_read_delta, coverage, sample_count, "
"active_seconds, idle_seconds, powered_off_seconds, unknown_seconds "
"FROM hour_observations "
"WHERE hour LIKE ? ORDER BY hour",
@@ -800,14 +836,15 @@ def _query_hourly_graph_data(
for row in rows:
hour = row[0]
bw = row[1] or 0
coverage = row[2] or 0.0
sample_count = row[3] or 0
active = row[4] or 0
idle = row[5] or 0
powered_off = row[6] or 0
unknown = row[7] or 0
br = row[2] or 0
coverage = row[3] or 0.0
sample_count = row[4] or 0
active = row[5] or 0
idle = row[6] or 0
powered_off = row[7] or 0
unknown = row[8] or 0
is_zero = bw == 0
is_zero = bw == 0 and br == 0
local_label = hour[11:13] if len(hour) >= 13 else hour
hour_number = int(local_label)
@@ -815,6 +852,7 @@ def _query_hourly_graph_data(
"hour": hour,
"local_label": local_label,
"bytes_written": bw,
"bytes_read": br,
"coverage": coverage,
"sample_count": sample_count,
"active_seconds": active,
@@ -841,6 +879,7 @@ def _query_hourly_graph_data(
"hour": hour_start.isoformat(),
"local_label": "%02d" % hour_number,
"bytes_written": 0,
"bytes_read": 0,
"coverage": 0.0,
"sample_count": 0,
"active_seconds": 0,
@@ -1135,22 +1174,28 @@ class FenrisTuiApp(App):
return
# Summarise visible days as text (issue #81 AC2)
total_bytes = sum(d.get("total_bytes", 0) for d in graph._day_data)
days_with_data = sum(1 for d in graph._day_data if d.get("total_bytes", 0) > 0)
total_written = sum(d.get("total_written", d.get("total_bytes", 0)) for d in graph._day_data)
total_read = sum(d.get("total_read", 0) for d in graph._day_data)
days_with_data = sum(
1 for d in graph._day_data
if d.get("total_bytes", 0) > 0 or d.get("total_read", 0) > 0
)
n = len(graph._day_data)
first = graph._day_data[0].get("local_label", "")
last = graph._day_data[-1].get("local_label", "")
text = (
"[dim]Graph needs ≥80×24[/dim]\n"
" %d days · %d with writes · %.3f GB total\n"
" %s → %s" % (n, days_with_data, total_bytes / 1e9, first, last)
" %d days · %d with activity · W %.3f GB R %.3f GB\n"
" %s → %s" % (n, days_with_data, total_written / 1e9, total_read / 1e9, first, last)
)
# Preserve selected-day context (issue #81 AC2)
if 0 <= graph.selected_index < len(graph._day_data):
day = graph._day_data[graph.selected_index]
text += "\n [bold]%s[/bold] · %.3f GB" % (
day.get("local_label", ""), day.get("total_bytes", 0) / 1e9
text += "\n [bold]%s[/bold] \u00b7 W %.3f GB \u00b7 R %.3f GB" % (
day.get("local_label", ""),
day.get("total_written", day.get("total_bytes", 0)) / 1e9,
day.get("total_read", 0) / 1e9,
)
summary.update(text)