Compare commits
15
Commits
v0.5.0
..
d045e88043
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d045e88043 | ||
|
|
c4524cf654 | ||
|
|
f780212a02 | ||
|
|
3e1ecc90e4 | ||
|
|
59d2dd2634 | ||
|
|
fd4db45ae1 | ||
|
|
5aeee7b6f1 | ||
|
|
d69753690a | ||
|
|
017a562566 | ||
|
|
7e392c4ea2 | ||
|
|
8a5ef05188 | ||
|
|
847bde1be0 | ||
|
|
0d45f263d2 | ||
|
|
7c21b044ba | ||
|
|
437ea6be73 |
@@ -263,6 +263,12 @@ measured read/write volumes, not transfer speed; missing evidence breaks the
|
|||||||
trace. `?` marks a gap, `~` a partial total, and `u` unallocated daily volume.
|
trace. `?` marks a gap, `~` a partial total, and `u` unallocated daily volume.
|
||||||
Use the selected-point readout for exact values and evidence state.
|
Use the selected-point readout for exact values and evidence state.
|
||||||
|
|
||||||
|
Local-day totals use measured intervals inside each recorded local date. An
|
||||||
|
interval that crosses local midnight appears once as shared evidence and stays
|
||||||
|
outside both known totals. The dashboard labels current totals as so far and
|
||||||
|
shows incomplete or unavailable dates without treating them as zero. Older
|
||||||
|
UTC-only summaries cannot establish exact local-day totals.
|
||||||
|
|
||||||
- `v` cycles Live / Day / History; the tabs are also clickable.
|
- `v` cycles Live / Day / History; the tabs are also clickable.
|
||||||
- `←` / `→` inspect points; `w` switches read/write volume in every view.
|
- `←` / `→` inspect points; `w` switches read/write volume in every view.
|
||||||
- `[` / `]` browse dates, `g` enters a date, and `t` returns to today/live.
|
- `[` / `]` browse dates, `g` enters a date, and `t` returns to today/live.
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# 10. Preserve local-day activity history with its recorded timezone
|
||||||
|
|
||||||
|
Status: Accepted; legacy-history migration and repair implemented in issue #99.
|
||||||
|
|
||||||
|
Fenris will retain local-day read/write summaries and the boundary evidence needed to interpret them in the existing observation store before three-minute detail expires after 14 days. Each historical summary retains its recorded timezone and day boundaries; this extends [ADR 0001](0001-observation-store-sqlite.md) while preserving the UTC hour/day evidence used for endurance projections. Collection derives new measured intervals. Ordered schema migration and explicit repair rebuild legacy summaries from surviving evidence; TUI and CLI readers remain read-only.
|
||||||
|
|
||||||
|
UTC hourly summaries alone cannot recover a local day whose midnight falls inside a UTC hour. Preserving labelled local summaries trades arbitrary future timezone reinterpretation for bounded detailed-history retention; retaining all fine-grained intervals indefinitely or estimating a split from UTC summaries would violate the agreed retention or evidence semantics.
|
||||||
|
|
||||||
|
Only surviving evidence may be used to derive older local summaries. Dates without enough evidence remain incomplete or unavailable; a measured interval crossing midnight is retained once as shared boundary evidence, never prorated or counted in full on both days. A later timezone change does not silently rewrite historical day boundaries.
|
||||||
|
|
||||||
|
The agreed presentation and validation requirements are in the [live drive activity specification](../spec/live-drive-activity.md).
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
"""User choices for live and historical activity views."""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
HISTORY_RANGE_DEFAULT = 14
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class IntervalIdentity:
|
||||||
|
"""Stable identity for one measured interval."""
|
||||||
|
|
||||||
|
start_ts: str
|
||||||
|
end_ts: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class HistoryDayIdentity:
|
||||||
|
"""Stable identity for one recorded local-day summary."""
|
||||||
|
|
||||||
|
local_date: str
|
||||||
|
timezone: str
|
||||||
|
utc_start: str
|
||||||
|
utc_end: str
|
||||||
|
|
||||||
|
|
||||||
|
class ActivitySelection:
|
||||||
|
"""Keep activity navigation state separate from its Textual rendering."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.view = "live"
|
||||||
|
self.browse_date: str | None = None
|
||||||
|
self.selected_history_day: HistoryDayIdentity | None = None
|
||||||
|
self.selected_history_hour: str | None = None
|
||||||
|
self.history_range_days = HISTORY_RANGE_DEFAULT
|
||||||
|
self.measure = "written"
|
||||||
|
self.following_live = True
|
||||||
|
self.selected_live_interval: IntervalIdentity | None = None
|
||||||
|
self.live_interval_expired = False
|
||||||
|
|
||||||
|
def set_view(self, view: str) -> None:
|
||||||
|
"""Select live, day, or history view."""
|
||||||
|
if view not in ("live", "day", "history"):
|
||||||
|
raise ValueError(f"unknown activity view: {view}")
|
||||||
|
self.view = view
|
||||||
|
if view == "live":
|
||||||
|
self.follow_live()
|
||||||
|
|
||||||
|
def select_history_date(
|
||||||
|
self,
|
||||||
|
date: str | None,
|
||||||
|
identity: HistoryDayIdentity | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Keep requested date and evidence identity across refresh."""
|
||||||
|
if date != self.browse_date or (
|
||||||
|
identity is not None and identity != self.selected_history_day
|
||||||
|
):
|
||||||
|
self.selected_history_hour = None
|
||||||
|
if date != self.browse_date:
|
||||||
|
self.selected_history_day = None
|
||||||
|
self.browse_date = date
|
||||||
|
if identity is not None:
|
||||||
|
self.selected_history_day = identity
|
||||||
|
elif date is None:
|
||||||
|
self.selected_history_day = None
|
||||||
|
|
||||||
|
def select_history_hour(self, hour: str | None) -> None:
|
||||||
|
"""Keep the chosen historical hour across refresh."""
|
||||||
|
self.selected_history_hour = hour
|
||||||
|
|
||||||
|
def toggle_measure(self) -> str:
|
||||||
|
"""Switch read/write presentation without changing point selection."""
|
||||||
|
self.measure = "read" if self.measure == "written" else "written"
|
||||||
|
return self.measure
|
||||||
|
|
||||||
|
def follow_live(self) -> None:
|
||||||
|
"""Resume following the newest interval and clear stale-pin notices."""
|
||||||
|
self.view = "live"
|
||||||
|
self.browse_date = None
|
||||||
|
self.selected_history_day = None
|
||||||
|
self.selected_history_hour = None
|
||||||
|
self.following_live = True
|
||||||
|
self.selected_live_interval = None
|
||||||
|
self.live_interval_expired = False
|
||||||
|
|
||||||
|
def update_live(self, intervals: Sequence[IntervalIdentity]) -> None:
|
||||||
|
"""Reconcile the selected interval with the rolling live window."""
|
||||||
|
if self.following_live:
|
||||||
|
self.selected_live_interval = intervals[-1] if intervals else None
|
||||||
|
return
|
||||||
|
if self.selected_live_interval in intervals:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.following_live = True
|
||||||
|
self.selected_live_interval = intervals[-1] if intervals else None
|
||||||
|
self.live_interval_expired = True
|
||||||
|
|
||||||
|
def move_live(
|
||||||
|
self, intervals: Sequence[IntervalIdentity], offset: int,
|
||||||
|
) -> IntervalIdentity | None:
|
||||||
|
"""Inspect an adjacent interval by stable identity."""
|
||||||
|
if not intervals:
|
||||||
|
return None
|
||||||
|
if self.selected_live_interval in intervals:
|
||||||
|
current = intervals.index(self.selected_live_interval)
|
||||||
|
else:
|
||||||
|
current = len(intervals) - 1
|
||||||
|
selected = intervals[max(0, min(len(intervals) - 1, current + offset))]
|
||||||
|
self._pin_live_interval(selected)
|
||||||
|
return selected
|
||||||
|
|
||||||
|
def inspect_live(
|
||||||
|
self,
|
||||||
|
intervals: Sequence[IntervalIdentity],
|
||||||
|
interval: IntervalIdentity,
|
||||||
|
) -> None:
|
||||||
|
"""Pin the interval chosen by mouse inspection."""
|
||||||
|
if interval in intervals:
|
||||||
|
self._pin_live_interval(interval)
|
||||||
|
|
||||||
|
def selected_live_index(self, intervals: Sequence[IntervalIdentity]) -> int:
|
||||||
|
"""Return the selected point's current render index, or -1."""
|
||||||
|
if self.selected_live_interval is None:
|
||||||
|
return -1
|
||||||
|
try:
|
||||||
|
return intervals.index(self.selected_live_interval)
|
||||||
|
except ValueError:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
def _pin_live_interval(self, interval: IntervalIdentity) -> None:
|
||||||
|
self.selected_live_interval = interval
|
||||||
|
self.following_live = False
|
||||||
|
self.live_interval_expired = False
|
||||||
+14
-10
@@ -27,7 +27,6 @@ if VENV_DIR.exists():
|
|||||||
if site_packages:
|
if site_packages:
|
||||||
sys.path.insert(0, str(site_packages))
|
sys.path.insert(0, str(site_packages))
|
||||||
|
|
||||||
from fenris.store import init_store, get_store_path
|
|
||||||
from fenris.collector import run_collection
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
|
||||||
@@ -103,14 +102,6 @@ def main() -> None:
|
|||||||
config = load_config()
|
config = load_config()
|
||||||
device = config["device"]
|
device = config["device"]
|
||||||
|
|
||||||
# Interrogate the drive
|
|
||||||
smartctl_data = interrogate_drive(device)
|
|
||||||
|
|
||||||
# Find sysfs path
|
|
||||||
sysfs_path = find_nvme_sysfs()
|
|
||||||
if sysfs_path is None:
|
|
||||||
raise RuntimeError("No NVMe controller found in sysfs")
|
|
||||||
|
|
||||||
# Inject a simple clock
|
# Inject a simple clock
|
||||||
class SimpleClock:
|
class SimpleClock:
|
||||||
def utcnow(self):
|
def utcnow(self):
|
||||||
@@ -118,11 +109,24 @@ def main() -> None:
|
|||||||
|
|
||||||
clock = SimpleClock()
|
clock = SimpleClock()
|
||||||
|
|
||||||
|
def acquire():
|
||||||
|
"""Interrogate the drive after pending-work preflight."""
|
||||||
|
smartctl_data = interrogate_drive(device)
|
||||||
|
sysfs_path = find_nvme_sysfs()
|
||||||
|
if sysfs_path is None:
|
||||||
|
raise RuntimeError("No NVMe controller found in sysfs")
|
||||||
|
return smartctl_data, sysfs_path
|
||||||
|
|
||||||
# Run collection
|
# Run collection
|
||||||
result = run_collection(smartctl_data, sysfs_path, config, clock)
|
result = run_collection(config=config, clock=clock, acquire=acquire)
|
||||||
|
|
||||||
if result["ok"]:
|
if result["ok"]:
|
||||||
print(f"Collection successful: {result['sample_count']} sample(s)")
|
print(f"Collection successful: {result['sample_count']} sample(s)")
|
||||||
|
if result.get("retention_error"):
|
||||||
|
print(
|
||||||
|
f"Retention deferred: {result['retention_error']}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
else:
|
else:
|
||||||
print(f"Collection failed: {result['error']}", file=sys.stderr)
|
print(f"Collection failed: {result['error']}", file=sys.stderr)
|
||||||
|
|||||||
+304
-101
@@ -5,19 +5,21 @@ This module implements the thinnest complete write path:
|
|||||||
- Acquire controller identity from sysfs
|
- Acquire controller identity from sysfs
|
||||||
- Normalize identity exactly once at write time
|
- Normalize identity exactly once at write time
|
||||||
- Validate every row against store invariants
|
- Validate every row against store invariants
|
||||||
- Commit one well-formed sample
|
- Stage valid observations privately before derivation
|
||||||
|
- Publish the sample and derived evidence in one collection-owned transaction
|
||||||
|
|
||||||
No code path outside the collector interrogates the device.
|
No code path outside the collector interrogates the device.
|
||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
from collections.abc import Callable
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, Optional, Tuple
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
from .store import init_store, get_store_path
|
from .derive import derive_hours_from_interval, find_previous_sample
|
||||||
from .monitoring_periods import ensure_period_open
|
from .monitoring_periods import ensure_period_open
|
||||||
from .derive import find_previous_sample, derive_hours_from_interval
|
from .store import init_store, get_store_path
|
||||||
|
|
||||||
|
|
||||||
class AcquisitionError(Exception):
|
class AcquisitionError(Exception):
|
||||||
@@ -30,6 +32,29 @@ class InvariantViolationError(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _PendingDerivationError(Exception):
|
||||||
|
"""A valid staged observation failed while deriving dependent evidence."""
|
||||||
|
|
||||||
|
def __init__(self, cause: Exception):
|
||||||
|
super().__init__(str(cause))
|
||||||
|
self.cause = cause
|
||||||
|
|
||||||
|
|
||||||
|
class _PendingRecoveryFailure(Exception):
|
||||||
|
"""A publication attempt failed after earlier pending rows committed."""
|
||||||
|
|
||||||
|
def __init__(self, error: Exception, published_count: int):
|
||||||
|
derivation_error = isinstance(error, _PendingDerivationError)
|
||||||
|
cause = error.cause if derivation_error else error
|
||||||
|
super().__init__(str(cause))
|
||||||
|
self.cause = cause
|
||||||
|
self.published_count = published_count
|
||||||
|
self.derivation_error = derivation_error
|
||||||
|
|
||||||
|
|
||||||
|
PENDING_PUBLICATION_LIMIT = 6720
|
||||||
|
|
||||||
|
|
||||||
def acquire_from_smartctl(smartctl_data: Dict[str, Any]) -> Dict[str, Any]:
|
def acquire_from_smartctl(smartctl_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
"""Acquire counters and thermal evidence from smartctl -a -j data.
|
"""Acquire counters and thermal evidence from smartctl -a -j data.
|
||||||
|
|
||||||
@@ -195,12 +220,12 @@ def write_sample(
|
|||||||
sample: Dict[str, Any],
|
sample: Dict[str, Any],
|
||||||
identity: Dict[str, Any],
|
identity: Dict[str, Any],
|
||||||
conn: sqlite3.Connection,
|
conn: sqlite3.Connection,
|
||||||
clock,
|
observed_at: datetime,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Write one sample to the observation store.
|
"""Write one sample to the observation store.
|
||||||
|
|
||||||
Identity normalization happens exactly once here.
|
Identity normalization happens exactly once here.
|
||||||
Returns segment info for the caller.
|
Returns segment info for the caller. Caller owns the transaction.
|
||||||
"""
|
"""
|
||||||
from .segment import find_current_segment, should_open_new_segment, open_segment
|
from .segment import find_current_segment, should_open_new_segment, open_segment
|
||||||
|
|
||||||
@@ -219,8 +244,7 @@ def write_sample(
|
|||||||
# Open new segment if needed
|
# Open new segment if needed
|
||||||
segment_opened = False
|
segment_opened = False
|
||||||
if should_open:
|
if should_open:
|
||||||
now = clock.utcnow()
|
open_segment(conn, observed_at, identity, identity_key, identity_degraded)
|
||||||
open_segment(conn, now, identity, identity_key, identity_degraded)
|
|
||||||
segment_opened = True
|
segment_opened = True
|
||||||
|
|
||||||
# Get current segment_id for provenance
|
# Get current segment_id for provenance
|
||||||
@@ -235,8 +259,8 @@ def write_sample(
|
|||||||
percentage_used, available_spare, media_errors, power_on_hours,
|
percentage_used, available_spare, media_errors, power_on_hours,
|
||||||
power_cycles, unsafe_shutdowns, temperature_c,
|
power_cycles, unsafe_shutdowns, temperature_c,
|
||||||
data_units_written, data_units_read, bytes_written, bytes_read,
|
data_units_written, data_units_read, bytes_written, bytes_read,
|
||||||
critical_warning, segment_id
|
critical_warning, segment_id, local_tz
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
sample["ts"],
|
sample["ts"],
|
||||||
@@ -259,122 +283,301 @@ def write_sample(
|
|||||||
sample["bytes_read"],
|
sample["bytes_read"],
|
||||||
sample["critical_warning"],
|
sample["critical_warning"],
|
||||||
segment_id,
|
segment_id,
|
||||||
|
sample.get("local_tz"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"segment_opened": segment_opened,
|
"segment_opened": segment_opened,
|
||||||
"segment_reason": reason,
|
"segment_reason": reason,
|
||||||
"identity_key": identity_key,
|
"identity_key": identity_key,
|
||||||
"identity_degraded": identity_degraded,
|
"identity_degraded": identity_degraded,
|
||||||
"segment_id": segment_id,
|
"segment_id": segment_id,
|
||||||
|
"sample_id": cursor.lastrowid,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def run_collection(
|
def _observation_time(sample: Dict[str, Any]) -> datetime:
|
||||||
smartctl_data: Dict[str, Any],
|
"""Read an observation's original timestamp for ordered recovery."""
|
||||||
sysfs_path: Path,
|
observed_at = datetime.fromisoformat(sample["ts"])
|
||||||
config: Dict[str, Any],
|
if observed_at.tzinfo is None:
|
||||||
clock,
|
return observed_at.replace(tzinfo=timezone.utc)
|
||||||
) -> Dict[str, Any]:
|
return observed_at.astimezone(timezone.utc)
|
||||||
"""Run one collection run.
|
|
||||||
|
|
||||||
This is the main entry point for the collector.
|
def _publish_observation(
|
||||||
Returns the run outcome.
|
conn: sqlite3.Connection,
|
||||||
"""
|
sample: Dict[str, Any],
|
||||||
|
identity: Dict[str, Any],
|
||||||
|
tz_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""Publish one staged sample and all dependent evidence in caller transaction."""
|
||||||
|
observed_at = _observation_time(sample)
|
||||||
|
sample["local_tz"] = tz_name
|
||||||
|
validate_sample_invariants(sample, conn)
|
||||||
|
ensure_period_open(conn, observed_at)
|
||||||
|
seg_info = write_sample(sample, identity, conn, observed_at)
|
||||||
|
|
||||||
|
current_id = seg_info["sample_id"]
|
||||||
|
prev = find_previous_sample(conn, seg_info.get("segment_id"), current_id)
|
||||||
try:
|
try:
|
||||||
# Acquire counters and thermal evidence
|
if prev is not None:
|
||||||
counters = acquire_from_smartctl(smartctl_data)
|
current = {
|
||||||
|
"id": current_id,
|
||||||
# Acquire controller identity
|
"ts": sample["ts"],
|
||||||
identity = acquire_from_sysfs(sysfs_path)
|
"bytes_written": sample["bytes_written"],
|
||||||
|
"bytes_read": sample["bytes_read"],
|
||||||
# Build sample with injected clock
|
"power_on_hours": sample["power_on_hours"],
|
||||||
sample = {
|
"temperature_c": sample["temperature_c"],
|
||||||
"ts": clock.utcnow().isoformat(),
|
"data_units_written": sample["data_units_written"],
|
||||||
"device": config["device"],
|
"data_units_read": sample["data_units_read"],
|
||||||
**counters,
|
"local_tz": tz_name,
|
||||||
**identity,
|
}
|
||||||
}
|
derive_hours_from_interval(conn, prev, current)
|
||||||
|
from .local_day import record_local_activity_interval
|
||||||
# Initialize store if needed
|
record_local_activity_interval(
|
||||||
|
conn,
|
||||||
|
prev,
|
||||||
|
current,
|
||||||
|
start_sample_id=prev["id"],
|
||||||
|
end_sample_id=current_id,
|
||||||
|
segment_id=seg_info.get("segment_id"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
previous_any_segment = find_previous_sample(conn, None, current_id)
|
||||||
|
if previous_any_segment is not None:
|
||||||
|
from .local_day import mark_local_activity_gap
|
||||||
|
mark_local_activity_gap(
|
||||||
|
conn,
|
||||||
|
sample["ts"],
|
||||||
|
tz_name,
|
||||||
|
previous_any_segment,
|
||||||
|
)
|
||||||
|
|
||||||
|
from .day_aggregate import derive_all_days, persist_day_aggregate
|
||||||
|
for aggregate in derive_all_days(conn):
|
||||||
|
persist_day_aggregate(conn, aggregate)
|
||||||
|
|
||||||
|
from .local_day import derive_local_day_summary, persist_local_day
|
||||||
|
local_summary = derive_local_day_summary(conn, tz_name, observed_at)
|
||||||
|
if local_summary is not None:
|
||||||
|
persist_local_day(conn, local_summary)
|
||||||
|
except Exception as exc: # noqa: BLE001 - preserve valid staged evidence for retry.
|
||||||
|
raise _PendingDerivationError(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _pending_count(conn: sqlite3.Connection) -> int:
|
||||||
|
return conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _stage_observation(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
sample: Dict[str, Any],
|
||||||
|
identity: Dict[str, Any],
|
||||||
|
tz_name: str,
|
||||||
|
) -> int:
|
||||||
|
"""Durably stage valid acquired evidence before attempting publication."""
|
||||||
|
payload = json.dumps(
|
||||||
|
{"sample": sample, "identity": identity, "tz_name": tz_name},
|
||||||
|
separators=(",", ":"),
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
cursor = conn.execute(
|
||||||
|
"INSERT INTO pending_publications (sample_ts, payload) VALUES (?, ?)",
|
||||||
|
(sample["ts"], payload),
|
||||||
|
)
|
||||||
|
return cursor.lastrowid
|
||||||
|
|
||||||
|
|
||||||
|
def _recover_pending(conn: sqlite3.Connection) -> int:
|
||||||
|
"""Publish pending observations oldest first; stop at first failure."""
|
||||||
|
recovered = 0
|
||||||
|
while True:
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT id, payload FROM pending_publications ORDER BY id LIMIT 1"
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
conn.rollback()
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
pending_id, payload = row
|
||||||
|
observation = json.loads(payload)
|
||||||
|
try:
|
||||||
|
_publish_observation(
|
||||||
|
conn,
|
||||||
|
observation["sample"],
|
||||||
|
observation["identity"],
|
||||||
|
observation["tz_name"],
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise _PendingRecoveryFailure(exc, recovered) from exc
|
||||||
|
conn.execute("DELETE FROM pending_publications WHERE id = ?", (pending_id,))
|
||||||
|
conn.commit()
|
||||||
|
recovered += 1
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _is_store_or_invariant_failure(exc: Exception) -> bool:
|
||||||
|
return (
|
||||||
|
isinstance(exc, sqlite3.Error)
|
||||||
|
and not isinstance(exc, sqlite3.IntegrityError)
|
||||||
|
) or isinstance(exc, InvariantViolationError)
|
||||||
|
|
||||||
|
|
||||||
|
def _pending_capacity_error(
|
||||||
|
reason: str,
|
||||||
|
*,
|
||||||
|
acquisition_skipped: bool = False,
|
||||||
|
) -> RuntimeError:
|
||||||
|
outcome = "; no new observation acquired" if acquisition_skipped else ""
|
||||||
|
return RuntimeError(
|
||||||
|
"pending publication capacity full "
|
||||||
|
f"({PENDING_PUBLICATION_LIMIT} observations){outcome}; "
|
||||||
|
f"{reason}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _recover_pending_for_collection(conn: sqlite3.Connection) -> int:
|
||||||
|
"""Retry queued work and report exhausted capacity without masking store faults."""
|
||||||
|
try:
|
||||||
|
return _recover_pending(conn)
|
||||||
|
except Exception as exc: # noqa: BLE001 - preserve pending evidence on recovery errors.
|
||||||
|
cause = exc.cause if isinstance(exc, _PendingRecoveryFailure) else exc
|
||||||
|
if _is_store_or_invariant_failure(cause):
|
||||||
|
raise cause
|
||||||
|
try:
|
||||||
|
capacity_full = _pending_count(conn) >= PENDING_PUBLICATION_LIMIT
|
||||||
|
except sqlite3.Error:
|
||||||
|
raise cause
|
||||||
|
if capacity_full:
|
||||||
|
raise _pending_capacity_error(f"recovery failed: {cause}") from cause
|
||||||
|
raise cause
|
||||||
|
|
||||||
|
|
||||||
|
def _recover_pending_for_admission(conn: sqlite3.Connection) -> int:
|
||||||
|
"""Recover in order, then admit one observation if bounded space remains."""
|
||||||
|
try:
|
||||||
|
recovered = _recover_pending(conn)
|
||||||
|
except _PendingRecoveryFailure as failure:
|
||||||
|
if (
|
||||||
|
not failure.derivation_error
|
||||||
|
or _is_store_or_invariant_failure(failure.cause)
|
||||||
|
):
|
||||||
|
raise failure.cause
|
||||||
|
try:
|
||||||
|
pending_count = _pending_count(conn)
|
||||||
|
except sqlite3.Error:
|
||||||
|
raise failure.cause
|
||||||
|
if pending_count >= PENDING_PUBLICATION_LIMIT:
|
||||||
|
raise _pending_capacity_error(
|
||||||
|
f"recovery failed: {failure.cause}",
|
||||||
|
acquisition_skipped=True,
|
||||||
|
) from failure.cause
|
||||||
|
return failure.published_count
|
||||||
|
|
||||||
|
if _pending_count(conn) >= PENDING_PUBLICATION_LIMIT:
|
||||||
|
raise _pending_capacity_error(
|
||||||
|
"recovery left the queue full",
|
||||||
|
acquisition_skipped=True,
|
||||||
|
)
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
|
def run_collection(
|
||||||
|
smartctl_data: Optional[Dict[str, Any]] = None,
|
||||||
|
sysfs_path: Optional[Path] = None,
|
||||||
|
config: Optional[Dict[str, Any]] = None,
|
||||||
|
clock=None,
|
||||||
|
*,
|
||||||
|
acquire: Optional[Callable[[], tuple[Dict[str, Any], Path]]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Recover old work, acquire one observation, then publish it atomically.
|
||||||
|
|
||||||
|
Production callers pass ``acquire`` so recovery and capacity checks run
|
||||||
|
before device interrogation. Direct sample arguments remain useful for
|
||||||
|
deterministic collector tests.
|
||||||
|
"""
|
||||||
|
conn = None
|
||||||
|
try:
|
||||||
|
if config is None or clock is None:
|
||||||
|
raise ValueError("config and clock are required")
|
||||||
|
|
||||||
store_path = get_store_path(config)
|
store_path = get_store_path(config)
|
||||||
conn = init_store(store_path)
|
conn = init_store(store_path)
|
||||||
|
|
||||||
# Run legacy import if needed (idempotent)
|
|
||||||
from .legacy import import_legacy_history
|
from .legacy import import_legacy_history
|
||||||
history_path = Path(config.get("data_dir", ".")) / "history.jsonl"
|
history_path = Path(config.get("data_dir", ".")) / "history.jsonl"
|
||||||
if history_path.exists():
|
if history_path.exists():
|
||||||
import_legacy_history(conn, history_path, clock=clock)
|
import_legacy_history(conn, history_path, clock=clock)
|
||||||
|
|
||||||
|
published_count = _recover_pending_for_admission(conn)
|
||||||
try:
|
checked_pending_count = _pending_count(conn)
|
||||||
# Ensure monitoring period is open (issue #73 AC2)
|
|
||||||
ensure_period_open(conn, clock.utcnow())
|
|
||||||
|
|
||||||
# Validate invariants
|
|
||||||
validate_sample_invariants(sample, conn)
|
|
||||||
|
|
||||||
# Write sample and get segment info
|
|
||||||
seg_info = write_sample(sample, identity, conn, clock)
|
|
||||||
|
|
||||||
# Derive hour observations from interval with previous sample
|
|
||||||
try:
|
|
||||||
# Find the sample we just wrote
|
|
||||||
cursor = conn.execute("SELECT id FROM samples ORDER BY id DESC LIMIT 1")
|
|
||||||
current_id = cursor.fetchone()[0]
|
|
||||||
|
|
||||||
prev = find_previous_sample(conn, seg_info.get("segment_id"), current_id)
|
# Hold the writer reservation across the capacity check and acquisition.
|
||||||
if prev is not None:
|
# Recheck recovery if another collector appended work during preflight.
|
||||||
# Build current sample dict for derivation
|
while True:
|
||||||
current = {
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
"id": current_id,
|
waiting = _pending_count(conn)
|
||||||
"ts": sample["ts"],
|
if (
|
||||||
"bytes_written": sample["bytes_written"],
|
waiting >= PENDING_PUBLICATION_LIMIT
|
||||||
"bytes_read": sample["bytes_read"],
|
or waiting > checked_pending_count
|
||||||
"power_on_hours": sample["power_on_hours"],
|
):
|
||||||
"temperature_c": sample["temperature_c"],
|
conn.rollback()
|
||||||
"data_units_written": sample["data_units_written"],
|
published_count += _recover_pending_for_admission(conn)
|
||||||
"data_units_read": sample["data_units_read"],
|
checked_pending_count = _pending_count(conn)
|
||||||
}
|
continue
|
||||||
derive_hours_from_interval(conn, prev, current)
|
|
||||||
|
|
||||||
# Rebuild day aggregates from hour observations
|
from .tz_util import detect_system_tz
|
||||||
from .day_aggregate import derive_all_days, persist_day_aggregate
|
tz_name = detect_system_tz()
|
||||||
for agg in derive_all_days(conn):
|
if acquire is not None:
|
||||||
persist_day_aggregate(conn, agg)
|
smartctl_data, sysfs_path = acquire()
|
||||||
|
if smartctl_data is None or sysfs_path is None:
|
||||||
|
raise AcquisitionError("No acquired SMART data or sysfs identity")
|
||||||
|
|
||||||
# Derive local-day summary using system timezone (issue #90)
|
counters = acquire_from_smartctl(smartctl_data)
|
||||||
try:
|
identity = acquire_from_sysfs(sysfs_path)
|
||||||
from .tz_util import detect_system_tz
|
sample = {
|
||||||
from .local_day import derive_local_day_summary, persist_local_day
|
"ts": clock.utcnow().isoformat(),
|
||||||
tz_name = detect_system_tz()
|
"device": config["device"],
|
||||||
clock_now = clock.utcnow()
|
**counters,
|
||||||
local_summary = derive_local_day_summary(conn, tz_name, clock_now)
|
**identity,
|
||||||
if local_summary is not None:
|
|
||||||
persist_local_day(conn, local_summary)
|
|
||||||
except Exception:
|
|
||||||
# Local-day derivation failure must not prevent publication
|
|
||||||
pass
|
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
except Exception:
|
|
||||||
# Derivation failure must not prevent sample persistence (issue #73 AC6)
|
|
||||||
pass
|
|
||||||
|
|
||||||
return {
|
|
||||||
"ok": True,
|
|
||||||
"sample_count": 1,
|
|
||||||
"store_path": str(store_path),
|
|
||||||
}
|
}
|
||||||
finally:
|
validate_sample_invariants(sample, conn)
|
||||||
conn.close()
|
_stage_observation(conn, sample, identity, tz_name)
|
||||||
|
conn.commit()
|
||||||
except (AcquisitionError, InvariantViolationError) as e:
|
break
|
||||||
|
|
||||||
|
published_count += _recover_pending_for_collection(conn)
|
||||||
|
retention_error = None
|
||||||
|
try:
|
||||||
|
if _pending_count(conn) == 0:
|
||||||
|
from .pruning import prune_old_samples
|
||||||
|
|
||||||
|
prune_old_samples(conn, clock.utcnow())
|
||||||
|
except Exception as exc: # noqa: BLE001 - publication already committed.
|
||||||
|
# Publication already committed. Keep collection successful and
|
||||||
|
# retry atomic retention on the next normal collection.
|
||||||
|
conn.rollback()
|
||||||
|
retention_error = str(exc)
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"sample_count": published_count,
|
||||||
|
"retention_error": retention_error,
|
||||||
|
"store_path": str(store_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
if conn is not None:
|
||||||
|
conn.rollback()
|
||||||
return {
|
return {
|
||||||
"ok": False,
|
"ok": False,
|
||||||
"error": str(e),
|
"error": str(exc),
|
||||||
"error_type": type(e).__name__,
|
"error_type": type(exc).__name__,
|
||||||
}
|
}
|
||||||
|
finally:
|
||||||
|
if conn is not None:
|
||||||
|
conn.close()
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ def find_previous_sample(
|
|||||||
if segment_id is not None:
|
if segment_id is not None:
|
||||||
cursor = conn.execute(
|
cursor = conn.execute(
|
||||||
"SELECT id, ts, bytes_written, bytes_read, power_on_hours, "
|
"SELECT id, ts, bytes_written, bytes_read, power_on_hours, "
|
||||||
" temperature_c, data_units_written, data_units_read "
|
" temperature_c, data_units_written, data_units_read, local_tz "
|
||||||
"FROM samples WHERE id < ? AND segment_id = ? "
|
"FROM samples WHERE id < ? AND segment_id = ? "
|
||||||
"ORDER BY id DESC LIMIT 1",
|
"ORDER BY id DESC LIMIT 1",
|
||||||
(current_sample_id, segment_id),
|
(current_sample_id, segment_id),
|
||||||
@@ -37,7 +37,7 @@ def find_previous_sample(
|
|||||||
else:
|
else:
|
||||||
cursor = conn.execute(
|
cursor = conn.execute(
|
||||||
"SELECT id, ts, bytes_written, bytes_read, power_on_hours, "
|
"SELECT id, ts, bytes_written, bytes_read, power_on_hours, "
|
||||||
" temperature_c, data_units_written, data_units_read "
|
" temperature_c, data_units_written, data_units_read, local_tz "
|
||||||
"FROM samples WHERE id < ? "
|
"FROM samples WHERE id < ? "
|
||||||
"ORDER BY id DESC LIMIT 1",
|
"ORDER BY id DESC LIMIT 1",
|
||||||
(current_sample_id,),
|
(current_sample_id,),
|
||||||
@@ -49,7 +49,7 @@ def find_previous_sample(
|
|||||||
"id": row[0], "ts": row[1], "bytes_written": row[2],
|
"id": row[0], "ts": row[1], "bytes_written": row[2],
|
||||||
"bytes_read": row[3], "power_on_hours": row[4],
|
"bytes_read": row[3], "power_on_hours": row[4],
|
||||||
"temperature_c": row[5], "data_units_written": row[6],
|
"temperature_c": row[5], "data_units_written": row[6],
|
||||||
"data_units_read": row[7],
|
"data_units_read": row[7], "local_tz": row[8],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -96,6 +96,7 @@ def derive_hours_from_interval(
|
|||||||
"""Derive hour observations from a sample pair interval.
|
"""Derive hour observations from a sample pair interval.
|
||||||
|
|
||||||
Returns list of hour observation dicts that were written/updated.
|
Returns list of hour observation dicts that were written/updated.
|
||||||
|
Caller owns the transaction.
|
||||||
"""
|
"""
|
||||||
prev_ts = _parse_ts(prev_sample["ts"])
|
prev_ts = _parse_ts(prev_sample["ts"])
|
||||||
next_ts = _parse_ts(next_sample["ts"])
|
next_ts = _parse_ts(next_sample["ts"])
|
||||||
@@ -207,9 +208,6 @@ def _upsert_hour_observation(
|
|||||||
"WHERE id = ?",
|
"WHERE id = ?",
|
||||||
(new_bw, new_br, new_samples, existing[0]),
|
(new_bw, new_br, new_samples, existing[0]),
|
||||||
)
|
)
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def _add_unattributed_bytes(
|
def _add_unattributed_bytes(
|
||||||
conn: sqlite3.Connection,
|
conn: sqlite3.Connection,
|
||||||
prev_ts: datetime,
|
prev_ts: datetime,
|
||||||
@@ -241,4 +239,3 @@ def _add_unattributed_bytes(
|
|||||||
"unattributed_bytes_read = unattributed_bytes_read + ? WHERE day = ?",
|
"unattributed_bytes_read = unattributed_bytes_read + ? WHERE day = ?",
|
||||||
(bw_delta, br_delta, day),
|
(bw_delta, br_delta, day),
|
||||||
)
|
)
|
||||||
conn.commit()
|
|
||||||
|
|||||||
+1011
-132
File diff suppressed because it is too large
Load Diff
@@ -69,6 +69,7 @@ def cmd_enable(args: argparse.Namespace) -> None:
|
|||||||
open_period = get_open_period(conn)
|
open_period = get_open_period(conn)
|
||||||
if open_period is None:
|
if open_period is None:
|
||||||
ensure_period_open(conn, now)
|
ensure_period_open(conn, now)
|
||||||
|
conn.commit()
|
||||||
print("Monitoring period opened at", now.isoformat())
|
print("Monitoring period opened at", now.isoformat())
|
||||||
else:
|
else:
|
||||||
print("Monitoring period already open (id=%d)" % open_period["id"])
|
print("Monitoring period already open (id=%d)" % open_period["id"])
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ def ensure_period_open(conn: sqlite3.Connection, run_time: datetime) -> None:
|
|||||||
"""Ensure a monitoring period is open. If none exists, open one at run_time.
|
"""Ensure a monitoring period is open. If none exists, open one at run_time.
|
||||||
|
|
||||||
Spec §9.8: A collection run finding no open monitoring period opens one
|
Spec §9.8: A collection run finding no open monitoring period opens one
|
||||||
at the run moment, never backdated.
|
at the run moment, never backdated. Caller owns the transaction.
|
||||||
"""
|
"""
|
||||||
if get_open_period(conn) is not None:
|
if get_open_period(conn) is not None:
|
||||||
return # Already open — no-op
|
return # Already open — no-op
|
||||||
@@ -26,7 +26,6 @@ def ensure_period_open(conn: sqlite3.Connection, run_time: datetime) -> None:
|
|||||||
"INSERT INTO monitoring_periods (started_at) VALUES (?)",
|
"INSERT INTO monitoring_periods (started_at) VALUES (?)",
|
||||||
(ts,),
|
(ts,),
|
||||||
)
|
)
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def close_period(
|
def close_period(
|
||||||
|
|||||||
@@ -187,7 +187,7 @@ def _wall_clock_in_range(conn, start, end):
|
|||||||
|
|
||||||
def _resolve_baseline(conn, current_segment):
|
def _resolve_baseline(conn, current_segment):
|
||||||
baseline = _get_baseline(conn)
|
baseline = _get_baseline(conn)
|
||||||
facts = []
|
facts: list[str] = []
|
||||||
if baseline is None:
|
if baseline is None:
|
||||||
return BaselineTier.NONE, None, "no baseline", facts
|
return BaselineTier.NONE, None, "no baseline", facts
|
||||||
|
|
||||||
@@ -494,16 +494,16 @@ def _has_complete_local_day(conn):
|
|||||||
within a monitoring period that has usable observation evidence.
|
within a monitoring period that has usable observation evidence.
|
||||||
This is the prerequisite for showing an endurance outlook.
|
This is the prerequisite for showing an endurance outlook.
|
||||||
"""
|
"""
|
||||||
row = conn.execute(
|
return _count_complete_local_days(conn) > 0
|
||||||
"SELECT 1 FROM local_days WHERE complete = 1 LIMIT 1"
|
|
||||||
).fetchone()
|
|
||||||
return row is not None
|
|
||||||
|
|
||||||
|
|
||||||
def _count_complete_local_days(conn):
|
def _count_complete_local_days(conn):
|
||||||
"""Count the number of complete local observation days."""
|
"""Count the number of complete local observation days."""
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
"SELECT COUNT(*) FROM local_days WHERE complete = 1"
|
"SELECT COUNT(*) FROM local_days "
|
||||||
|
"WHERE complete = 1 "
|
||||||
|
"AND activity_precision IN ('measured', 'coarse') "
|
||||||
|
"AND activity_intervals > 0"
|
||||||
).fetchone()
|
).fetchone()
|
||||||
return row[0] if row else 0
|
return row[0] if row else 0
|
||||||
|
|
||||||
|
|||||||
+360
-145
@@ -1,131 +1,332 @@
|
|||||||
"""Raw sample pruning per spec §3.4, ST-5.
|
"""Expire detail only after UTC and local_days replacement evidence is durable.
|
||||||
|
|
||||||
Raw samples are pruned opportunistically to 14 days.
|
The 14-day cutoff never overrides local-day preservation, shared boundary
|
||||||
Hour observations and day aggregates are retained indefinitely.
|
evidence, publication recovery, or the newest sample's successor-anchor role.
|
||||||
Boundary anchors required for successor evidence are retained.
|
|
||||||
|
|
||||||
Local-day summaries (local_days table) are never touched by pruning.
|
|
||||||
They are persisted at collection time from hour observations and survive
|
|
||||||
raw-sample pruning because they depend on hour observations, not on raw
|
|
||||||
samples. This is the mechanism that keeps local-day history trustworthy
|
|
||||||
after detail expires (issue #93).
|
|
||||||
"""
|
"""
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
|
||||||
# Spec §3.4: Raw-sample retention
|
|
||||||
RAW_SAMPLE_RETENTION_DAYS = 14
|
RAW_SAMPLE_RETENTION_DAYS = 14
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_datetime(value: datetime) -> datetime:
|
||||||
|
"""Normalize a clock value to aware UTC."""
|
||||||
|
if value.tzinfo is None:
|
||||||
|
value = value.replace(tzinfo=timezone.utc)
|
||||||
|
return value.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_sample_time(value: str) -> datetime | None:
|
||||||
|
"""Parse old timestamps conservatively; malformed legacy values stay."""
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
return _utc_datetime(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_rows(conn: sqlite3.Connection) -> list[tuple]:
|
||||||
|
return conn.execute(
|
||||||
|
"SELECT id, ts, segment_id, bytes_written, bytes_read, local_tz "
|
||||||
|
"FROM samples ORDER BY id"
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
|
||||||
|
def _required_hour_window(
|
||||||
|
start: datetime,
|
||||||
|
end: datetime,
|
||||||
|
) -> tuple[datetime, datetime, int] | None:
|
||||||
|
"""Return first hour, exclusive end and count for [start, end)."""
|
||||||
|
if end <= start:
|
||||||
|
return None
|
||||||
|
first_hour = start.replace(minute=0, second=0, microsecond=0)
|
||||||
|
last_hour = end.replace(minute=0, second=0, microsecond=0)
|
||||||
|
try:
|
||||||
|
exclusive_end = last_hour if end == last_hour else last_hour + timedelta(hours=1)
|
||||||
|
except OverflowError:
|
||||||
|
return None
|
||||||
|
count = int((exclusive_end - first_hour).total_seconds() // 3600)
|
||||||
|
return first_hour, exclusive_end, count
|
||||||
|
|
||||||
|
|
||||||
|
def _has_utc_replacement(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
previous: tuple,
|
||||||
|
current: tuple,
|
||||||
|
start: datetime,
|
||||||
|
end: datetime,
|
||||||
|
) -> bool:
|
||||||
|
window = _required_hour_window(start, end)
|
||||||
|
if window is None:
|
||||||
|
return False
|
||||||
|
first_hour, exclusive_end, expected_hour_count = window
|
||||||
|
|
||||||
|
actual_hour_count = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM hour_observations WHERE hour >= ? AND hour < ?",
|
||||||
|
(
|
||||||
|
first_hour.strftime("%Y-%m-%dT%H:00:00+00:00"),
|
||||||
|
exclusive_end.strftime("%Y-%m-%dT%H:00:00+00:00"),
|
||||||
|
),
|
||||||
|
).fetchone()[0]
|
||||||
|
if actual_hour_count < expected_hour_count:
|
||||||
|
return False
|
||||||
|
|
||||||
|
start_written, end_written = previous[3], current[3]
|
||||||
|
start_read, end_read = previous[4], current[4]
|
||||||
|
if None in (start_written, end_written, start_read, end_read):
|
||||||
|
return False
|
||||||
|
if end_written < start_written or end_read < start_read:
|
||||||
|
return False
|
||||||
|
bytes_written = end_written - start_written
|
||||||
|
bytes_read = end_read - start_read
|
||||||
|
if expected_hour_count == 1:
|
||||||
|
totals = conn.execute(
|
||||||
|
"SELECT bytes_written_delta, bytes_read_delta FROM hour_observations "
|
||||||
|
"WHERE hour = ?",
|
||||||
|
(first_hour.strftime("%Y-%m-%dT%H:00:00+00:00"),),
|
||||||
|
).fetchone()
|
||||||
|
else:
|
||||||
|
totals = conn.execute(
|
||||||
|
"SELECT unattributed_bytes_written, unattributed_bytes_read "
|
||||||
|
"FROM day_aggregates WHERE day = ?",
|
||||||
|
(start.date().isoformat(),),
|
||||||
|
).fetchone()
|
||||||
|
if totals is None or totals[0] < bytes_written or totals[1] < bytes_read:
|
||||||
|
return False
|
||||||
|
|
||||||
|
last_day = (exclusive_end - timedelta(microseconds=1)).date()
|
||||||
|
expected_day_count = (last_day - first_hour.date()).days + 1
|
||||||
|
actual_day_count = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM day_aggregates WHERE day >= ? AND day <= ?",
|
||||||
|
(first_hour.date().isoformat(), last_day.isoformat()),
|
||||||
|
).fetchone()[0]
|
||||||
|
return actual_day_count >= expected_day_count
|
||||||
|
|
||||||
|
|
||||||
|
def _local_day_row_exists(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
local_date: str,
|
||||||
|
tz_name: str,
|
||||||
|
) -> bool:
|
||||||
|
return conn.execute(
|
||||||
|
"SELECT 1 FROM local_days WHERE local_date = ? AND tz_name = ?",
|
||||||
|
(local_date, tz_name),
|
||||||
|
).fetchone() is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _local_day_bounds(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
local_date: str,
|
||||||
|
tz_name: str,
|
||||||
|
) -> tuple[datetime, datetime] | None:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT utc_start, utc_end FROM local_days "
|
||||||
|
"WHERE local_date = ? AND tz_name = ?",
|
||||||
|
(local_date, tz_name),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
start = _parse_sample_time(row[0])
|
||||||
|
end = _parse_sample_time(row[1])
|
||||||
|
if start is None or end is None or end <= start:
|
||||||
|
return None
|
||||||
|
return start, end
|
||||||
|
|
||||||
|
|
||||||
|
def _contains_instant(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
local_date: str,
|
||||||
|
tz_name: str,
|
||||||
|
instant: datetime,
|
||||||
|
) -> bool:
|
||||||
|
bounds = _local_day_bounds(conn, local_date, tz_name)
|
||||||
|
return bounds is not None and bounds[0] <= instant < bounds[1]
|
||||||
|
|
||||||
|
|
||||||
|
def _monitoring_period_covers(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
start: datetime,
|
||||||
|
end: datetime,
|
||||||
|
) -> bool:
|
||||||
|
"""Require continuous monitoring before a local-day total replaces detail."""
|
||||||
|
for started_at, ended_at in conn.execute(
|
||||||
|
"SELECT started_at, ended_at FROM monitoring_periods"
|
||||||
|
):
|
||||||
|
period_start = _parse_sample_time(started_at)
|
||||||
|
period_end = _parse_sample_time(ended_at) if ended_at is not None else None
|
||||||
|
if period_start is None or period_start > start:
|
||||||
|
continue
|
||||||
|
if ended_at is None or (period_end is not None and end <= period_end):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _has_local_replacement(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
previous: tuple,
|
||||||
|
current: tuple,
|
||||||
|
start: datetime,
|
||||||
|
end: datetime,
|
||||||
|
) -> bool:
|
||||||
|
start_tz = previous[5]
|
||||||
|
end_tz = current[5]
|
||||||
|
if not end_tz:
|
||||||
|
return False
|
||||||
|
|
||||||
|
start_id, end_id = previous[0], current[0]
|
||||||
|
segment_id = current[2]
|
||||||
|
if (
|
||||||
|
start_tz
|
||||||
|
and start_tz == end_tz
|
||||||
|
and segment_id is not None
|
||||||
|
and _monitoring_period_covers(conn, start, end)
|
||||||
|
):
|
||||||
|
known_days = conn.execute(
|
||||||
|
"SELECT local_days.utc_start, local_days.utc_end "
|
||||||
|
"FROM local_days JOIN local_day_segment_totals "
|
||||||
|
" ON local_day_segment_totals.local_day_id = local_days.id "
|
||||||
|
"WHERE local_days.tz_name = ? "
|
||||||
|
" AND local_days.activity_precision IN ('measured', 'coarse') "
|
||||||
|
" AND local_days.last_sample_id >= ? "
|
||||||
|
" AND local_days.activity_intervals > 0 "
|
||||||
|
" AND local_day_segment_totals.segment_id = ? "
|
||||||
|
" AND local_day_segment_totals.activity_intervals > 0",
|
||||||
|
(end_tz, end_id, segment_id),
|
||||||
|
).fetchall()
|
||||||
|
for row in known_days:
|
||||||
|
bounds_start = _parse_sample_time(row[0])
|
||||||
|
bounds_end = _parse_sample_time(row[1])
|
||||||
|
if (
|
||||||
|
bounds_start is not None
|
||||||
|
and bounds_end is not None
|
||||||
|
and bounds_start <= start
|
||||||
|
and end < bounds_end
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
|
||||||
|
evidence = conn.execute(
|
||||||
|
"SELECT bytes_written, bytes_read, reason, start_local_date, end_local_date, "
|
||||||
|
" start_tz_name, end_tz_name, started_at, ended_at "
|
||||||
|
"FROM local_day_unallocated_evidence "
|
||||||
|
"WHERE start_sample_id = ? AND end_sample_id = ?",
|
||||||
|
(start_id, end_id),
|
||||||
|
).fetchone()
|
||||||
|
if evidence is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
start_written, end_written = previous[3], current[3]
|
||||||
|
start_read, end_read = previous[4], current[4]
|
||||||
|
if None in (start_written, end_written, start_read, end_read):
|
||||||
|
return False
|
||||||
|
if end_written < start_written or end_read < start_read:
|
||||||
|
return False
|
||||||
|
# The source IDs uniquely identify this interval. Confirm its durable
|
||||||
|
# deltas and retain every recorded local-day boundary it touches.
|
||||||
|
if (end_written - start_written, end_read - start_read) != evidence[:2]:
|
||||||
|
return False
|
||||||
|
if evidence[5] != start_tz or evidence[6] != end_tz:
|
||||||
|
return False
|
||||||
|
if evidence[2] not in {
|
||||||
|
"counter_discontinuity",
|
||||||
|
"legacy_timezone_unknown",
|
||||||
|
"timezone_change",
|
||||||
|
"monitoring_period",
|
||||||
|
"local_midnight",
|
||||||
|
"segment_unknown",
|
||||||
|
}:
|
||||||
|
return False
|
||||||
|
if _parse_sample_time(evidence[7]) != start or _parse_sample_time(evidence[8]) != end:
|
||||||
|
return False
|
||||||
|
|
||||||
|
affected_days: set[tuple[str, str]] = set()
|
||||||
|
if start_tz:
|
||||||
|
affected_days.add((evidence[3], start_tz))
|
||||||
|
affected_days.add((evidence[4], end_tz))
|
||||||
|
if start_tz and start_tz == end_tz and evidence[3] < evidence[4]:
|
||||||
|
day = date.fromisoformat(evidence[3]) + timedelta(days=1)
|
||||||
|
last_day = date.fromisoformat(evidence[4])
|
||||||
|
while day < last_day:
|
||||||
|
affected_days.add((day.isoformat(), end_tz))
|
||||||
|
day += timedelta(days=1)
|
||||||
|
if not all(
|
||||||
|
_local_day_row_exists(conn, local_date, zone)
|
||||||
|
for local_date, zone in affected_days
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
if start_tz and not _contains_instant(conn, evidence[3], start_tz, start):
|
||||||
|
return False
|
||||||
|
return _contains_instant(conn, evidence[4], end_tz, end)
|
||||||
|
|
||||||
|
|
||||||
|
def _interval_has_replacement(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
previous: tuple,
|
||||||
|
current: tuple,
|
||||||
|
) -> bool:
|
||||||
|
start = _parse_sample_time(previous[1])
|
||||||
|
end = _parse_sample_time(current[1])
|
||||||
|
if start is None or end is None or end <= start:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return (
|
||||||
|
_has_utc_replacement(conn, previous, current, start, end)
|
||||||
|
and _has_local_replacement(conn, previous, current, start, end)
|
||||||
|
)
|
||||||
|
except (sqlite3.Error, ValueError, OverflowError, TypeError):
|
||||||
|
# Missing or unreadable derived evidence never authorizes deletion.
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_needs_preservation(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
rows: list[tuple],
|
||||||
|
index: int,
|
||||||
|
newest_id: int,
|
||||||
|
) -> bool:
|
||||||
|
sample = rows[index]
|
||||||
|
segment_id = sample[2]
|
||||||
|
if segment_id is None or conn.execute(
|
||||||
|
"SELECT 1 FROM controller_segments WHERE id = ?",
|
||||||
|
(segment_id,),
|
||||||
|
).fetchone() is None:
|
||||||
|
return True
|
||||||
|
if sample[0] == newest_id:
|
||||||
|
# Keep newest sample as the source anchor for the next collection.
|
||||||
|
return True
|
||||||
|
|
||||||
|
pairs = []
|
||||||
|
if index > 0 and rows[index - 1][2] == segment_id:
|
||||||
|
pairs.append((rows[index - 1], sample))
|
||||||
|
if index + 1 < len(rows) and rows[index + 1][2] == segment_id:
|
||||||
|
pairs.append((sample, rows[index + 1]))
|
||||||
|
return any(
|
||||||
|
not _interval_has_replacement(conn, previous, current)
|
||||||
|
for previous, current in pairs
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def needs_boundary_anchor(
|
def needs_boundary_anchor(
|
||||||
conn: sqlite3.Connection,
|
conn: sqlite3.Connection,
|
||||||
sample_ts: str,
|
sample_ts: str,
|
||||||
now: datetime,
|
now: datetime,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Check if a sample is needed as a boundary anchor for derivation.
|
"""Return whether an old sample still carries unreplaced evidence."""
|
||||||
|
sample_time = _parse_sample_time(sample_ts)
|
||||||
A sample is a boundary anchor if:
|
cutoff = _utc_datetime(now) - timedelta(days=RAW_SAMPLE_RETENTION_DAYS)
|
||||||
1. It's older than retention_days (strictly before cutoff)
|
if sample_time is None:
|
||||||
2. It has a next sample that forms an interval spanning the retention boundary
|
return True
|
||||||
3. The interval hasn't been derived yet
|
if sample_time >= cutoff:
|
||||||
|
|
||||||
The interval spans the boundary if:
|
|
||||||
- The sample is before the cutoff, AND
|
|
||||||
- The next sample is strictly after the cutoff (or within retention)
|
|
||||||
"""
|
|
||||||
from .derive import _parse_ts
|
|
||||||
|
|
||||||
sample_dt = _parse_ts(sample_ts)
|
|
||||||
retention_cutoff = now - timedelta(days=RAW_SAMPLE_RETENTION_DAYS)
|
|
||||||
|
|
||||||
# If sample is within retention (strictly after cutoff), not an anchor
|
|
||||||
if sample_dt > retention_cutoff:
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Check if this sample has a next sample
|
rows = _sample_rows(conn)
|
||||||
cursor = conn.execute(
|
matching = [index for index, row in enumerate(rows) if row[1] == sample_ts]
|
||||||
"""SELECT ts, segment_id FROM samples WHERE ts > ? ORDER BY ts LIMIT 1""",
|
if not matching:
|
||||||
(sample_ts,),
|
return False
|
||||||
|
newest_id = rows[-1][0] if rows else -1
|
||||||
|
return any(
|
||||||
|
_sample_needs_preservation(conn, rows, index, newest_id)
|
||||||
|
for index in matching
|
||||||
)
|
)
|
||||||
next_row = cursor.fetchone()
|
|
||||||
|
|
||||||
if next_row is None:
|
|
||||||
# No next sample - this is the last sample
|
|
||||||
# It's not needed for derivation (no interval to derive)
|
|
||||||
return False
|
|
||||||
|
|
||||||
next_ts_str = next_row[0]
|
|
||||||
next_segment_id = next_row[1]
|
|
||||||
next_dt = _parse_ts(next_ts_str)
|
|
||||||
|
|
||||||
# Check if the next sample is strictly after the cutoff (i.e., interval spans boundary)
|
|
||||||
if next_dt > retention_cutoff:
|
|
||||||
# The interval spans the retention boundary
|
|
||||||
# Check if the interval needs derivation
|
|
||||||
# Get current sample's segment_id
|
|
||||||
cursor = conn.execute(
|
|
||||||
"SELECT segment_id FROM samples WHERE ts = ?",
|
|
||||||
(sample_ts,),
|
|
||||||
)
|
|
||||||
current_segment_row = cursor.fetchone()
|
|
||||||
current_segment_id = current_segment_row[0] if current_segment_row else None
|
|
||||||
|
|
||||||
# If different segments, no interval to derive
|
|
||||||
if current_segment_id != next_segment_id:
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Check if the interval [sample_ts, next_ts] needs derivation
|
|
||||||
# It needs derivation if any hour in the span lacks an observation
|
|
||||||
current_hour = sample_dt.replace(minute=0, second=0, microsecond=0)
|
|
||||||
end_hour = next_dt.replace(minute=0, second=0, microsecond=0)
|
|
||||||
|
|
||||||
while current_hour <= end_hour:
|
|
||||||
cursor = conn.execute(
|
|
||||||
"SELECT id FROM hour_observations WHERE hour = ?",
|
|
||||||
(current_hour.isoformat(),),
|
|
||||||
)
|
|
||||||
if cursor.fetchone() is None:
|
|
||||||
# This hour lacks an observation - interval needs derivation
|
|
||||||
return True
|
|
||||||
current_hour += timedelta(hours=1)
|
|
||||||
|
|
||||||
# All hours in the span have observations - interval is derived
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
# The interval doesn't span the boundary (both samples are old)
|
|
||||||
# Check if the interval needs derivation
|
|
||||||
# Get current sample's segment_id
|
|
||||||
cursor = conn.execute(
|
|
||||||
"SELECT segment_id FROM samples WHERE ts = ?",
|
|
||||||
(sample_ts,),
|
|
||||||
)
|
|
||||||
current_segment_row = cursor.fetchone()
|
|
||||||
current_segment_id = current_segment_row[0] if current_segment_row else None
|
|
||||||
|
|
||||||
# If different segments, no interval to derive
|
|
||||||
if current_segment_id != next_segment_id:
|
|
||||||
return False
|
|
||||||
|
|
||||||
# Check if the interval [sample_ts, next_ts] needs derivation
|
|
||||||
current_hour = sample_dt.replace(minute=0, second=0, microsecond=0)
|
|
||||||
end_hour = next_dt.replace(minute=0, second=0, microsecond=0)
|
|
||||||
|
|
||||||
while current_hour <= end_hour:
|
|
||||||
cursor = conn.execute(
|
|
||||||
"SELECT id FROM hour_observations WHERE hour = ?",
|
|
||||||
(current_hour.isoformat(),),
|
|
||||||
)
|
|
||||||
if cursor.fetchone() is None:
|
|
||||||
# This hour lacks an observation - interval needs derivation
|
|
||||||
# But only keep if the interval is significant (spans multiple hours)
|
|
||||||
# or if the next sample is the last sample before a gap
|
|
||||||
gap = (next_dt - sample_dt).total_seconds()
|
|
||||||
if gap > 24 * 3600: # Significant gap (> 24 hours)
|
|
||||||
return True
|
|
||||||
current_hour += timedelta(hours=1)
|
|
||||||
|
|
||||||
# All hours in the span have observations or gap is not significant
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def prune_old_samples(
|
def prune_old_samples(
|
||||||
@@ -133,36 +334,50 @@ def prune_old_samples(
|
|||||||
now: datetime,
|
now: datetime,
|
||||||
retention_days: int = RAW_SAMPLE_RETENTION_DAYS,
|
retention_days: int = RAW_SAMPLE_RETENTION_DAYS,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Remove raw samples older than retention_days.
|
"""Delete expired samples only after every dependent fact is durable.
|
||||||
|
|
||||||
Retains boundary anchors required for successor evidence.
|
Pending publications are stored separately and never selected here. A
|
||||||
|
sample stays when it is the newest successor anchor, has an unpublishable
|
||||||
Args:
|
neighbour interval, lacks UTC or local-day replacement evidence, or has a
|
||||||
conn: Connection to the observation store.
|
timestamp that cannot be safely interpreted.
|
||||||
now: Current UTC time.
|
|
||||||
retention_days: Number of days to retain (default 14).
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Number of samples removed.
|
|
||||||
"""
|
"""
|
||||||
cutoff = (now - timedelta(days=retention_days)).isoformat()
|
cutoff = _utc_datetime(now) - timedelta(days=retention_days)
|
||||||
|
owns_transaction = not conn.in_transaction
|
||||||
# Get all samples older than cutoff
|
savepoint = "fenris_sample_retention"
|
||||||
cursor = conn.execute(
|
if owns_transaction:
|
||||||
"SELECT id, ts FROM samples WHERE ts < ? ORDER BY ts",
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
(cutoff,),
|
else:
|
||||||
)
|
conn.execute(f"SAVEPOINT {savepoint}")
|
||||||
old_samples = cursor.fetchall()
|
|
||||||
|
try:
|
||||||
removed = 0
|
rows = _sample_rows(conn)
|
||||||
for sample_id, sample_ts in old_samples:
|
if not rows:
|
||||||
# Check if this sample is a boundary anchor
|
if owns_transaction:
|
||||||
if needs_boundary_anchor(conn, sample_ts, now):
|
conn.commit()
|
||||||
continue # Skip - it's a boundary anchor
|
else:
|
||||||
|
conn.execute(f"RELEASE SAVEPOINT {savepoint}")
|
||||||
# Remove the sample
|
return 0
|
||||||
conn.execute("DELETE FROM samples WHERE id = ?", (sample_id,))
|
|
||||||
removed += 1
|
newest_id = rows[-1][0]
|
||||||
|
expired = []
|
||||||
conn.commit()
|
for index, row in enumerate(rows):
|
||||||
return removed
|
sample_time = _parse_sample_time(row[1])
|
||||||
|
if sample_time is None or sample_time >= cutoff:
|
||||||
|
continue
|
||||||
|
if _sample_needs_preservation(conn, rows, index, newest_id):
|
||||||
|
continue
|
||||||
|
expired.append(row[0])
|
||||||
|
|
||||||
|
conn.executemany("DELETE FROM samples WHERE id = ?", ((sample_id,) for sample_id in expired))
|
||||||
|
if owns_transaction:
|
||||||
|
conn.commit()
|
||||||
|
else:
|
||||||
|
conn.execute(f"RELEASE SAVEPOINT {savepoint}")
|
||||||
|
return len(expired)
|
||||||
|
except Exception:
|
||||||
|
if owns_transaction:
|
||||||
|
conn.rollback()
|
||||||
|
else:
|
||||||
|
conn.execute(f"ROLLBACK TO SAVEPOINT {savepoint}")
|
||||||
|
conn.execute(f"RELEASE SAVEPOINT {savepoint}")
|
||||||
|
raise
|
||||||
|
|||||||
+26
-26
@@ -1,8 +1,9 @@
|
|||||||
"""Historical repair and retention (issue #74).
|
"""Historical repair and retention (issue #74, #99).
|
||||||
|
|
||||||
Re-derives hour observations and day aggregates from surviving raw samples,
|
Re-derives hour observations and day aggregates from surviving raw samples,
|
||||||
with idempotent and interruption-safe guarantees. Preserves import markers,
|
rebuilds legacy local-day evidence from surviving samples or trustworthy UTC
|
||||||
boundary anchors, and valid historical summaries.
|
hours, and preserves import markers, boundary anchors, and valid summaries.
|
||||||
|
Repair is idempotent and interruption-safe.
|
||||||
|
|
||||||
Contracts:
|
Contracts:
|
||||||
- Idempotent: running repair multiple times produces no duplicates
|
- Idempotent: running repair multiple times produces no duplicates
|
||||||
@@ -71,7 +72,6 @@ def _set_repair_in_progress(conn: sqlite3.Connection, in_progress: bool) -> None
|
|||||||
"INSERT OR REPLACE INTO store_metadata (key, value) VALUES (?, ?)",
|
"INSERT OR REPLACE INTO store_metadata (key, value) VALUES (?, ?)",
|
||||||
("repair_in_progress", "true" if in_progress else "false"),
|
("repair_in_progress", "true" if in_progress else "false"),
|
||||||
)
|
)
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def _update_repair_status(conn: sqlite3.Connection, result: RepairResult) -> None:
|
def _update_repair_status(conn: sqlite3.Connection, result: RepairResult) -> None:
|
||||||
@@ -100,8 +100,6 @@ def _update_repair_status(conn: sqlite3.Connection, result: RepairResult) -> Non
|
|||||||
"INSERT OR REPLACE INTO store_metadata (key, value) VALUES (?, ?)",
|
"INSERT OR REPLACE INTO store_metadata (key, value) VALUES (?, ?)",
|
||||||
("days_derived", str(days)),
|
("days_derived", str(days)),
|
||||||
)
|
)
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def get_repair_status(conn: sqlite3.Connection) -> RepairStatus:
|
def get_repair_status(conn: sqlite3.Connection) -> RepairStatus:
|
||||||
@@ -364,20 +362,16 @@ def repair_derivation(
|
|||||||
elif hasattr(clock, 'utcnow'):
|
elif hasattr(clock, 'utcnow'):
|
||||||
clock = clock.utcnow()
|
clock = clock.utcnow()
|
||||||
|
|
||||||
# Check if repair is already in progress
|
|
||||||
if is_repair_in_progress(conn):
|
|
||||||
return RepairResult(
|
|
||||||
ok=False,
|
|
||||||
error="Repair already in progress",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Mark repair as in progress
|
|
||||||
_set_repair_in_progress(conn, True)
|
|
||||||
|
|
||||||
result = RepairResult(ok=True)
|
result = RepairResult(ok=True)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Begin transaction
|
# A committed in-progress marker may be stale after process death.
|
||||||
|
# SQLite's write lock serializes active repairs; retry idempotently.
|
||||||
|
if conn.in_transaction:
|
||||||
|
conn.commit()
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
_set_repair_in_progress(conn, True)
|
||||||
|
conn.commit()
|
||||||
conn.execute("BEGIN IMMEDIATE")
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
|
||||||
# 1. Find and derive intervals from sample pairs
|
# 1. Find and derive intervals from sample pairs
|
||||||
@@ -407,7 +401,12 @@ def repair_derivation(
|
|||||||
result.days_created += 1
|
result.days_created += 1
|
||||||
else:
|
else:
|
||||||
result.days_updated += 1
|
result.days_updated += 1
|
||||||
|
|
||||||
|
# Rebuild only legacy or unavailable local-day activity. This shares
|
||||||
|
# the migration derivation and leaves trustworthy published totals intact.
|
||||||
|
from .local_day import repair_legacy_local_day_evidence
|
||||||
|
repair_legacy_local_day_evidence(conn)
|
||||||
|
|
||||||
# 3. Count boundary anchors retained
|
# 3. Count boundary anchors retained
|
||||||
now = clock if isinstance(clock, datetime) else datetime.now(timezone.utc)
|
now = clock if isinstance(clock, datetime) else datetime.now(timezone.utc)
|
||||||
cursor = conn.execute("SELECT ts FROM samples ORDER BY ts")
|
cursor = conn.execute("SELECT ts FROM samples ORDER BY ts")
|
||||||
@@ -428,21 +427,22 @@ def repair_derivation(
|
|||||||
)
|
)
|
||||||
result.legacy_summaries_preserved = cursor.fetchone()[0]
|
result.legacy_summaries_preserved = cursor.fetchone()[0]
|
||||||
|
|
||||||
# Commit transaction
|
# Publish derived rows and completion status together. A process
|
||||||
conn.commit()
|
# interruption rolls back the in-progress marker with the repair.
|
||||||
|
|
||||||
# Update repair status
|
|
||||||
_update_repair_status(conn, result)
|
_update_repair_status(conn, result)
|
||||||
|
_set_repair_in_progress(conn, False)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
conn.rollback()
|
conn.rollback()
|
||||||
|
try:
|
||||||
|
_set_repair_in_progress(conn, False)
|
||||||
|
conn.commit()
|
||||||
|
except sqlite3.Error:
|
||||||
|
conn.rollback()
|
||||||
logger.error("Repair failed: %s", e)
|
logger.error("Repair failed: %s", e)
|
||||||
return RepairResult(
|
return RepairResult(
|
||||||
ok=False,
|
ok=False,
|
||||||
error=str(e),
|
error=str(e),
|
||||||
)
|
)
|
||||||
finally:
|
|
||||||
# Mark repair as complete
|
|
||||||
_set_repair_in_progress(conn, False)
|
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -115,6 +115,7 @@ class StatusComposition:
|
|||||||
# Sample counts for waiting explanations
|
# Sample counts for waiting explanations
|
||||||
sample_count: int = 0
|
sample_count: int = 0
|
||||||
day_count: int = 0
|
day_count: int = 0
|
||||||
|
pending_publication_count: int = 0
|
||||||
|
|
||||||
# Whether the status dot should blink (only Monitoring)
|
# Whether the status dot should blink (only Monitoring)
|
||||||
should_blink: bool = False
|
should_blink: bool = False
|
||||||
@@ -286,6 +287,7 @@ def compose_status(
|
|||||||
freshness_age_s = None
|
freshness_age_s = None
|
||||||
sample_count = 0
|
sample_count = 0
|
||||||
day_count = 0
|
day_count = 0
|
||||||
|
pending_publication_count = 0
|
||||||
deliberately_paused = False
|
deliberately_paused = False
|
||||||
|
|
||||||
if conn is None and store_fault is None and newer_schema is None:
|
if conn is None and store_fault is None and newer_schema is None:
|
||||||
@@ -315,6 +317,10 @@ def compose_status(
|
|||||||
sample_count = cursor.fetchone()[0]
|
sample_count = cursor.fetchone()[0]
|
||||||
cursor = conn.execute("SELECT COUNT(*) FROM day_aggregates")
|
cursor = conn.execute("SELECT COUNT(*) FROM day_aggregates")
|
||||||
day_count = cursor.fetchone()[0]
|
day_count = cursor.fetchone()[0]
|
||||||
|
schema_version = conn.execute("PRAGMA user_version").fetchone()[0]
|
||||||
|
if schema_version >= 4:
|
||||||
|
cursor = conn.execute("SELECT COUNT(*) FROM pending_publications")
|
||||||
|
pending_publication_count = cursor.fetchone()[0]
|
||||||
except sqlite3.Error as exc:
|
except sqlite3.Error as exc:
|
||||||
store_fault = str(exc)
|
store_fault = str(exc)
|
||||||
|
|
||||||
@@ -328,6 +334,7 @@ def compose_status(
|
|||||||
freshness = "unknown"
|
freshness = "unknown"
|
||||||
freshness_age_s = None
|
freshness_age_s = None
|
||||||
sample_count = day_count = 0
|
sample_count = day_count = 0
|
||||||
|
pending_publication_count = 0
|
||||||
deliberately_paused = False
|
deliberately_paused = False
|
||||||
|
|
||||||
# --- External stop detection ---
|
# --- External stop detection ---
|
||||||
@@ -417,6 +424,7 @@ def compose_status(
|
|||||||
overlay_base=overlay_base,
|
overlay_base=overlay_base,
|
||||||
sample_count=sample_count,
|
sample_count=sample_count,
|
||||||
day_count=day_count,
|
day_count=day_count,
|
||||||
|
pending_publication_count=pending_publication_count,
|
||||||
continuity=continuity,
|
continuity=continuity,
|
||||||
paused_lines=paused_lines,
|
paused_lines=paused_lines,
|
||||||
)
|
)
|
||||||
@@ -459,6 +467,8 @@ def render_status_cli(comp: StatusComposition) -> str:
|
|||||||
facts.append("timer: %s" % (
|
facts.append("timer: %s" % (
|
||||||
"unknown" if comp.timer_active is None else "active" if comp.timer_active else "inactive"
|
"unknown" if comp.timer_active is None else "active" if comp.timer_active else "inactive"
|
||||||
))
|
))
|
||||||
|
if comp.pending_publication_count:
|
||||||
|
facts.append(_pending_publication_text(comp.pending_publication_count))
|
||||||
|
|
||||||
if facts:
|
if facts:
|
||||||
lines.append(" · ".join(facts))
|
lines.append(" · ".join(facts))
|
||||||
@@ -524,6 +534,8 @@ def render_status_tui(comp: StatusComposition) -> str:
|
|||||||
facts.append("Timer: %s" % (
|
facts.append("Timer: %s" % (
|
||||||
"unknown" if comp.timer_active is None else "active" if comp.timer_active else "inactive"
|
"unknown" if comp.timer_active is None else "active" if comp.timer_active else "inactive"
|
||||||
))
|
))
|
||||||
|
if comp.pending_publication_count:
|
||||||
|
facts.append(_pending_publication_text(comp.pending_publication_count).capitalize())
|
||||||
|
|
||||||
if facts:
|
if facts:
|
||||||
lines.append(" · ".join(facts))
|
lines.append(" · ".join(facts))
|
||||||
@@ -539,3 +551,8 @@ def render_status_tui(comp: StatusComposition) -> str:
|
|||||||
lines.append(pl[:1].upper() + pl[1:])
|
lines.append(pl[:1].upper() + pl[1:])
|
||||||
|
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _pending_publication_text(count: int) -> str:
|
||||||
|
noun = "observation" if count == 1 else "observations"
|
||||||
|
return f"pending publication: {count} {noun} retained for retry"
|
||||||
|
|||||||
+192
-58
@@ -3,8 +3,7 @@
|
|||||||
This module handles:
|
This module handles:
|
||||||
- Store initialization with WAL mode
|
- Store initialization with WAL mode
|
||||||
- Schema versioning with PRAGMA user_version
|
- Schema versioning with PRAGMA user_version
|
||||||
- The six entities: samples, hour_observations, day_aggregates,
|
- Observation records, derived activity, publication state, and metadata
|
||||||
monitoring_periods, controller_segments, endurance_baseline
|
|
||||||
"""
|
"""
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -12,7 +11,7 @@ from typing import Optional
|
|||||||
|
|
||||||
|
|
||||||
# Schema version - increment on each migration
|
# Schema version - increment on each migration
|
||||||
SCHEMA_VERSION = 3
|
SCHEMA_VERSION = 6
|
||||||
|
|
||||||
|
|
||||||
# Packaged default placement (spec §8.3). The config may override it, but a
|
# Packaged default placement (spec §8.3). The config may override it, but a
|
||||||
@@ -33,7 +32,7 @@ def get_store_path(config: dict) -> Path:
|
|||||||
def init_store(store_path: Path) -> sqlite3.Connection:
|
def init_store(store_path: Path) -> sqlite3.Connection:
|
||||||
"""Initialize the observation store if not present.
|
"""Initialize the observation store if not present.
|
||||||
|
|
||||||
Creates the database with WAL mode and all six entities.
|
Creates the observation schema, including private pending-publication storage.
|
||||||
Returns a connection to the store.
|
Returns a connection to the store.
|
||||||
"""
|
"""
|
||||||
conn = sqlite3.connect(str(store_path))
|
conn = sqlite3.connect(str(store_path))
|
||||||
@@ -73,9 +72,11 @@ def init_store(store_path: Path) -> sqlite3.Connection:
|
|||||||
)
|
)
|
||||||
elif current_version < SCHEMA_VERSION:
|
elif current_version < SCHEMA_VERSION:
|
||||||
# Older version - apply migrations
|
# Older version - apply migrations
|
||||||
_apply_migrations(conn, current_version)
|
try:
|
||||||
conn.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
|
_apply_migrations(conn, current_version)
|
||||||
conn.commit()
|
except Exception:
|
||||||
|
conn.close()
|
||||||
|
raise
|
||||||
|
|
||||||
return conn
|
return conn
|
||||||
|
|
||||||
@@ -107,7 +108,8 @@ def _create_schema(conn: sqlite3.Connection):
|
|||||||
bytes_written INTEGER,
|
bytes_written INTEGER,
|
||||||
bytes_read INTEGER,
|
bytes_read INTEGER,
|
||||||
critical_warning INTEGER,
|
critical_warning INTEGER,
|
||||||
segment_id INTEGER
|
segment_id INTEGER,
|
||||||
|
local_tz TEXT
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
|
||||||
@@ -209,10 +211,18 @@ def _create_schema(conn: sqlite3.Connection):
|
|||||||
coverage REAL DEFAULT 0.0,
|
coverage REAL DEFAULT 0.0,
|
||||||
sample_count INTEGER DEFAULT 0,
|
sample_count INTEGER DEFAULT 0,
|
||||||
complete BOOLEAN DEFAULT 0,
|
complete BOOLEAN DEFAULT 0,
|
||||||
|
activity_seconds INTEGER NOT NULL DEFAULT 0,
|
||||||
|
activity_intervals INTEGER NOT NULL DEFAULT 0,
|
||||||
|
activity_incomplete BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
activity_precision TEXT NOT NULL DEFAULT 'measured',
|
||||||
|
last_sample_id INTEGER,
|
||||||
UNIQUE(local_date, tz_name)
|
UNIQUE(local_date, tz_name)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
|
||||||
|
_create_local_day_shared_evidence(conn)
|
||||||
|
_create_local_day_segment_totals(conn)
|
||||||
|
|
||||||
# Metadata table for store state (e.g., legacy import marker)
|
# Metadata table for store state (e.g., legacy import marker)
|
||||||
conn.execute("""
|
conn.execute("""
|
||||||
CREATE TABLE IF NOT EXISTS store_metadata (
|
CREATE TABLE IF NOT EXISTS store_metadata (
|
||||||
@@ -221,59 +231,181 @@ def _create_schema(conn: sqlite3.Connection):
|
|||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
|
||||||
|
_create_pending_publications(conn)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_pending_publications(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Create private staging for valid observations awaiting derivation."""
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS pending_publications (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
sample_ts TEXT NOT NULL,
|
||||||
|
payload TEXT NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def _create_local_day_shared_evidence(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Create once-only local activity evidence that cannot be day-allocated."""
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS local_day_unallocated_evidence (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
start_sample_id INTEGER NOT NULL,
|
||||||
|
end_sample_id INTEGER NOT NULL,
|
||||||
|
start_local_date TEXT NOT NULL,
|
||||||
|
end_local_date TEXT NOT NULL,
|
||||||
|
start_tz_name TEXT,
|
||||||
|
end_tz_name TEXT,
|
||||||
|
started_at TEXT NOT NULL,
|
||||||
|
ended_at TEXT NOT NULL,
|
||||||
|
bytes_written INTEGER NOT NULL DEFAULT 0,
|
||||||
|
bytes_read INTEGER NOT NULL DEFAULT 0,
|
||||||
|
reason TEXT NOT NULL,
|
||||||
|
segment_id INTEGER,
|
||||||
|
UNIQUE(start_sample_id, end_sample_id)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS local_day_evidence_start "
|
||||||
|
"ON local_day_unallocated_evidence(start_local_date, start_tz_name)"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS local_day_evidence_end "
|
||||||
|
"ON local_day_unallocated_evidence(end_local_date, end_tz_name)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_local_day_segment_totals(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Retain the controller-segment provenance behind known day totals."""
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS local_day_segment_totals (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
local_day_id INTEGER NOT NULL,
|
||||||
|
segment_id INTEGER NOT NULL,
|
||||||
|
bytes_written INTEGER NOT NULL DEFAULT 0,
|
||||||
|
bytes_read INTEGER NOT NULL DEFAULT 0,
|
||||||
|
activity_seconds INTEGER NOT NULL DEFAULT 0,
|
||||||
|
activity_intervals INTEGER NOT NULL DEFAULT 0,
|
||||||
|
UNIQUE(local_day_id, segment_id)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
def _apply_migrations(conn: sqlite3.Connection, current_version: int):
|
def _apply_migrations(conn: sqlite3.Connection, current_version: int):
|
||||||
"""Apply forward-only migrations from current_version to SCHEMA_VERSION.
|
"""Apply forward-only migrations from current_version to SCHEMA_VERSION.
|
||||||
|
|
||||||
Each migration step is a transactional block. Add new steps as sequential
|
|
||||||
elif branches when SCHEMA_VERSION increases.
|
|
||||||
|
|
||||||
Spec: §3.6, §10.2
|
|
||||||
"""
|
|
||||||
# Migration 1→2: add segment_id provenance to samples,
|
|
||||||
# unattributed byte tracking to day_aggregates (issue #73)
|
|
||||||
if current_version < 2:
|
|
||||||
# Defensive: only ALTER if table exists (handles minimal v1 stores)
|
|
||||||
tables = {row[0] for row in conn.execute(
|
|
||||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
||||||
).fetchall()}
|
|
||||||
if "samples" in tables:
|
|
||||||
# Check if column already exists (idempotent)
|
|
||||||
cols = {row[1] for row in conn.execute("PRAGMA table_info(samples)").fetchall()}
|
|
||||||
if "segment_id" not in cols:
|
|
||||||
conn.execute("ALTER TABLE samples ADD COLUMN segment_id INTEGER")
|
|
||||||
if "day_aggregates" in tables:
|
|
||||||
cols = {row[1] for row in conn.execute("PRAGMA table_info(day_aggregates)").fetchall()}
|
|
||||||
if "unattributed_bytes_written" not in cols:
|
|
||||||
conn.execute("ALTER TABLE day_aggregates ADD COLUMN unattributed_bytes_written INTEGER DEFAULT 0")
|
|
||||||
if "unattributed_bytes_read" not in cols:
|
|
||||||
conn.execute("ALTER TABLE day_aggregates ADD COLUMN unattributed_bytes_read INTEGER DEFAULT 0")
|
|
||||||
current_version = 2
|
|
||||||
|
|
||||||
# Migration 2→3: add local_days table for local-day activity totals
|
Commit each version transition independently. A failed step rolls back in
|
||||||
# (issue #90, ADR 0010). Pure addition — no existing rows touched.
|
full while earlier successful steps remain versioned and retryable.
|
||||||
if current_version < 3:
|
"""
|
||||||
tables = {row[0] for row in conn.execute(
|
migrations = {
|
||||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
2: _migrate_1_to_2,
|
||||||
|
3: _migrate_2_to_3,
|
||||||
|
4: _migrate_3_to_4,
|
||||||
|
5: _migrate_4_to_5,
|
||||||
|
6: _migrate_5_to_6,
|
||||||
|
}
|
||||||
|
while current_version < SCHEMA_VERSION:
|
||||||
|
target_version = current_version + 1
|
||||||
|
migration = migrations.get(target_version)
|
||||||
|
if migration is None:
|
||||||
|
raise ValueError(f"No migration registered for schema {target_version}")
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
try:
|
||||||
|
migration(conn)
|
||||||
|
conn.execute(f"PRAGMA user_version={target_version}")
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
current_version = target_version
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_1_to_2(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Add segment provenance and unattributed UTC byte tracking."""
|
||||||
|
tables = {row[0] for row in conn.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||||
|
).fetchall()}
|
||||||
|
if "samples" in tables:
|
||||||
|
cols = {row[1] for row in conn.execute(
|
||||||
|
"PRAGMA table_info(samples)"
|
||||||
).fetchall()}
|
).fetchall()}
|
||||||
if "local_days" not in tables:
|
if "segment_id" not in cols:
|
||||||
conn.execute("""
|
conn.execute("ALTER TABLE samples ADD COLUMN segment_id INTEGER")
|
||||||
CREATE TABLE IF NOT EXISTS local_days (
|
if "day_aggregates" in tables:
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
cols = {row[1] for row in conn.execute(
|
||||||
local_date TEXT NOT NULL,
|
"PRAGMA table_info(day_aggregates)"
|
||||||
tz_name TEXT NOT NULL,
|
).fetchall()}
|
||||||
tz_offset TEXT NOT NULL,
|
for column in ("unattributed_bytes_written", "unattributed_bytes_read"):
|
||||||
utc_start TEXT NOT NULL,
|
if column not in cols:
|
||||||
utc_end TEXT NOT NULL,
|
conn.execute(
|
||||||
bytes_written INTEGER DEFAULT 0,
|
f"ALTER TABLE day_aggregates ADD COLUMN {column} INTEGER DEFAULT 0"
|
||||||
bytes_read INTEGER DEFAULT 0,
|
|
||||||
coverage REAL DEFAULT 0.0,
|
|
||||||
sample_count INTEGER DEFAULT 0,
|
|
||||||
complete BOOLEAN DEFAULT 0,
|
|
||||||
UNIQUE(local_date, tz_name)
|
|
||||||
)
|
)
|
||||||
""")
|
|
||||||
current_version = 3
|
|
||||||
|
def _migrate_2_to_3(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Add local-day activity summaries."""
|
||||||
|
tables = {row[0] for row in conn.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||||
|
).fetchall()}
|
||||||
|
if "local_days" not in tables:
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE local_days (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
local_date TEXT NOT NULL,
|
||||||
|
tz_name TEXT NOT NULL,
|
||||||
|
tz_offset TEXT NOT NULL,
|
||||||
|
utc_start TEXT NOT NULL,
|
||||||
|
utc_end TEXT NOT NULL,
|
||||||
|
bytes_written INTEGER DEFAULT 0,
|
||||||
|
bytes_read INTEGER DEFAULT 0,
|
||||||
|
coverage REAL DEFAULT 0.0,
|
||||||
|
sample_count INTEGER DEFAULT 0,
|
||||||
|
complete BOOLEAN DEFAULT 0,
|
||||||
|
UNIQUE(local_date, tz_name)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_3_to_4(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Add private publication staging for acquired observations."""
|
||||||
|
_create_pending_publications(conn)
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_4_to_5(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Add measured local-day evidence storage and mark old totals legacy."""
|
||||||
|
tables = {row[0] for row in conn.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||||
|
).fetchall()}
|
||||||
|
if "samples" in tables:
|
||||||
|
sample_cols = {row[1] for row in conn.execute(
|
||||||
|
"PRAGMA table_info(samples)"
|
||||||
|
).fetchall()}
|
||||||
|
if "local_tz" not in sample_cols:
|
||||||
|
conn.execute("ALTER TABLE samples ADD COLUMN local_tz TEXT")
|
||||||
|
if "local_days" in tables:
|
||||||
|
local_cols = {row[1] for row in conn.execute(
|
||||||
|
"PRAGMA table_info(local_days)"
|
||||||
|
).fetchall()}
|
||||||
|
for column, declaration in (
|
||||||
|
("activity_seconds", "INTEGER NOT NULL DEFAULT 0"),
|
||||||
|
("activity_intervals", "INTEGER NOT NULL DEFAULT 0"),
|
||||||
|
("activity_incomplete", "BOOLEAN NOT NULL DEFAULT 0"),
|
||||||
|
("activity_precision", "TEXT NOT NULL DEFAULT 'legacy'"),
|
||||||
|
("last_sample_id", "INTEGER"),
|
||||||
|
):
|
||||||
|
if column not in local_cols:
|
||||||
|
conn.execute(
|
||||||
|
f"ALTER TABLE local_days ADD COLUMN {column} {declaration}"
|
||||||
|
)
|
||||||
|
_create_local_day_shared_evidence(conn)
|
||||||
|
_create_local_day_segment_totals(conn)
|
||||||
|
|
||||||
|
|
||||||
|
def _migrate_5_to_6(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Rebuild local-day summaries from surviving trustworthy evidence."""
|
||||||
|
from .local_day import repair_legacy_local_day_evidence
|
||||||
|
|
||||||
|
repair_legacy_local_day_evidence(conn)
|
||||||
|
|
||||||
|
|
||||||
def migrate_to_latest(store_path: Path) -> int:
|
def migrate_to_latest(store_path: Path) -> int:
|
||||||
@@ -310,9 +442,11 @@ def migrate_to_latest(store_path: Path) -> int:
|
|||||||
return SCHEMA_VERSION
|
return SCHEMA_VERSION
|
||||||
|
|
||||||
steps = SCHEMA_VERSION - current_version
|
steps = SCHEMA_VERSION - current_version
|
||||||
_apply_migrations(conn, current_version)
|
try:
|
||||||
conn.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
|
_apply_migrations(conn, current_version)
|
||||||
conn.commit()
|
except Exception:
|
||||||
|
conn.close()
|
||||||
|
raise
|
||||||
conn.close()
|
conn.close()
|
||||||
return steps
|
return steps
|
||||||
|
|
||||||
|
|||||||
+671
-164
File diff suppressed because it is too large
Load Diff
@@ -25,11 +25,14 @@ def detect_system_tz() -> str:
|
|||||||
|
|
||||||
localtime = Path("/etc/localtime")
|
localtime = Path("/etc/localtime")
|
||||||
if localtime.is_symlink():
|
if localtime.is_symlink():
|
||||||
target = os.readlink(str(localtime))
|
target_path = Path(os.readlink(str(localtime)))
|
||||||
# Strip common prefixes: /usr/share/zoneinfo/, /usr/lib/zoneinfo/
|
if not target_path.is_absolute():
|
||||||
for prefix in ("/usr/share/zoneinfo/", "/usr/lib/zoneinfo/"):
|
target_path = localtime.parent / target_path
|
||||||
if target.startswith(prefix):
|
target = target_path.resolve().as_posix()
|
||||||
return target[len(prefix):]
|
marker = "/zoneinfo/"
|
||||||
|
marker_index = target.find(marker)
|
||||||
|
if marker_index >= 0:
|
||||||
|
return target[marker_index + len(marker):]
|
||||||
return target
|
return target
|
||||||
|
|
||||||
return "UTC"
|
return "UTC"
|
||||||
|
|||||||
@@ -121,8 +121,8 @@ def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO local_days "
|
"INSERT INTO local_days "
|
||||||
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
||||||
" bytes_written, bytes_read, coverage, sample_count, complete) "
|
" bytes_written, bytes_read, coverage, sample_count, complete, activity_intervals) "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)",
|
||||||
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
||||||
bw, br, coverage, samples, complete),
|
bw, br, coverage, samples, complete),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""Activity-selection transitions independent from Textual widgets."""
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from fenris.activity_selection import ActivitySelection, IntervalIdentity
|
||||||
|
|
||||||
|
|
||||||
|
def _interval(n):
|
||||||
|
return IntervalIdentity(f"start-{n}", f"end-{n}")
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_selection_starts_and_stays_on_newest_interval():
|
||||||
|
selection = ActivitySelection()
|
||||||
|
first = [_interval(1), _interval(2)]
|
||||||
|
|
||||||
|
selection.update_live(first)
|
||||||
|
assert selection.following_live
|
||||||
|
assert selection.selected_live_interval == first[-1]
|
||||||
|
|
||||||
|
second = first + [_interval(3)]
|
||||||
|
selection.update_live(second)
|
||||||
|
assert selection.selected_live_interval == second[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_keyboard_and_mouse_inspection_pin_interval_identity():
|
||||||
|
intervals = [_interval(1), _interval(2), _interval(3)]
|
||||||
|
keyboard = ActivitySelection()
|
||||||
|
mouse = ActivitySelection()
|
||||||
|
|
||||||
|
keyboard.update_live(intervals)
|
||||||
|
keyboard.move_live(intervals, -1)
|
||||||
|
mouse.update_live(intervals)
|
||||||
|
mouse.inspect_live(intervals, intervals[1])
|
||||||
|
|
||||||
|
assert keyboard.selected_live_interval == mouse.selected_live_interval == intervals[1]
|
||||||
|
assert not keyboard.following_live
|
||||||
|
assert not mouse.following_live
|
||||||
|
|
||||||
|
refreshed = [_interval(2), _interval(3), _interval(4)]
|
||||||
|
keyboard.update_live(refreshed)
|
||||||
|
mouse.update_live(refreshed)
|
||||||
|
assert keyboard.selected_live_interval == mouse.selected_live_interval == intervals[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_pin_explains_expiration_and_resumes_following():
|
||||||
|
selection = ActivitySelection()
|
||||||
|
intervals = [_interval(1), _interval(2), _interval(3)]
|
||||||
|
selection.update_live(intervals)
|
||||||
|
selection.inspect_live(intervals, intervals[0])
|
||||||
|
|
||||||
|
newest = _interval(4)
|
||||||
|
selection.update_live([_interval(2), _interval(3), newest])
|
||||||
|
|
||||||
|
assert selection.following_live
|
||||||
|
assert selection.selected_live_interval == newest
|
||||||
|
assert selection.live_interval_expired
|
||||||
|
|
||||||
|
|
||||||
|
def test_today_resets_expired_pin_and_read_write_toggle_keeps_identity():
|
||||||
|
selection = ActivitySelection()
|
||||||
|
intervals = [_interval(1), _interval(2)]
|
||||||
|
selection.update_live(intervals)
|
||||||
|
selection.inspect_live(intervals, intervals[0])
|
||||||
|
selection.update_live([intervals[1]])
|
||||||
|
newest = _interval(3)
|
||||||
|
selection.update_live([intervals[1], newest])
|
||||||
|
assert selection.live_interval_expired
|
||||||
|
|
||||||
|
selection.toggle_measure()
|
||||||
|
assert selection.measure == "read"
|
||||||
|
assert selection.selected_live_interval == newest
|
||||||
|
|
||||||
|
selection.select_history_date("2026-09-27")
|
||||||
|
selection.set_view("history")
|
||||||
|
selection.set_view("live")
|
||||||
|
assert selection.browse_date is None
|
||||||
|
assert selection.following_live
|
||||||
|
assert not selection.live_interval_expired
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_identity_uses_both_interval_boundaries():
|
||||||
|
first = IntervalIdentity("2026-09-28T10:00:00Z", "2026-09-28T10:03:00Z")
|
||||||
|
changed_start = IntervalIdentity("2026-09-28T09:57:00Z", "2026-09-28T10:03:00Z")
|
||||||
|
selection = ActivitySelection()
|
||||||
|
selection.update_live([first])
|
||||||
|
selection.inspect_live([first], first)
|
||||||
|
|
||||||
|
selection.update_live([changed_start])
|
||||||
|
|
||||||
|
assert selection.following_live
|
||||||
|
assert selection.selected_live_interval == changed_start
|
||||||
|
assert selection.live_interval_expired
|
||||||
@@ -154,7 +154,7 @@ class TestSchemaMigration:
|
|||||||
|
|
||||||
# Migrate
|
# Migrate
|
||||||
steps = migrate_to_latest(db)
|
steps = migrate_to_latest(db)
|
||||||
assert steps == 2 # v1→v2→v3
|
assert steps == 5 # v1→v2→v3→v4→v5→v6
|
||||||
|
|
||||||
# Verify data preserved
|
# Verify data preserved
|
||||||
conn = sqlite3.connect(str(db))
|
conn = sqlite3.connect(str(db))
|
||||||
@@ -356,6 +356,167 @@ class TestCollectorDerivation:
|
|||||||
assert count_after == 2
|
assert count_after == 2
|
||||||
|
|
||||||
|
|
||||||
|
class TestCollectionAtomicity:
|
||||||
|
"""Collection exposes sample and derived evidence as one publication."""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_collection_is_visible_through_cli_and_tui_readers(
|
||||||
|
self, config_fixture, smartctl_fixture, sysfs_fixture_tree, monkeypatch
|
||||||
|
):
|
||||||
|
"""Ordinary readers observe matching published sample and activity evidence."""
|
||||||
|
from fenris.status import get_status, read_status
|
||||||
|
from fenris.tui import FenrisTuiApp
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
service = {
|
||||||
|
"boot_enabled": True,
|
||||||
|
"timer_active": True,
|
||||||
|
"last_collect_ok": True,
|
||||||
|
}
|
||||||
|
monkeypatch.setattr("fenris.status.query_service_state", lambda: service)
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc).replace(
|
||||||
|
minute=35, second=0, microsecond=0,
|
||||||
|
)
|
||||||
|
first = {
|
||||||
|
**smartctl_fixture,
|
||||||
|
"nvme_smart_health_information_log": {
|
||||||
|
**smartctl_fixture["nvme_smart_health_information_log"],
|
||||||
|
"data_units_written": 12345678,
|
||||||
|
"data_units_read": 9876543,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
second = {
|
||||||
|
**smartctl_fixture,
|
||||||
|
"nvme_smart_health_information_log": {
|
||||||
|
**smartctl_fixture["nvme_smart_health_information_log"],
|
||||||
|
"data_units_written": 12345698,
|
||||||
|
"data_units_read": 9876553,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
sysfs_path = sysfs_fixture_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
|
||||||
|
assert run_collection(
|
||||||
|
first, sysfs_path, config_fixture, FakeClock(now - timedelta(minutes=5))
|
||||||
|
)["ok"] is True
|
||||||
|
assert run_collection(
|
||||||
|
second, sysfs_path, config_fixture, FakeClock(now)
|
||||||
|
)["ok"] is True
|
||||||
|
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
with read_status(store_path, now, query_services=False) as (reader, composition):
|
||||||
|
assert reader is not None
|
||||||
|
assert composition.sample_count == 2
|
||||||
|
assert composition.day_count == 1
|
||||||
|
utc_day = reader.execute(
|
||||||
|
"SELECT bytes_written_delta, bytes_read_delta FROM day_aggregates"
|
||||||
|
).fetchone()
|
||||||
|
local_day = reader.execute(
|
||||||
|
"SELECT bytes_written, bytes_read FROM local_days"
|
||||||
|
).fetchone()
|
||||||
|
assert tuple(utc_day) == (10_240_000, 5_120_000)
|
||||||
|
assert tuple(local_day) == (10_240_000, 5_120_000)
|
||||||
|
|
||||||
|
cli_output = get_status(
|
||||||
|
store_path, now, query_services=True, query_journal=False
|
||||||
|
)
|
||||||
|
assert "Monitoring" in cli_output
|
||||||
|
|
||||||
|
app = FenrisTuiApp(store_path=store_path)
|
||||||
|
async with app.run_test(size=(100, 30)) as pilot:
|
||||||
|
await pilot.pause()
|
||||||
|
live_readout = str(app.query_one("#live-readout").render())
|
||||||
|
assert "W 0.010 GB" in live_readout
|
||||||
|
assert "R 0.005 GB" in live_readout
|
||||||
|
|
||||||
|
repeated_result = run_collection(
|
||||||
|
second, sysfs_path, config_fixture, FakeClock(now + timedelta(minutes=5))
|
||||||
|
)
|
||||||
|
assert repeated_result["ok"] is True
|
||||||
|
with read_status(
|
||||||
|
store_path, now + timedelta(minutes=5), query_services=False
|
||||||
|
) as (reader, composition):
|
||||||
|
assert reader is not None
|
||||||
|
assert composition.sample_count == 3
|
||||||
|
utc_day = reader.execute(
|
||||||
|
"SELECT bytes_written_delta, bytes_read_delta FROM day_aggregates"
|
||||||
|
).fetchone()
|
||||||
|
local_day = reader.execute(
|
||||||
|
"SELECT bytes_written, bytes_read FROM local_days"
|
||||||
|
).fetchone()
|
||||||
|
assert tuple(utc_day) == (10_240_000, 5_120_000)
|
||||||
|
assert tuple(local_day) == (10_240_000, 5_120_000)
|
||||||
|
|
||||||
|
def test_failed_local_day_publication_keeps_previous_publication(
|
||||||
|
self, config_fixture, smartctl_fixture, sysfs_fixture_tree, monkeypatch
|
||||||
|
):
|
||||||
|
"""A failed final derivation step leaves all prior reader state intact."""
|
||||||
|
from fenris.status import read_status
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
|
||||||
|
first_clock = FakeClock(now)
|
||||||
|
first = {
|
||||||
|
**smartctl_fixture,
|
||||||
|
"nvme_smart_health_information_log": {
|
||||||
|
**smartctl_fixture["nvme_smart_health_information_log"],
|
||||||
|
"data_units_written": 12345678,
|
||||||
|
"data_units_read": 9876543,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
first_result = run_collection(
|
||||||
|
first,
|
||||||
|
sysfs_fixture_tree / "sys" / "class" / "nvme" / "nvme0",
|
||||||
|
config_fixture,
|
||||||
|
first_clock,
|
||||||
|
)
|
||||||
|
assert first_result["ok"] is True, first_result
|
||||||
|
|
||||||
|
writer = sqlite3.connect(config_fixture["store_path"])
|
||||||
|
writer.execute(
|
||||||
|
"CREATE TRIGGER fail_local_day_publication "
|
||||||
|
"BEFORE INSERT ON local_days "
|
||||||
|
"BEGIN SELECT RAISE(ABORT, 'injected local-day publication failure'); END"
|
||||||
|
)
|
||||||
|
writer.commit()
|
||||||
|
writer.close()
|
||||||
|
|
||||||
|
next_time = now + timedelta(minutes=5)
|
||||||
|
second = {
|
||||||
|
**smartctl_fixture,
|
||||||
|
"nvme_smart_health_information_log": {
|
||||||
|
**smartctl_fixture["nvme_smart_health_information_log"],
|
||||||
|
"data_units_written": 12345698,
|
||||||
|
"data_units_read": 9876548,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
failed_result = run_collection(
|
||||||
|
second,
|
||||||
|
sysfs_fixture_tree / "sys" / "class" / "nvme" / "nvme0",
|
||||||
|
config_fixture,
|
||||||
|
FakeClock(next_time),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert failed_result["ok"] is False
|
||||||
|
assert "injected local-day publication failure" in failed_result["error"]
|
||||||
|
|
||||||
|
with read_status(
|
||||||
|
Path(config_fixture["store_path"]), next_time, query_services=False
|
||||||
|
) as (reader, composition):
|
||||||
|
assert reader is not None
|
||||||
|
assert composition.sample_count == 1
|
||||||
|
public_counts = reader.execute(
|
||||||
|
"SELECT (SELECT COUNT(*) FROM samples), "
|
||||||
|
"(SELECT COUNT(*) FROM controller_segments), "
|
||||||
|
"(SELECT COUNT(*) FROM monitoring_periods), "
|
||||||
|
"(SELECT COUNT(*) FROM hour_observations), "
|
||||||
|
"(SELECT COUNT(*) FROM day_aggregates), "
|
||||||
|
"(SELECT COUNT(*) FROM local_days)"
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
assert tuple(public_counts) == (1, 1, 1, 0, 0, 0)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Display states: awaiting first sample, awaiting another sample
|
# Display states: awaiting first sample, awaiting another sample
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -232,7 +232,7 @@ def test_store_initialization(config_fixture: Dict[str, Any]):
|
|||||||
# Initialize store
|
# Initialize store
|
||||||
conn = init_store(store_path)
|
conn = init_store(store_path)
|
||||||
|
|
||||||
# Verify all six entities exist
|
# Verify the observation, derived-history, and metadata tables exist.
|
||||||
cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||||
tables = {row[0] for row in cursor.fetchall()}
|
tables = {row[0] for row in cursor.fetchall()}
|
||||||
|
|
||||||
@@ -249,6 +249,9 @@ def test_store_initialization(config_fixture: Dict[str, Any]):
|
|||||||
expected_tables.add("sqlite_sequence")
|
expected_tables.add("sqlite_sequence")
|
||||||
expected_tables.add("store_metadata")
|
expected_tables.add("store_metadata")
|
||||||
expected_tables.add("local_days")
|
expected_tables.add("local_days")
|
||||||
|
expected_tables.add("pending_publications")
|
||||||
|
expected_tables.add("local_day_unallocated_evidence")
|
||||||
|
expected_tables.add("local_day_segment_totals")
|
||||||
assert expected_tables == tables
|
assert expected_tables == tables
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|||||||
@@ -98,8 +98,8 @@ def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO local_days "
|
"INSERT INTO local_days "
|
||||||
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
||||||
" bytes_written, bytes_read, coverage, sample_count, complete) "
|
" bytes_written, bytes_read, coverage, sample_count, complete, activity_intervals) "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)",
|
||||||
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
||||||
bw, br, coverage, samples, complete),
|
bw, br, coverage, samples, complete),
|
||||||
)
|
)
|
||||||
@@ -166,6 +166,47 @@ class TestGateNoCompleteDay:
|
|||||||
# The key assertion: gate message should NOT appear when gate IS met
|
# The key assertion: gate message should NOT appear when gate IS met
|
||||||
assert not any("full local observation day" in f for f in result.contributing_facts)
|
assert not any("full local observation day" in f for f in result.contributing_facts)
|
||||||
|
|
||||||
|
def test_complete_legacy_day_without_trusted_activity_does_not_open_gate(
|
||||||
|
self, store,
|
||||||
|
):
|
||||||
|
"""A complete flag cannot make unavailable local activity qualify."""
|
||||||
|
_insert_baseline(store)
|
||||||
|
_insert_segment(store)
|
||||||
|
_open_period(store)
|
||||||
|
for i in range(3):
|
||||||
|
day = (datetime(2026, 9, 27) + timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
|
_insert_day(store, day, bw=1024 * 1024 * 100)
|
||||||
|
_insert_sample(store, "2026-09-30T10:00:00+00:00", pu=5)
|
||||||
|
_insert_local_day(store, "2026-09-29", complete=True)
|
||||||
|
store.execute(
|
||||||
|
"UPDATE local_days SET activity_precision = 'measured', activity_intervals = 0 "
|
||||||
|
"WHERE local_date = '2026-09-29'"
|
||||||
|
)
|
||||||
|
result = compute_projection(store, _clock())
|
||||||
|
assert result.confidence_state == ConfidenceState.UNSUPPORTED
|
||||||
|
assert any("full local observation day" in fact
|
||||||
|
for fact in result.contributing_facts)
|
||||||
|
|
||||||
|
def test_partial_but_trusted_local_activity_opens_gate(self, store):
|
||||||
|
"""Known local intervals can coexist with an incomplete day total."""
|
||||||
|
_insert_baseline(store)
|
||||||
|
_insert_segment(store)
|
||||||
|
_open_period(store)
|
||||||
|
for i in range(3):
|
||||||
|
day = (datetime(2026, 9, 27) + timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
|
_insert_day(store, day, bw=1024 * 1024 * 100)
|
||||||
|
_insert_sample(store, "2026-09-30T10:00:00+00:00", pu=5)
|
||||||
|
_insert_local_day(store, "2026-09-29", complete=True)
|
||||||
|
store.execute(
|
||||||
|
"UPDATE local_days SET activity_incomplete = 1 "
|
||||||
|
"WHERE local_date = '2026-09-29'"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = compute_projection(store, _clock())
|
||||||
|
|
||||||
|
assert not any("full local observation day" in fact
|
||||||
|
for fact in result.contributing_facts)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Gate-2: One complete local day → Limited confidence
|
# Gate-2: One complete local day → Limited confidence
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ async def test_tabs_date_entry_and_hourly_inspection_keep_context(dashboard):
|
|||||||
app.on_refresh_tick()
|
app.on_refresh_tick()
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
assert str(app.query_one("#bar-readout").render()) == readout
|
assert str(app.query_one("#bar-readout").render()) == readout
|
||||||
|
assert "Reads" in str(app.query_one("#bar-legend").render())
|
||||||
await pilot.press("g")
|
await pilot.press("g")
|
||||||
await pilot.press(*list("2026-09-17"))
|
await pilot.press(*list("2026-09-17"))
|
||||||
await pilot.press("escape")
|
await pilot.press("escape")
|
||||||
|
|||||||
@@ -634,6 +634,14 @@ class TestMonitorIntegration:
|
|||||||
cmd_enable(args)
|
cmd_enable(args)
|
||||||
mock_enable.assert_called_once_with(True)
|
mock_enable.assert_called_once_with(True)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store_path)
|
||||||
|
try:
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM monitoring_periods WHERE ended_at IS NULL"
|
||||||
|
).fetchone()[0] == 1
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
def test_cmd_disable_uses_init_system(self, tmp_path):
|
def test_cmd_disable_uses_init_system(self, tmp_path):
|
||||||
"""cmd_disable uses init_system.disable_timer."""
|
"""cmd_disable uses init_system.disable_timer."""
|
||||||
from fenris.monitor import cmd_disable
|
from fenris.monitor import cmd_disable
|
||||||
|
|||||||
@@ -0,0 +1,341 @@
|
|||||||
|
"""Normal-collection retention acceptance tests for issue #100."""
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
from fenris.local_day import query_local_day_summary
|
||||||
|
from fenris.pruning import prune_old_samples
|
||||||
|
from fenris.status import read_status
|
||||||
|
from fenris.store import init_store
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sysfs_controller(tmp_path):
|
||||||
|
controller = tmp_path / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
controller.mkdir(parents=True)
|
||||||
|
(controller / "subsysnqn").write_text("nqn.example:drive-1\n")
|
||||||
|
(controller / "model").write_text("Fenris Test Drive\n")
|
||||||
|
(controller / "serial").write_text("drive-1\n")
|
||||||
|
(controller / "firmware_rev").write_text("1.0\n")
|
||||||
|
transport = controller / "transport"
|
||||||
|
transport.mkdir()
|
||||||
|
(transport / "address").write_text("0000:00:01.0\n")
|
||||||
|
(transport / "trstring").write_text("pcie\n")
|
||||||
|
return controller
|
||||||
|
|
||||||
|
|
||||||
|
class FakeClock:
|
||||||
|
def __init__(self, now):
|
||||||
|
self.now = now
|
||||||
|
|
||||||
|
def utcnow(self):
|
||||||
|
return self.now
|
||||||
|
|
||||||
|
|
||||||
|
def _smartctl(written, read):
|
||||||
|
return {
|
||||||
|
"nvme_smart_health_information_log": {
|
||||||
|
"critical_warning": 0,
|
||||||
|
"temperature": 35,
|
||||||
|
"available_spare": 100,
|
||||||
|
"percentage_used": 1,
|
||||||
|
"data_units_written": written,
|
||||||
|
"data_units_read": read,
|
||||||
|
"power_on_hours": 100,
|
||||||
|
"power_cycles": 1,
|
||||||
|
"unsafe_shutdowns": 0,
|
||||||
|
"media_errors": 0,
|
||||||
|
},
|
||||||
|
"user_capacity": {"bytes": 1_000_000_000_000},
|
||||||
|
"model_name": "Fenris Test Drive",
|
||||||
|
"serial_number": "drive-1",
|
||||||
|
"firmware_version": "1.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _collect(config, controller, now, written, read):
|
||||||
|
return run_collection(
|
||||||
|
smartctl_data=_smartctl(written, read),
|
||||||
|
sysfs_path=controller,
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_normal_collection_expires_detail_and_keeps_local_day_evidence(
|
||||||
|
tmp_path, sysfs_controller, monkeypatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
store_path = tmp_path / "observations.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
first_time = datetime(2026, 9, 1, 0, 0, tzinfo=timezone.utc)
|
||||||
|
second_time = first_time + timedelta(days=15)
|
||||||
|
|
||||||
|
first = _collect(config, sysfs_controller, first_time, 1000, 2000)
|
||||||
|
assert first["ok"] is True, first
|
||||||
|
second = _collect(config, sysfs_controller, second_time, 1001, 2003)
|
||||||
|
assert second["ok"] is True, second
|
||||||
|
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute("SELECT ts FROM samples ORDER BY id").fetchall() == [
|
||||||
|
(second_time.isoformat(),)
|
||||||
|
]
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT bytes_written, bytes_read, reason "
|
||||||
|
"FROM local_day_unallocated_evidence"
|
||||||
|
).fetchall() == [(512_000, 1_536_000, "local_midnight")]
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 0
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM controller_segments").fetchone()[0] == 1
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM day_aggregates").fetchone()[0] == 15
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "Asia/Kolkata")
|
||||||
|
with read_status(store_path, second_time, query_services=False) as (reader, state):
|
||||||
|
assert reader is not None
|
||||||
|
assert state.store_fault is None
|
||||||
|
old_day = query_local_day_summary(
|
||||||
|
reader, "2026-09-01", "UTC", second_time,
|
||||||
|
)
|
||||||
|
assert old_day["utc_start"] == "2026-09-01T00:00:00+00:00"
|
||||||
|
assert old_day["utc_end"] == "2026-09-02T00:00:00+00:00"
|
||||||
|
assert old_day["bytes_written"] is None
|
||||||
|
assert old_day["shared_bytes_written"] == 512_000
|
||||||
|
assert old_day["shared_bytes_read"] == 1_536_000
|
||||||
|
assert old_day["shared_evidence_count"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_pruning_keeps_samples_when_replacement_evidence_is_missing(tmp_path):
|
||||||
|
store_path = tmp_path / "incomplete.db"
|
||||||
|
conn = init_store(store_path)
|
||||||
|
now = datetime(2026, 9, 30, tzinfo=timezone.utc)
|
||||||
|
old_time = now - timedelta(days=20)
|
||||||
|
next_time = now - timedelta(days=1)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO controller_segments (id, opened_at, identity_key) "
|
||||||
|
"VALUES (1, ?, 'test')",
|
||||||
|
(old_time.isoformat(),),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, segment_id, local_tz) "
|
||||||
|
"VALUES (?, '/dev/nvme0', 100, 100, 1, 'UTC')",
|
||||||
|
(old_time.isoformat(),),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, segment_id, local_tz) "
|
||||||
|
"VALUES (?, '/dev/nvme0', 200, 200, 1, 'UTC')",
|
||||||
|
(next_time.isoformat(),),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
assert prune_old_samples(conn, now) == 0
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM samples").fetchone()[0] == 2
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_later_same_day_activity_does_not_replace_gap_interval(tmp_path):
|
||||||
|
store_path = tmp_path / "unrelated-local-activity.db"
|
||||||
|
conn = init_store(store_path)
|
||||||
|
gap_start = datetime(2026, 9, 1, 0, 0, tzinfo=timezone.utc)
|
||||||
|
gap_end = gap_start + timedelta(minutes=10)
|
||||||
|
later_end = gap_start + timedelta(minutes=20)
|
||||||
|
now = datetime(2026, 9, 30, tzinfo=timezone.utc)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO controller_segments (id, opened_at, identity_key) "
|
||||||
|
"VALUES (1, ?, 'test')",
|
||||||
|
(gap_start.isoformat(),),
|
||||||
|
)
|
||||||
|
for sample_time, written, read in (
|
||||||
|
(gap_start, 1000, 100),
|
||||||
|
(gap_end, 2000, 200),
|
||||||
|
(later_end, 2100, 210),
|
||||||
|
):
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, segment_id, local_tz) "
|
||||||
|
"VALUES (?, '/dev/nvme0', ?, ?, 1, 'UTC')",
|
||||||
|
(sample_time.isoformat(), written, read),
|
||||||
|
)
|
||||||
|
|
||||||
|
# UTC aggregates can replace the old counter pair, but the local-day
|
||||||
|
# summary contains only the later B->C interval, after monitoring resumed.
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO hour_observations (hour, bytes_written_delta, bytes_read_delta) "
|
||||||
|
"VALUES ('2026-09-01T00:00:00+00:00', 1000, 100)"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO day_aggregates (day) VALUES ('2026-09-01')"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO monitoring_periods (started_at, ended_at) VALUES (?, ?)",
|
||||||
|
(gap_end.isoformat(), later_end.isoformat()),
|
||||||
|
)
|
||||||
|
last_sample_id = conn.execute("SELECT MAX(id) FROM samples").fetchone()[0]
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_days "
|
||||||
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, bytes_written, "
|
||||||
|
" bytes_read, activity_intervals, activity_precision, last_sample_id) "
|
||||||
|
"VALUES ('2026-09-01', 'UTC', '+00:00', ?, ?, 100, 10, 1, 'measured', ?)",
|
||||||
|
(
|
||||||
|
gap_start.isoformat(),
|
||||||
|
(gap_start + timedelta(days=1)).isoformat(),
|
||||||
|
last_sample_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
local_day_id = conn.execute("SELECT MAX(id) FROM local_days").fetchone()[0]
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_day_segment_totals "
|
||||||
|
"(local_day_id, segment_id, bytes_written, bytes_read, activity_intervals) "
|
||||||
|
"VALUES (?, 1, 100, 10, 1)",
|
||||||
|
(local_day_id,),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
assert prune_old_samples(conn, now) == 0
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM samples").fetchone()[0] == 3
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pruning_keeps_oldest_unpaired_sample_as_successor_anchor(tmp_path):
|
||||||
|
store_path = tmp_path / "anchor.db"
|
||||||
|
conn = init_store(store_path)
|
||||||
|
now = datetime(2026, 9, 30, tzinfo=timezone.utc)
|
||||||
|
old_time = now - timedelta(days=20)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO controller_segments (id, opened_at, identity_key) "
|
||||||
|
"VALUES (1, ?, 'test')",
|
||||||
|
(old_time.isoformat(),),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, segment_id, local_tz) "
|
||||||
|
"VALUES (?, '/dev/nvme0', 100, 100, 1, 'UTC')",
|
||||||
|
(old_time.isoformat(),),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
assert prune_old_samples(conn, now) == 0
|
||||||
|
assert conn.execute("SELECT ts FROM samples").fetchone()[0] == old_time.isoformat()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pruning_never_expires_pending_publications(tmp_path):
|
||||||
|
store_path = tmp_path / "pending.db"
|
||||||
|
conn = init_store(store_path)
|
||||||
|
old_time = datetime(2026, 9, 1, tzinfo=timezone.utc)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO pending_publications (sample_ts, payload) VALUES (?, '{}')",
|
||||||
|
(old_time.isoformat(),),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
assert prune_old_samples(
|
||||||
|
conn, datetime(2026, 9, 30, tzinfo=timezone.utc),
|
||||||
|
) == 0
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT sample_ts FROM pending_publications"
|
||||||
|
).fetchone()[0] == old_time.isoformat()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pruning_preserves_unparseable_legacy_timestamp(tmp_path):
|
||||||
|
store_path = tmp_path / "legacy.db"
|
||||||
|
conn = init_store(store_path)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read) "
|
||||||
|
"VALUES ('legacy timestamp', '/dev/nvme0', 100, 100)"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
assert prune_old_samples(conn, datetime(2026, 9, 30, tzinfo=timezone.utc)) == 0
|
||||||
|
assert conn.execute("SELECT ts FROM samples").fetchone()[0] == "legacy timestamp"
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_collection_defers_interrupted_pruning_and_retries_next_run(
|
||||||
|
tmp_path, sysfs_controller, monkeypatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
store_path = tmp_path / "interrupted.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
first_time = datetime(2026, 9, 1, 0, 0, tzinfo=timezone.utc)
|
||||||
|
second_time = first_time + timedelta(days=15)
|
||||||
|
assert _collect(config, sysfs_controller, first_time, 1000, 2000)["ok"] is True
|
||||||
|
|
||||||
|
writer = sqlite3.connect(store_path)
|
||||||
|
writer.execute(
|
||||||
|
"CREATE TRIGGER stop_retention BEFORE DELETE ON samples "
|
||||||
|
"BEGIN SELECT RAISE(ABORT, 'simulated retention interruption'); END"
|
||||||
|
)
|
||||||
|
writer.commit()
|
||||||
|
writer.close()
|
||||||
|
|
||||||
|
second = _collect(config, sysfs_controller, second_time, 1001, 2001)
|
||||||
|
assert second["ok"] is True, second
|
||||||
|
assert "simulated retention interruption" in second["retention_error"]
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM samples").fetchone()[0] == 2
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 0
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM local_day_unallocated_evidence").fetchone()[0] == 1
|
||||||
|
|
||||||
|
writer = sqlite3.connect(store_path)
|
||||||
|
writer.execute("DROP TRIGGER stop_retention")
|
||||||
|
writer.commit()
|
||||||
|
writer.close()
|
||||||
|
|
||||||
|
third_time = second_time + timedelta(days=15)
|
||||||
|
third = _collect(config, sysfs_controller, third_time, 1002, 2002)
|
||||||
|
assert third["ok"] is True, third
|
||||||
|
assert third["retention_error"] is None
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute("SELECT ts FROM samples ORDER BY id").fetchall() == [
|
||||||
|
(third_time.isoformat(),)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_collection_recovers_old_pending_work_before_pruning(
|
||||||
|
tmp_path, sysfs_controller, monkeypatch,
|
||||||
|
):
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
store_path = tmp_path / "pending-recovery.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
first_time = datetime(2026, 9, 1, 0, 0, tzinfo=timezone.utc)
|
||||||
|
second_time = first_time + timedelta(days=15)
|
||||||
|
assert _collect(config, sysfs_controller, first_time, 1000, 2000)["ok"] is True
|
||||||
|
|
||||||
|
writer = sqlite3.connect(store_path)
|
||||||
|
writer.execute(
|
||||||
|
"CREATE TRIGGER interrupt_publication "
|
||||||
|
"BEFORE INSERT ON local_day_unallocated_evidence "
|
||||||
|
"BEGIN SELECT RAISE(ABORT, 'simulated publication interruption'); END"
|
||||||
|
)
|
||||||
|
writer.commit()
|
||||||
|
writer.close()
|
||||||
|
|
||||||
|
failed = _collect(config, sysfs_controller, second_time, 1001, 2001)
|
||||||
|
assert failed["ok"] is False
|
||||||
|
assert "simulated publication interruption" in failed["error"]
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM samples").fetchone()[0] == 1
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 1
|
||||||
|
|
||||||
|
writer = sqlite3.connect(store_path)
|
||||||
|
writer.execute("DROP TRIGGER interrupt_publication")
|
||||||
|
writer.commit()
|
||||||
|
writer.close()
|
||||||
|
|
||||||
|
third_time = second_time + timedelta(minutes=5)
|
||||||
|
recovered = _collect(config, sysfs_controller, third_time, 1002, 2002)
|
||||||
|
assert recovered["ok"] is True, recovered
|
||||||
|
assert recovered["retention_error"] is None
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 0
|
||||||
|
assert conn.execute("SELECT ts FROM samples ORDER BY id").fetchall() == [
|
||||||
|
(second_time.isoformat(),),
|
||||||
|
(third_time.isoformat(),),
|
||||||
|
]
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone()[0] == 1
|
||||||
@@ -0,0 +1,257 @@
|
|||||||
|
"""Bound pending-publication admission before device acquisition."""
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from fenris import collector
|
||||||
|
|
||||||
|
|
||||||
|
class FakeClock:
|
||||||
|
def __init__(self, now):
|
||||||
|
self.now = now
|
||||||
|
|
||||||
|
def utcnow(self):
|
||||||
|
return self.now
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sysfs_controller(tmp_path):
|
||||||
|
controller_path = tmp_path / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
controller_path.mkdir(parents=True)
|
||||||
|
(controller_path / "subsysnqn").write_text("nqn.test:drive\n")
|
||||||
|
(controller_path / "model").write_text("Test NVMe\n")
|
||||||
|
(controller_path / "serial").write_text("test-serial\n")
|
||||||
|
(controller_path / "firmware_rev").write_text("1.0\n")
|
||||||
|
transport = controller_path / "transport"
|
||||||
|
transport.mkdir()
|
||||||
|
(transport / "trstring").write_text("pcie\n")
|
||||||
|
return controller_path
|
||||||
|
|
||||||
|
|
||||||
|
def _smartctl(units_written):
|
||||||
|
return {
|
||||||
|
"nvme_smart_health_information_log": {
|
||||||
|
"critical_warning": 0,
|
||||||
|
"temperature": 35,
|
||||||
|
"available_spare": 100,
|
||||||
|
"percentage_used": 5,
|
||||||
|
"data_units_written": units_written,
|
||||||
|
"data_units_read": 500,
|
||||||
|
"power_on_hours": 100,
|
||||||
|
},
|
||||||
|
"user_capacity": {"bytes": 1_024_000_000_000},
|
||||||
|
"model_name": "Test NVMe",
|
||||||
|
"serial_number": "test-serial",
|
||||||
|
"firmware_version": "1.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _collect(config, controller, now, units_written):
|
||||||
|
return collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now),
|
||||||
|
acquire=lambda: (_smartctl(units_written), controller),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_partial_recovery_frees_admission_and_keeps_pending_order(
|
||||||
|
tmp_path, sysfs_controller, monkeypatch,
|
||||||
|
):
|
||||||
|
"""A freed slot admits one observation behind older pending evidence."""
|
||||||
|
assert collector.PENDING_PUBLICATION_LIMIT == 6_720
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
monkeypatch.setattr(collector, "PENDING_PUBLICATION_LIMIT", 2)
|
||||||
|
|
||||||
|
store_path = tmp_path / "observations.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
old_time = datetime(2026, 8, 1, tzinfo=timezone.utc)
|
||||||
|
controller = sysfs_controller
|
||||||
|
|
||||||
|
assert _collect(config, controller, old_time, 1000)["ok"] is True
|
||||||
|
|
||||||
|
original_derive = collector.derive_hours_from_interval
|
||||||
|
|
||||||
|
def unavailable_derivation(*_args):
|
||||||
|
raise RuntimeError("derivation unavailable")
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "derive_hours_from_interval", unavailable_derivation)
|
||||||
|
first_pending = _collect(
|
||||||
|
config, controller, old_time + timedelta(minutes=5), 1010,
|
||||||
|
)
|
||||||
|
assert first_pending["ok"] is False
|
||||||
|
|
||||||
|
second_pending = _collect(
|
||||||
|
config, controller, old_time + timedelta(minutes=10), 1020,
|
||||||
|
)
|
||||||
|
assert second_pending["ok"] is False
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM pending_publications"
|
||||||
|
).fetchone()[0] == 2
|
||||||
|
|
||||||
|
called = False
|
||||||
|
|
||||||
|
def must_not_acquire():
|
||||||
|
nonlocal called
|
||||||
|
called = True
|
||||||
|
return _smartctl(1030), controller
|
||||||
|
|
||||||
|
full = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(old_time + timedelta(days=40)),
|
||||||
|
acquire=must_not_acquire,
|
||||||
|
)
|
||||||
|
assert full["ok"] is False
|
||||||
|
assert "capacity full (2 observations)" in full["error"]
|
||||||
|
assert called is False
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*), MIN(sample_ts) FROM pending_publications"
|
||||||
|
).fetchone() == (
|
||||||
|
2,
|
||||||
|
(old_time + timedelta(minutes=5)).isoformat(),
|
||||||
|
)
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM samples").fetchone()[0] == 1
|
||||||
|
|
||||||
|
def fail_second_pending(conn, previous, current):
|
||||||
|
if current["data_units_written"] == 1020:
|
||||||
|
raise RuntimeError("second observation still blocked")
|
||||||
|
return original_derive(conn, previous, current)
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "derive_hours_from_interval", fail_second_pending)
|
||||||
|
acquired = False
|
||||||
|
|
||||||
|
def acquire_after_partial_recovery():
|
||||||
|
nonlocal acquired
|
||||||
|
acquired = True
|
||||||
|
return _smartctl(1030), controller
|
||||||
|
|
||||||
|
partial = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(old_time + timedelta(days=40, minutes=5)),
|
||||||
|
acquire=acquire_after_partial_recovery,
|
||||||
|
)
|
||||||
|
assert partial["ok"] is False
|
||||||
|
assert acquired is True, partial
|
||||||
|
assert "no new observation acquired" not in partial["error"]
|
||||||
|
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT data_units_written FROM samples ORDER BY id"
|
||||||
|
).fetchall() == [(1000,), (1010,)]
|
||||||
|
queued_units = [
|
||||||
|
json.loads(row[0])["sample"]["data_units_written"]
|
||||||
|
for row in conn.execute(
|
||||||
|
"SELECT payload FROM pending_publications ORDER BY id"
|
||||||
|
)
|
||||||
|
]
|
||||||
|
assert queued_units == [1020, 1030]
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM monitoring_periods").fetchone()[0] == 1
|
||||||
|
|
||||||
|
recovered_in_order = []
|
||||||
|
|
||||||
|
def record_recovery_order(conn, previous, current):
|
||||||
|
recovered_in_order.append(current["data_units_written"])
|
||||||
|
return original_derive(conn, previous, current)
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "derive_hours_from_interval", record_recovery_order)
|
||||||
|
recovered = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(old_time + timedelta(days=40, minutes=10)),
|
||||||
|
acquire=lambda: (_smartctl(1040), controller),
|
||||||
|
)
|
||||||
|
assert recovered["ok"] is True, recovered
|
||||||
|
assert recovered_in_order == [1020, 1030, 1040]
|
||||||
|
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT data_units_written FROM samples ORDER BY id DESC LIMIT 1"
|
||||||
|
).fetchone() == (1040,)
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_actual_pending_limit_fails_before_acquisition_and_keeps_old_evidence(
|
||||||
|
tmp_path, sysfs_controller, monkeypatch,
|
||||||
|
):
|
||||||
|
"""The production limit retains every old row and refuses device acquisition."""
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
store_path = tmp_path / "full-observations.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
controller = sysfs_controller
|
||||||
|
old_time = datetime(2026, 7, 1, tzinfo=timezone.utc)
|
||||||
|
assert _collect(config, controller, old_time, 1000)["ok"] is True
|
||||||
|
|
||||||
|
def unavailable_derivation(*_args):
|
||||||
|
raise RuntimeError("derivation unavailable")
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "derive_hours_from_interval", unavailable_derivation)
|
||||||
|
failed = _collect(
|
||||||
|
config, controller, old_time + timedelta(minutes=5), 1010,
|
||||||
|
)
|
||||||
|
assert failed["ok"] is False
|
||||||
|
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
payload_row = conn.execute(
|
||||||
|
"SELECT payload FROM pending_publications ORDER BY id LIMIT 1"
|
||||||
|
).fetchone()
|
||||||
|
assert payload_row is not None
|
||||||
|
base = json.loads(payload_row[0])
|
||||||
|
base_observed_at = datetime.fromisoformat(base["sample"]["ts"])
|
||||||
|
base_units = base["sample"]["data_units_written"]
|
||||||
|
rows = []
|
||||||
|
for offset in range(1, collector.PENDING_PUBLICATION_LIMIT):
|
||||||
|
observation = json.loads(payload_row[0])
|
||||||
|
units_written = base_units + offset
|
||||||
|
observed_at = base_observed_at + timedelta(minutes=5 * offset)
|
||||||
|
observation["sample"]["ts"] = observed_at.isoformat()
|
||||||
|
observation["sample"]["data_units_written"] = units_written
|
||||||
|
observation["sample"]["bytes_written"] = units_written * 512_000
|
||||||
|
rows.append((observed_at.isoformat(), json.dumps(observation)))
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT INTO pending_publications (sample_ts, payload) VALUES (?, ?)",
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
called = False
|
||||||
|
|
||||||
|
def must_not_acquire():
|
||||||
|
nonlocal called
|
||||||
|
called = True
|
||||||
|
return _smartctl(9000), controller
|
||||||
|
|
||||||
|
result = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(old_time + timedelta(days=40)),
|
||||||
|
acquire=must_not_acquire,
|
||||||
|
)
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert f"capacity full ({collector.PENDING_PUBLICATION_LIMIT} observations)" in result[
|
||||||
|
"error"
|
||||||
|
]
|
||||||
|
assert called is False
|
||||||
|
|
||||||
|
with sqlite3.connect(store_path) as conn:
|
||||||
|
pending_rows = conn.execute(
|
||||||
|
"SELECT sample_ts, payload FROM pending_publications ORDER BY id"
|
||||||
|
).fetchall()
|
||||||
|
assert len(pending_rows) == collector.PENDING_PUBLICATION_LIMIT
|
||||||
|
assert (pending_rows[0][0], pending_rows[-1][0]) == (
|
||||||
|
(old_time + timedelta(minutes=5)).isoformat(),
|
||||||
|
(
|
||||||
|
old_time
|
||||||
|
+ timedelta(minutes=5 * collector.PENDING_PUBLICATION_LIMIT)
|
||||||
|
).isoformat(),
|
||||||
|
)
|
||||||
|
assert [
|
||||||
|
json.loads(payload)["sample"]["data_units_written"]
|
||||||
|
for _, payload in pending_rows
|
||||||
|
] == list(range(1010, 1010 + collector.PENDING_PUBLICATION_LIMIT))
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM samples").fetchone()[0] == 1
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM monitoring_periods").fetchone()[0] == 1
|
||||||
@@ -90,8 +90,8 @@ def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO local_days "
|
"INSERT INTO local_days "
|
||||||
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
||||||
" bytes_written, bytes_read, coverage, sample_count, complete) "
|
" bytes_written, bytes_read, coverage, sample_count, complete, activity_intervals) "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)",
|
||||||
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
||||||
bw, br, coverage, samples, complete),
|
bw, br, coverage, samples, complete),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -98,8 +98,8 @@ def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO local_days "
|
"INSERT INTO local_days "
|
||||||
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
||||||
" bytes_written, bytes_read, coverage, sample_count, complete) "
|
" bytes_written, bytes_read, coverage, sample_count, complete, activity_intervals) "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)",
|
||||||
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
||||||
bw, br, coverage, samples, complete),
|
bw, br, coverage, samples, complete),
|
||||||
)
|
)
|
||||||
|
|||||||
+306
-65
@@ -24,13 +24,12 @@ from fenris.store import init_store
|
|||||||
from fenris.monitoring_periods import ensure_period_open
|
from fenris.monitoring_periods import ensure_period_open
|
||||||
from fenris.local_day import (
|
from fenris.local_day import (
|
||||||
LocalDaySummary,
|
LocalDaySummary,
|
||||||
|
local_day_boundaries,
|
||||||
persist_local_day,
|
persist_local_day,
|
||||||
query_local_day_summary,
|
|
||||||
)
|
)
|
||||||
|
from fenris.tz_util import detect_system_tz
|
||||||
from fenris.tui import (
|
from fenris.tui import (
|
||||||
FenrisTuiApp,
|
FenrisTuiApp,
|
||||||
HistoryGraph,
|
|
||||||
_RANGE_OPTIONS,
|
|
||||||
_RANGE_DEFAULT,
|
_RANGE_DEFAULT,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -70,13 +69,18 @@ def _open_period(conn, start="2026-09-01T00:00:00+00:00"):
|
|||||||
|
|
||||||
|
|
||||||
def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
||||||
bw=300, br=130, complete=True):
|
bw=300, br=130, complete=True,
|
||||||
|
utc_start=None, utc_end=None):
|
||||||
|
if utc_start is None:
|
||||||
|
utc_start = f"{local_date}T00:00:00+00:00"
|
||||||
|
if utc_end is None:
|
||||||
|
utc_end = f"{(datetime.fromisoformat(local_date) + timedelta(days=1)).strftime('%Y-%m-%d')}T00:00:00+00:00"
|
||||||
summary = LocalDaySummary(
|
summary = LocalDaySummary(
|
||||||
local_date=local_date,
|
local_date=local_date,
|
||||||
tz_name=tz_name,
|
tz_name=tz_name,
|
||||||
tz_offset=tz_offset,
|
tz_offset=tz_offset,
|
||||||
utc_start=f"{local_date}T00:00:00+00:00",
|
utc_start=utc_start,
|
||||||
utc_end=f"{(datetime.fromisoformat(local_date) + timedelta(days=1)).strftime('%Y-%m-%d')}T00:00:00+00:00",
|
utc_end=utc_end,
|
||||||
bytes_written=bw,
|
bytes_written=bw,
|
||||||
bytes_read=br,
|
bytes_read=br,
|
||||||
coverage=0.95,
|
coverage=0.95,
|
||||||
@@ -124,6 +128,10 @@ def _make_app(tmp_path, clock=None):
|
|||||||
return app, patcher
|
return app, patcher
|
||||||
|
|
||||||
|
|
||||||
|
def _visible_selected_date(app):
|
||||||
|
return str(app.query_one("#local-day").render())[:10]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# AC92-1: Keyboard bindings exist
|
# AC92-1: Keyboard bindings exist
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -178,14 +186,13 @@ class TestBracketNavigation:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
# Move to an older day first so ] can advance
|
# Move to an older day first so ] can advance
|
||||||
for _ in range(3):
|
for _ in range(3):
|
||||||
await pilot.press("left_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
day_before = graph._day_data[graph.selected_index].get("day")
|
day_before = _visible_selected_date(app)
|
||||||
|
|
||||||
await pilot.press("right_square_bracket")
|
await pilot.press("right_square_bracket")
|
||||||
day_after = graph._day_data[graph.selected_index].get("day")
|
day_after = _visible_selected_date(app)
|
||||||
# ] should move to a newer day (closer to today)
|
# ] should move to a newer day (closer to today)
|
||||||
assert day_after > day_before
|
assert day_after > day_before
|
||||||
|
|
||||||
@@ -198,11 +205,10 @@ class TestBracketNavigation:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
day_before = _visible_selected_date(app)
|
||||||
day_before = graph._day_data[graph.selected_index].get("day")
|
|
||||||
|
|
||||||
await pilot.press("left_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
day_after = graph._day_data[graph.selected_index].get("day")
|
day_after = _visible_selected_date(app)
|
||||||
# [ should move to an older day (further from today)
|
# [ should move to an older day (further from today)
|
||||||
assert day_after < day_before
|
assert day_after < day_before
|
||||||
|
|
||||||
@@ -224,9 +230,8 @@ class TestBracketNavigation:
|
|||||||
for _ in range(29):
|
for _ in range(29):
|
||||||
await pilot.press("left_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
|
|
||||||
# Graph should now show a range that includes the oldest day
|
# Visible readout identifies oldest selected date.
|
||||||
assert graph.selected_index >= 0
|
selected_day = _visible_selected_date(app)
|
||||||
selected_day = graph._day_data[graph.selected_index].get("day")
|
|
||||||
oldest_day = (app_clock - timedelta(days=29)).strftime("%Y-%m-%d")
|
oldest_day = (app_clock - timedelta(days=29)).strftime("%Y-%m-%d")
|
||||||
assert selected_day == oldest_day
|
assert selected_day == oldest_day
|
||||||
finally:
|
finally:
|
||||||
@@ -241,11 +246,10 @@ class TestBracketNavigation:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
initial_date = _visible_selected_date(app)
|
||||||
initial_idx = graph.selected_index
|
|
||||||
|
|
||||||
await pilot.press("right_square_bracket")
|
await pilot.press("right_square_bracket")
|
||||||
assert graph.selected_index == initial_idx
|
assert _visible_selected_date(app) == initial_date
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -262,36 +266,54 @@ class TestArrowInspection:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
await pilot.press("v", "v")
|
await pilot.press("v", "v")
|
||||||
graph.focus()
|
app.query_one("#usage-history").focus()
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
day_before = graph._day_data[graph.selected_index].get("day")
|
day_before = _visible_selected_date(app)
|
||||||
|
|
||||||
await pilot.press("left")
|
await pilot.press("left")
|
||||||
day_after = graph._day_data[graph.selected_index].get("day")
|
day_after = _visible_selected_date(app)
|
||||||
assert day_after != day_before
|
assert day_after != day_before
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_arrow_keys_inspect_hours_in_drill(self, tmp_path):
|
async def test_arrow_keys_inspect_hours_in_drill(self, tmp_path):
|
||||||
"""Arrow keys navigate hours when in hourly drill-down."""
|
"""Arrow keys navigate hours when in hourly drill-down."""
|
||||||
conn, clock = _setup_store(tmp_path, n_days=14)
|
conn, clock = _setup_store(tmp_path, n_days=14)
|
||||||
|
local_date = datetime.now().astimezone().date().isoformat()
|
||||||
|
timezone_name = detect_system_tz()
|
||||||
|
utc_start, utc_end, tz_offset = local_day_boundaries(local_date, timezone_name)
|
||||||
|
_insert_local_day(
|
||||||
|
conn, local_date, tz_name=timezone_name, tz_offset=tz_offset,
|
||||||
|
utc_start=utc_start.isoformat(), utc_end=utc_end.isoformat(),
|
||||||
|
)
|
||||||
|
hour = utc_start.replace(minute=0, second=0, microsecond=0)
|
||||||
|
if hour < utc_start:
|
||||||
|
hour += timedelta(hours=1)
|
||||||
|
while hour + timedelta(hours=1) <= utc_end:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO hour_observations "
|
||||||
|
"(hour, bytes_written_delta, bytes_read_delta, coverage, sample_count) "
|
||||||
|
"VALUES (?, 1000000, 2000000, 1, 1)",
|
||||||
|
(hour.isoformat(),),
|
||||||
|
)
|
||||||
|
hour += timedelta(hours=1)
|
||||||
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
await pilot.press("v", "v")
|
await pilot.press("v", "v")
|
||||||
await pilot.click("#usage-history")
|
await pilot.click("#usage-history")
|
||||||
|
|
||||||
# Enter drill-down
|
# Enter drill-down
|
||||||
await pilot.press("enter")
|
await pilot.press("enter")
|
||||||
assert graph.view_mode == "hourly"
|
assert app.query_one("#activity-tabs").active == "view-day"
|
||||||
|
|
||||||
# Arrow keys navigate hours
|
# Arrow keys navigate hours
|
||||||
|
readout_before = str(app.query_one("#bar-readout").render())
|
||||||
await pilot.press("left")
|
await pilot.press("left")
|
||||||
assert graph._hourly_selected >= 0
|
assert str(app.query_one("#bar-readout").render()) != readout_before
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -308,18 +330,17 @@ class TestTodayBinding:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
|
|
||||||
# Browse to an older day using [
|
# Browse to an older day using [
|
||||||
for _ in range(5):
|
for _ in range(5):
|
||||||
await pilot.press("left_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
browsed_day = graph._day_data[graph.selected_index].get("day")
|
browsed_day = _visible_selected_date(app)
|
||||||
today = clock.strftime("%Y-%m-%d")
|
today = datetime.now().astimezone().date().isoformat()
|
||||||
assert browsed_day != today
|
assert browsed_day != today
|
||||||
|
|
||||||
# Press t to return to today
|
# Press t to return to today
|
||||||
await pilot.press("t")
|
await pilot.press("t")
|
||||||
assert graph.selected_index == len(graph._day_data) - 1
|
assert app.query_one("#activity-tabs").active == "view-live"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_t_exits_hourly_drill(self, tmp_path):
|
async def test_t_exits_hourly_drill(self, tmp_path):
|
||||||
@@ -330,17 +351,16 @@ class TestTodayBinding:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
await pilot.press("v", "v")
|
await pilot.press("v", "v")
|
||||||
await pilot.click("#usage-history")
|
await pilot.click("#usage-history")
|
||||||
|
|
||||||
# Enter drill
|
# Enter drill
|
||||||
await pilot.press("enter")
|
await pilot.press("enter")
|
||||||
assert graph.view_mode == "hourly"
|
assert app.query_one("#activity-tabs").active == "view-day"
|
||||||
|
|
||||||
# t returns to daily
|
# t returns to daily
|
||||||
await pilot.press("t")
|
await pilot.press("t")
|
||||||
assert graph.view_mode == "daily"
|
assert app.query_one("#activity-tabs").active == "view-live"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -360,7 +380,7 @@ class TestDateEntry:
|
|||||||
await pilot.press("g")
|
await pilot.press("g")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
# Date picker should be pushed as a screen
|
# Date picker should be pushed as a screen
|
||||||
assert len(app.screen_stack) > 1
|
assert app.screen.query_one("#date-input") is not None
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_valid_date_navigates(self, tmp_path):
|
async def test_valid_date_navigates(self, tmp_path):
|
||||||
@@ -375,13 +395,12 @@ class TestDateEntry:
|
|||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
target_date = (app_clock - timedelta(days=5)).strftime("%Y-%m-%d")
|
target_date = (app_clock - timedelta(days=5)).strftime("%Y-%m-%d")
|
||||||
|
|
||||||
# Simulate the date picker callback directly
|
await pilot.press("g")
|
||||||
app._go_to_date_callback(target_date)
|
await pilot.press(*target_date)
|
||||||
|
await pilot.press("enter")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
|
|
||||||
graph = app.query_one("#usage-history")
|
assert _visible_selected_date(app) == target_date
|
||||||
selected_day = graph._day_data[graph.selected_index].get("day")
|
|
||||||
assert selected_day == target_date
|
|
||||||
finally:
|
finally:
|
||||||
patcher.stop()
|
patcher.stop()
|
||||||
|
|
||||||
@@ -400,7 +419,8 @@ class TestDateEntry:
|
|||||||
await pilot.press("enter")
|
await pilot.press("enter")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
# Should remain on date picker (error shown, not dismissed)
|
# Should remain on date picker (error shown, not dismissed)
|
||||||
assert len(app.screen_stack) > 1
|
error = str(app.screen.query_one("#date-error").render())
|
||||||
|
assert "Invalid date format" in error
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_cancel_returns_to_prior(self, tmp_path):
|
async def test_cancel_returns_to_prior(self, tmp_path):
|
||||||
@@ -411,8 +431,7 @@ class TestDateEntry:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
prior_readout = str(app.query_one("#bar-readout").render())
|
||||||
prior_idx = graph.selected_index
|
|
||||||
|
|
||||||
await pilot.press("g")
|
await pilot.press("g")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
@@ -420,7 +439,7 @@ class TestDateEntry:
|
|||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
|
|
||||||
# Should be back to the graph with the same selection
|
# Should be back to the graph with the same selection
|
||||||
assert graph.selected_index == prior_idx
|
assert str(app.query_one("#bar-readout").render()) == prior_readout
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -442,8 +461,6 @@ class TestLocalDayEvidence:
|
|||||||
try:
|
try:
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
|
|
||||||
# Navigate to the target day
|
# Navigate to the target day
|
||||||
for _ in range(4):
|
for _ in range(4):
|
||||||
await pilot.press("left_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
@@ -489,12 +506,10 @@ class TestBrowsingStability:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=0.1)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=0.1)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
|
|
||||||
# Browse to a specific day
|
# Browse to a specific day
|
||||||
for _ in range(3):
|
for _ in range(3):
|
||||||
await pilot.press("right_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
browsed_day = graph._day_data[graph.selected_index].get("day")
|
browsed_day = _visible_selected_date(app)
|
||||||
|
|
||||||
# Wait for at least one refresh tick
|
# Wait for at least one refresh tick
|
||||||
import asyncio
|
import asyncio
|
||||||
@@ -502,8 +517,7 @@ class TestBrowsingStability:
|
|||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
|
|
||||||
# Selection should be preserved
|
# Selection should be preserved
|
||||||
assert graph.selected_index >= 0
|
assert _visible_selected_date(app) == browsed_day
|
||||||
assert graph._day_data[graph.selected_index].get("day") == browsed_day
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_navigate_away_does_not_jump_on_refresh(self, tmp_path):
|
async def test_navigate_away_does_not_jump_on_refresh(self, tmp_path):
|
||||||
@@ -514,14 +528,12 @@ class TestBrowsingStability:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=0.1)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=0.1)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
|
|
||||||
# Browse to older day using [
|
# Browse to older day using [
|
||||||
for _ in range(5):
|
for _ in range(5):
|
||||||
await pilot.press("left_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
|
|
||||||
browsed_day = graph._day_data[graph.selected_index].get("day")
|
browsed_day = _visible_selected_date(app)
|
||||||
today = clock.strftime("%Y-%m-%d")
|
today = datetime.now().astimezone().date().isoformat()
|
||||||
assert browsed_day != today
|
assert browsed_day != today
|
||||||
|
|
||||||
# Wait for refresh
|
# Wait for refresh
|
||||||
@@ -530,8 +542,8 @@ class TestBrowsingStability:
|
|||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
|
|
||||||
# Should NOT jump back to today
|
# Should NOT jump back to today
|
||||||
selected_day = graph._day_data[graph.selected_index].get("day")
|
assert _visible_selected_date(app) == browsed_day
|
||||||
assert selected_day != today
|
assert _visible_selected_date(app) != today
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -548,17 +560,16 @@ class TestDrillDownData:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(100, 40)) as pilot:
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
|
||||||
await pilot.press("v", "v")
|
await pilot.press("v", "v")
|
||||||
await pilot.click("#usage-history")
|
await pilot.click("#usage-history")
|
||||||
|
|
||||||
# Enter drill-down
|
# Enter drill-down
|
||||||
await pilot.press("enter")
|
await pilot.press("enter")
|
||||||
assert graph.view_mode == "hourly"
|
assert app.query_one("#activity-tabs").active == "view-day"
|
||||||
|
|
||||||
# After drill callback, data should be populated
|
assert "hourly evidence unavailable" in str(
|
||||||
# (empty since we have no hour_observations)
|
app.query_one("#bar-readout").render()
|
||||||
assert graph.view_mode == "hourly"
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -575,11 +586,10 @@ class TestConstrainedWidth:
|
|||||||
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
async with app.run_test(size=(80, 24)) as pilot:
|
async with app.run_test(size=(80, 24)) as pilot:
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
graph = app.query_one("#usage-history")
|
day_before = _visible_selected_date(app)
|
||||||
day_before = graph._day_data[graph.selected_index].get("day")
|
|
||||||
|
|
||||||
await pilot.press("left_square_bracket")
|
await pilot.press("left_square_bracket")
|
||||||
day_after = graph._day_data[graph.selected_index].get("day")
|
day_after = _visible_selected_date(app)
|
||||||
assert day_after < day_before
|
assert day_after < day_before
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -593,4 +603,235 @@ class TestConstrainedWidth:
|
|||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
await pilot.press("g")
|
await pilot.press("g")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
assert len(app.screen_stack) > 1
|
assert app.screen.query_one("#date-input") is not None
|
||||||
|
|
||||||
|
|
||||||
|
class TestClickableNavigation:
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("size", [(80, 24), (70, 20)])
|
||||||
|
async def test_click_and_keyboard_select_same_local_date(self, tmp_path, size):
|
||||||
|
"""Clickable and keyboard navigation share date-entry transitions."""
|
||||||
|
app_clock = _clock()
|
||||||
|
conn, _ = _setup_store(tmp_path, n_days=14, today=app_clock)
|
||||||
|
conn.close()
|
||||||
|
app, patcher = _make_app(tmp_path, clock=app_clock)
|
||||||
|
expected = (app_clock - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||||
|
direct = (app_clock - timedelta(days=5)).strftime("%Y-%m-%d")
|
||||||
|
try:
|
||||||
|
async with app.run_test(size=size) as pilot:
|
||||||
|
await pilot.press("left_square_bracket")
|
||||||
|
keyboard_date = _visible_selected_date(app)
|
||||||
|
assert keyboard_date == expected
|
||||||
|
|
||||||
|
await pilot.press("t")
|
||||||
|
await pilot.click("#activity-tools", offset=(2, 0))
|
||||||
|
assert _visible_selected_date(app) == keyboard_date
|
||||||
|
|
||||||
|
await pilot.press("g")
|
||||||
|
await pilot.press(*direct)
|
||||||
|
await pilot.press("enter")
|
||||||
|
await pilot.pause()
|
||||||
|
assert _visible_selected_date(app) == direct
|
||||||
|
finally:
|
||||||
|
patcher.stop()
|
||||||
|
|
||||||
|
|
||||||
|
class TestHistoricalSelectionIdentity:
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_unavailable_entered_date_stays_visible_after_refresh(self, tmp_path):
|
||||||
|
"""An unavailable requested date remains selected and explained."""
|
||||||
|
conn, clock = _setup_store(tmp_path, n_days=30)
|
||||||
|
conn.close()
|
||||||
|
app, patcher = _make_app(tmp_path, clock=clock)
|
||||||
|
target = (clock - timedelta(days=120)).strftime("%Y-%m-%d")
|
||||||
|
try:
|
||||||
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
|
await pilot.press("g")
|
||||||
|
await pilot.press(*target)
|
||||||
|
await pilot.press("enter")
|
||||||
|
await pilot.pause()
|
||||||
|
|
||||||
|
local_day = str(app.query_one("#local-day").render())
|
||||||
|
assert target in local_day
|
||||||
|
assert "unavailable" in local_day
|
||||||
|
|
||||||
|
app.on_refresh_tick()
|
||||||
|
await pilot.pause()
|
||||||
|
local_day = str(app.query_one("#local-day").render())
|
||||||
|
assert target in local_day
|
||||||
|
assert "unavailable" in local_day
|
||||||
|
finally:
|
||||||
|
patcher.stop()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retained_summary_outside_recent_window_remains_available(self, tmp_path):
|
||||||
|
"""Direct date entry can inspect durable local summaries beyond 90 days."""
|
||||||
|
clock = _clock()
|
||||||
|
conn, _ = _setup_store(tmp_path, n_days=30, today=clock)
|
||||||
|
target = (clock - timedelta(days=120)).strftime("%Y-%m-%d")
|
||||||
|
utc_start, utc_end, tz_offset = local_day_boundaries(
|
||||||
|
target, "Asia/Kolkata",
|
||||||
|
)
|
||||||
|
_insert_local_day(
|
||||||
|
conn,
|
||||||
|
target,
|
||||||
|
tz_name="Asia/Kolkata",
|
||||||
|
tz_offset=tz_offset,
|
||||||
|
bw=2_000_000_000,
|
||||||
|
br=1_000_000_000,
|
||||||
|
utc_start=utc_start.isoformat(),
|
||||||
|
utc_end=utc_end.isoformat(),
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
app, patcher = _make_app(tmp_path, clock=clock)
|
||||||
|
try:
|
||||||
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
|
await pilot.press("g")
|
||||||
|
await pilot.press(*target)
|
||||||
|
await pilot.press("enter")
|
||||||
|
await pilot.pause()
|
||||||
|
await pilot.press("v")
|
||||||
|
await pilot.pause()
|
||||||
|
|
||||||
|
readout = str(app.query_one("#bar-readout").render())
|
||||||
|
assert target in readout
|
||||||
|
assert "Asia/Kolkata +05:30" in readout
|
||||||
|
assert "W 2.000 GB known · R 1.000 GB known" in readout
|
||||||
|
finally:
|
||||||
|
patcher.stop()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_constrained_history_keeps_known_totals_and_evidence_state(self, tmp_path):
|
||||||
|
"""Small textual view retains selected day, totals, and shared evidence."""
|
||||||
|
clock = _clock()
|
||||||
|
conn, _ = _setup_store(tmp_path, n_days=14, today=clock)
|
||||||
|
target = (clock - timedelta(days=2)).strftime("%Y-%m-%d")
|
||||||
|
utc_start, utc_end, tz_offset = local_day_boundaries(
|
||||||
|
target, "Asia/Kolkata",
|
||||||
|
)
|
||||||
|
_insert_local_day(
|
||||||
|
conn,
|
||||||
|
target,
|
||||||
|
tz_name="Asia/Kolkata",
|
||||||
|
tz_offset=tz_offset,
|
||||||
|
bw=1_500_000_000,
|
||||||
|
br=500_000_000,
|
||||||
|
utc_start=utc_start.isoformat(),
|
||||||
|
utc_end=utc_end.isoformat(),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_day_unallocated_evidence "
|
||||||
|
"(start_sample_id, end_sample_id, start_local_date, end_local_date, "
|
||||||
|
" start_tz_name, end_tz_name, started_at, ended_at, bytes_written, "
|
||||||
|
" bytes_read, reason) VALUES (1, 2, ?, ?, ?, ?, ?, ?, ?, ?, 'local_midnight')",
|
||||||
|
(target, "2026-09-29", "Asia/Kolkata", "Asia/Kolkata",
|
||||||
|
utc_start.isoformat(), utc_start.isoformat(), 200_000_000, 300_000_000),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
app, patcher = _make_app(tmp_path, clock=clock)
|
||||||
|
try:
|
||||||
|
async with app.run_test(size=(70, 20)) as pilot:
|
||||||
|
await pilot.press("g")
|
||||||
|
await pilot.press(*target)
|
||||||
|
await pilot.press("enter")
|
||||||
|
await pilot.press("v")
|
||||||
|
await pilot.pause()
|
||||||
|
|
||||||
|
readout = str(app.query_one("#bar-readout").render())
|
||||||
|
assert target in readout
|
||||||
|
assert "Asia/Kolkata +05:30" in readout
|
||||||
|
assert "incomplete" in readout
|
||||||
|
assert "W 1.500 GB known · R 0.500 GB known" in readout
|
||||||
|
assert "shared at midnight W 0.200 GB · R 0.300 GB" in readout
|
||||||
|
finally:
|
||||||
|
patcher.stop()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_date_picker_keystrokes_do_not_toggle_measurement(self, tmp_path):
|
||||||
|
"""Typing in date entry does not trigger dashboard shortcuts."""
|
||||||
|
conn, _ = _setup_store(tmp_path, n_days=14)
|
||||||
|
conn.close()
|
||||||
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
|
await pilot.pause()
|
||||||
|
await pilot.press("g")
|
||||||
|
await pilot.press("w")
|
||||||
|
await pilot.press("escape")
|
||||||
|
await pilot.pause()
|
||||||
|
|
||||||
|
header = str(app.query_one("#live-header").render())
|
||||||
|
assert "Writes" in header
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_historical_hour_uses_recorded_timezone_and_utc_identity(self, tmp_path):
|
||||||
|
"""Repeated local clock labels retain distinct UTC-hour identities."""
|
||||||
|
clock = _clock(2026, 11, 2, 12)
|
||||||
|
conn, _ = _setup_store(tmp_path, n_days=30, today=clock)
|
||||||
|
target = "2026-11-01"
|
||||||
|
_insert_local_day(
|
||||||
|
conn,
|
||||||
|
target,
|
||||||
|
tz_name="America/New_York",
|
||||||
|
tz_offset="-04:00",
|
||||||
|
utc_start="2026-11-01T04:00:00+00:00",
|
||||||
|
utc_end="2026-11-02T05:00:00+00:00",
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_day_unallocated_evidence "
|
||||||
|
"(start_sample_id, end_sample_id, start_local_date, end_local_date, "
|
||||||
|
" start_tz_name, end_tz_name, started_at, ended_at, bytes_written, "
|
||||||
|
" bytes_read, reason) VALUES (1, 2, ?, ?, ?, ?, ?, ?, ?, ?, 'local_midnight')",
|
||||||
|
(target, "2026-11-02", "America/New_York", "America/New_York",
|
||||||
|
"2026-11-02T03:59:00+00:00", "2026-11-02T04:01:00+00:00",
|
||||||
|
2_000_000_000, 3_000_000_000),
|
||||||
|
)
|
||||||
|
for index in range(25):
|
||||||
|
hour = datetime(2026, 11, 1, 4, tzinfo=timezone.utc) + timedelta(hours=index)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO hour_observations "
|
||||||
|
"(hour, bytes_written_delta, bytes_read_delta, coverage, sample_count) "
|
||||||
|
"VALUES (?, ?, ?, 1, 1)",
|
||||||
|
(hour.isoformat(), (index + 1) * 1_000_000_000,
|
||||||
|
(index + 1) * 2_000_000_000),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE hour_observations SET coverage = 0.5, unknown_seconds = 1800 "
|
||||||
|
"WHERE hour = ?",
|
||||||
|
("2026-11-01T04:00:00+00:00",),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
app, patcher = _make_app(tmp_path, clock=clock)
|
||||||
|
try:
|
||||||
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
|
await pilot.click("#view-history")
|
||||||
|
await pilot.press("g")
|
||||||
|
await pilot.press(*target)
|
||||||
|
await pilot.press("enter")
|
||||||
|
await pilot.pause()
|
||||||
|
|
||||||
|
local_day = str(app.query_one("#local-day").render())
|
||||||
|
assert "America/New_York -04:00" in local_day
|
||||||
|
assert "incomplete" in local_day
|
||||||
|
assert "shared at midnight W 2.000 GB · R 3.000 GB" in local_day
|
||||||
|
|
||||||
|
# Move from final local hour to the second 01:00 occurrence.
|
||||||
|
await pilot.press(*(["left"] * 22))
|
||||||
|
second_occurrence = str(app.query_one("#bar-readout").render())
|
||||||
|
assert "01:00 -0500" in second_occurrence
|
||||||
|
assert "America/New_York -0500" in second_occurrence
|
||||||
|
assert "06:00 UTC" in second_occurrence
|
||||||
|
|
||||||
|
await pilot.press("left")
|
||||||
|
first_occurrence = str(app.query_one("#bar-readout").render())
|
||||||
|
assert "01:00 -0400" in first_occurrence
|
||||||
|
assert "America/New_York -0400" in first_occurrence
|
||||||
|
assert "05:00 UTC" in first_occurrence
|
||||||
|
|
||||||
|
await pilot.press("left")
|
||||||
|
incomplete_hour = str(app.query_one("#bar-readout").render())
|
||||||
|
assert "00:00 -0400" in incomplete_hour
|
||||||
|
assert "50% coverage · incomplete" in incomplete_hour
|
||||||
|
finally:
|
||||||
|
patcher.stop()
|
||||||
|
|||||||
+13
-16
@@ -151,8 +151,8 @@ class TestDetailExpiresSummariesRemain:
|
|||||||
# Run pruning
|
# Run pruning
|
||||||
pruned = prune_old_samples(conn, now, retention_days=14)
|
pruned = prune_old_samples(conn, now, retention_days=14)
|
||||||
|
|
||||||
# Old samples should be pruned, recent ones retained
|
# The handcrafted totals lack replacement provenance, so retain samples.
|
||||||
assert pruned >= 1
|
assert pruned == 0
|
||||||
|
|
||||||
# Local-day summaries must still be queryable
|
# Local-day summaries must still be queryable
|
||||||
old_result = query_local_day_summary(conn, "2026-09-16")
|
old_result = query_local_day_summary(conn, "2026-09-16")
|
||||||
@@ -277,7 +277,7 @@ class TestTimezonePreservation:
|
|||||||
now = datetime(2026, 10, 1, 12, 0, 0, tzinfo=timezone.utc)
|
now = datetime(2026, 10, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
old = LocalDaySummary(
|
old = LocalDaySummary(
|
||||||
local_date="2026-09-16", tz_name="US/Eastern", tz_offset="-04:00",
|
local_date="2026-09-16", tz_name="America/New_York", tz_offset="-04:00",
|
||||||
utc_start="2026-09-16T04:00:00+00:00",
|
utc_start="2026-09-16T04:00:00+00:00",
|
||||||
utc_end="2026-09-17T04:00:00+00:00",
|
utc_end="2026-09-17T04:00:00+00:00",
|
||||||
bytes_written=5000, bytes_read=2000,
|
bytes_written=5000, bytes_read=2000,
|
||||||
@@ -288,7 +288,7 @@ class TestTimezonePreservation:
|
|||||||
entries = query_local_day_history(conn, "2026-09-16", "2026-09-16", now)
|
entries = query_local_day_history(conn, "2026-09-16", "2026-09-16", now)
|
||||||
assert len(entries) == 1
|
assert len(entries) == 1
|
||||||
entry = entries[0]
|
entry = entries[0]
|
||||||
assert entry.tz_name == "US/Eastern"
|
assert entry.tz_name == "America/New_York"
|
||||||
assert entry.tz_offset == "-04:00"
|
assert entry.tz_offset == "-04:00"
|
||||||
assert entry.utc_start == "2026-09-16T04:00:00+00:00"
|
assert entry.utc_start == "2026-09-16T04:00:00+00:00"
|
||||||
assert entry.detail_available is False
|
assert entry.detail_available is False
|
||||||
@@ -389,10 +389,10 @@ class TestIdempotencySafety:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
class TestVolumeConservation:
|
class TestVolumeConservation:
|
||||||
"""AC6: Summary volume plus unallocated evidence conserves each delta once."""
|
"""AC6: UTC-hour totals never become fabricated local-day totals."""
|
||||||
|
|
||||||
def test_midnight_spanning_not_double_counted(self, tmp_path):
|
def test_hourly_bytes_are_unavailable_at_local_precision(self, tmp_path):
|
||||||
"""Midnight-spanning interval appears once, not in both days."""
|
"""UTC-day hour bytes cannot be split across local-day boundaries."""
|
||||||
conn = init_store(tmp_path / "obs.db")
|
conn = init_store(tmp_path / "obs.db")
|
||||||
|
|
||||||
# Insert hour observations that span midnight
|
# Insert hour observations that span midnight
|
||||||
@@ -409,13 +409,10 @@ class TestVolumeConservation:
|
|||||||
summary_17 = derive_local_day_summary(conn, "UTC", clock_17)
|
summary_17 = derive_local_day_summary(conn, "UTC", clock_17)
|
||||||
assert summary_17 is not None
|
assert summary_17 is not None
|
||||||
|
|
||||||
# Each day should only have its own hour's bytes
|
assert summary_16.bytes_written == 0
|
||||||
# Day 16 has the 23:00 hour (100 bw)
|
assert summary_17.bytes_written == 0
|
||||||
# Day 17 has the 00:00 hour (200 bw)
|
assert summary_16.activity_precision == "unavailable"
|
||||||
assert summary_16.bytes_written == 100
|
assert summary_17.activity_precision == "unavailable"
|
||||||
assert summary_17.bytes_written == 200
|
|
||||||
# Total is conserved: 100 + 200 = 300
|
|
||||||
assert summary_16.bytes_written + summary_17.bytes_written == 300
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
@@ -437,10 +434,10 @@ class TestExistingEntryPoints:
|
|||||||
_insert_sample(conn, ts, bw=days_ago * 100)
|
_insert_sample(conn, ts, bw=days_ago * 100)
|
||||||
|
|
||||||
pruned = prune_old_samples(conn, now, retention_days=14)
|
pruned = prune_old_samples(conn, now, retention_days=14)
|
||||||
assert pruned == 15
|
assert pruned == 0
|
||||||
|
|
||||||
cursor = conn.execute("SELECT COUNT(*) FROM samples")
|
cursor = conn.execute("SELECT COUNT(*) FROM samples")
|
||||||
assert cursor.fetchone()[0] == 14
|
assert cursor.fetchone()[0] == 29
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
def test_repair_with_real_temp_store(self, tmp_path):
|
def test_repair_with_real_temp_store(self, tmp_path):
|
||||||
|
|||||||
@@ -0,0 +1,294 @@
|
|||||||
|
"""Pending publication behavior from issue #97."""
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from fenris import collector
|
||||||
|
from fenris.projection import compute_projection
|
||||||
|
from fenris.status import _query_drive_facts, get_status, read_status
|
||||||
|
from fenris.status_composition import render_status_tui
|
||||||
|
from fenris.store import init_store
|
||||||
|
|
||||||
|
|
||||||
|
class FakeClock:
|
||||||
|
def __init__(self, now):
|
||||||
|
self.now = now
|
||||||
|
|
||||||
|
def utcnow(self):
|
||||||
|
return self.now
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sysfs_fixture_tree(tmp_path):
|
||||||
|
ctrl_dir = tmp_path / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
ctrl_dir.mkdir(parents=True)
|
||||||
|
(ctrl_dir / "subsysnqn").write_text("nqn.test:drive\n")
|
||||||
|
(ctrl_dir / "model").write_text("Test NVMe\n")
|
||||||
|
(ctrl_dir / "serial").write_text("test-serial\n")
|
||||||
|
(ctrl_dir / "firmware_rev").write_text("1.0\n")
|
||||||
|
transport_dir = ctrl_dir / "transport"
|
||||||
|
transport_dir.mkdir()
|
||||||
|
(transport_dir / "trstring").write_text("pcie\n")
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
def _smartctl(unit_written, unit_read=500):
|
||||||
|
return {
|
||||||
|
"nvme_smart_health_information_log": {
|
||||||
|
"critical_warning": 0,
|
||||||
|
"temperature": 35,
|
||||||
|
"available_spare": 100,
|
||||||
|
"percentage_used": 5,
|
||||||
|
"data_units_written": unit_written,
|
||||||
|
"data_units_read": unit_read,
|
||||||
|
"power_on_hours": 100,
|
||||||
|
},
|
||||||
|
"user_capacity": {"bytes": 1_024_000_000_000},
|
||||||
|
"model_name": "Test NVMe",
|
||||||
|
"serial_number": "test-serial",
|
||||||
|
"firmware_version": "1.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _acquirer(sysfs_path, unit_written, unit_read=500):
|
||||||
|
return lambda: (_smartctl(unit_written, unit_read), sysfs_path)
|
||||||
|
|
||||||
|
|
||||||
|
def _public_snapshot(store_path, clock_now):
|
||||||
|
with read_status(store_path, clock_now) as (conn, comp):
|
||||||
|
assert conn is not None
|
||||||
|
return {
|
||||||
|
"state": comp.state,
|
||||||
|
"freshness": comp.freshness,
|
||||||
|
"sample_count": comp.sample_count,
|
||||||
|
"day_count": comp.day_count,
|
||||||
|
"boot_enabled": comp.boot_enabled,
|
||||||
|
"timer_active": comp.timer_active,
|
||||||
|
"last_collect_ok": comp.last_collect_ok,
|
||||||
|
"latest_sample": tuple(conn.execute(
|
||||||
|
"SELECT ts, subnqn, mn, sn, fr, capacity_bytes, temperature_c, "
|
||||||
|
"available_spare, data_units_written, bytes_written, critical_warning, "
|
||||||
|
"media_errors, unsafe_shutdowns "
|
||||||
|
"FROM samples ORDER BY id DESC LIMIT 1"
|
||||||
|
).fetchone()),
|
||||||
|
"drive_facts": tuple(_query_drive_facts(conn)),
|
||||||
|
"projection": repr(compute_projection(conn, clock_now)),
|
||||||
|
"segments": conn.execute("SELECT COUNT(*) FROM controller_segments").fetchone()[0],
|
||||||
|
"periods": conn.execute("SELECT COUNT(*) FROM monitoring_periods").fetchone()[0],
|
||||||
|
"hours": tuple(conn.execute(
|
||||||
|
"SELECT hour, bytes_written_delta, bytes_read_delta, sample_count "
|
||||||
|
"FROM hour_observations ORDER BY hour"
|
||||||
|
).fetchall()),
|
||||||
|
"days": tuple(conn.execute(
|
||||||
|
"SELECT day, bytes_written_delta, bytes_read_delta "
|
||||||
|
"FROM day_aggregates ORDER BY day"
|
||||||
|
).fetchall()),
|
||||||
|
"local_days": tuple(conn.execute(
|
||||||
|
"SELECT local_date, bytes_written, bytes_read, sample_count "
|
||||||
|
"FROM local_days ORDER BY local_date, tz_name"
|
||||||
|
).fetchall()),
|
||||||
|
"pending": comp.pending_publication_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_publication_survives_restart_and_readers_keep_last_consistent_view(
|
||||||
|
tmp_path, sysfs_fixture_tree, monkeypatch
|
||||||
|
):
|
||||||
|
"""A failed derivation stays private, visible as pending, and replays in order."""
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
service = {"boot_enabled": True, "timer_active": True, "last_collect_ok": True}
|
||||||
|
monkeypatch.setattr("fenris.status.query_service_state", lambda: service)
|
||||||
|
now = datetime.now(timezone.utc).replace(
|
||||||
|
minute=35, second=0, microsecond=0,
|
||||||
|
)
|
||||||
|
store_path = tmp_path / "observations.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
sysfs_path = sysfs_fixture_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
|
||||||
|
for units, at in ((1000, now - timedelta(minutes=10)), (1010, now - timedelta(minutes=5))):
|
||||||
|
result = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(at),
|
||||||
|
acquire=_acquirer(sysfs_path, units),
|
||||||
|
)
|
||||||
|
assert result["ok"] is True, result
|
||||||
|
|
||||||
|
previous = _public_snapshot(store_path, now)
|
||||||
|
baseline_cli = get_status(store_path, now, query_services=False, query_journal=False)
|
||||||
|
original_derive = collector.derive_hours_from_interval
|
||||||
|
reader_during_write = {}
|
||||||
|
|
||||||
|
def fail_derivation(conn, previous_sample, next_sample):
|
||||||
|
# Read while writer has inserted the unpublished sample but not committed.
|
||||||
|
with read_status(store_path, now, query_services=False) as (reader, comp):
|
||||||
|
assert reader is not None
|
||||||
|
reader_during_write["samples"] = reader.execute(
|
||||||
|
"SELECT COUNT(*) FROM samples"
|
||||||
|
).fetchone()[0]
|
||||||
|
reader_during_write["pending"] = comp.pending_publication_count
|
||||||
|
reader_during_write["latest"] = tuple(reader.execute(
|
||||||
|
"SELECT data_units_written FROM samples ORDER BY id DESC LIMIT 1"
|
||||||
|
).fetchone())
|
||||||
|
raise RuntimeError("injected non-invariant derivation failure")
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "derive_hours_from_interval", fail_derivation)
|
||||||
|
failed = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now),
|
||||||
|
acquire=_acquirer(sysfs_path, 1025),
|
||||||
|
)
|
||||||
|
assert failed["ok"] is False
|
||||||
|
assert "injected non-invariant derivation failure" in failed["error"]
|
||||||
|
assert reader_during_write == {"samples": 2, "pending": 1, "latest": (1010,)}
|
||||||
|
|
||||||
|
after_failure = _public_snapshot(store_path, now)
|
||||||
|
assert {**after_failure, "pending": previous["pending"]} == previous
|
||||||
|
assert after_failure["pending"] == 1
|
||||||
|
|
||||||
|
cli_text = get_status(store_path, now, query_services=False, query_journal=False)
|
||||||
|
pending_fact = " · pending publication: 1 observation retained for retry"
|
||||||
|
assert pending_fact in cli_text
|
||||||
|
assert cli_text.replace(pending_fact, "", 1) == baseline_cli
|
||||||
|
|
||||||
|
with read_status(store_path, now) as (_, comp):
|
||||||
|
assert comp.state == previous["state"]
|
||||||
|
assert comp.boot_enabled is True
|
||||||
|
assert comp.timer_active is True
|
||||||
|
assert "pending publication: 1 observation retained for retry" in render_status_tui(comp).lower()
|
||||||
|
|
||||||
|
called = False
|
||||||
|
|
||||||
|
def should_wait_for_recovery():
|
||||||
|
nonlocal called
|
||||||
|
called = True
|
||||||
|
return _smartctl(1030), sysfs_path
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "PENDING_PUBLICATION_LIMIT", 1)
|
||||||
|
blocked = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now + timedelta(minutes=5)),
|
||||||
|
acquire=should_wait_for_recovery,
|
||||||
|
)
|
||||||
|
assert blocked["ok"] is False
|
||||||
|
assert called is False
|
||||||
|
assert _public_snapshot(store_path, now)["pending"] == 1
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "derive_hours_from_interval", original_derive)
|
||||||
|
monkeypatch.setenv("TZ", "Asia/Kolkata")
|
||||||
|
recovered = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now + timedelta(minutes=5)),
|
||||||
|
acquire=_acquirer(sysfs_path, 1040),
|
||||||
|
)
|
||||||
|
assert recovered["ok"] is True, recovered
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store_path)
|
||||||
|
try:
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT data_units_written FROM samples ORDER BY id"
|
||||||
|
).fetchall() == [(1000,), (1010,), (1025,), (1040,)]
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 0
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT SUM(bytes_written_delta) FROM day_aggregates"
|
||||||
|
).fetchone()[0] == (1040 - 1000) * 512_000
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT sample_count FROM local_days WHERE tz_name = 'UTC'"
|
||||||
|
).fetchone()[0] == 4
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM local_days WHERE tz_name = 'Asia/Kolkata'"
|
||||||
|
).fetchone()[0] == 1
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_sample_writes_no_sample_segment_period_or_pending_row(tmp_path, sysfs_fixture_tree):
|
||||||
|
"""Sample invariants run before staging or public monitoring changes."""
|
||||||
|
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
|
||||||
|
store_path = tmp_path / "observations.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
sysfs_path = sysfs_fixture_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
result = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now),
|
||||||
|
acquire=_acquirer(sysfs_path, -1),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert result["error_type"] == "InvariantViolationError"
|
||||||
|
conn = sqlite3.connect(store_path)
|
||||||
|
try:
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM samples").fetchone()[0] == 0
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM controller_segments").fetchone()[0] == 0
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM monitoring_periods").fetchone()[0] == 0
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 0
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_capacity_retries_before_skipping_acquisition(tmp_path, sysfs_fixture_tree, monkeypatch):
|
||||||
|
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
|
||||||
|
store_path = tmp_path / "observations.db"
|
||||||
|
config = {"device": "/dev/nvme0", "store_path": str(store_path)}
|
||||||
|
sysfs_path = sysfs_fixture_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
first = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now - timedelta(minutes=5)),
|
||||||
|
acquire=_acquirer(sysfs_path, 1000),
|
||||||
|
)
|
||||||
|
assert first["ok"] is True, first
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "PENDING_PUBLICATION_LIMIT", 1)
|
||||||
|
|
||||||
|
def fail_derivation(*_):
|
||||||
|
raise RuntimeError("derivation still unavailable")
|
||||||
|
|
||||||
|
monkeypatch.setattr(collector, "derive_hours_from_interval", fail_derivation)
|
||||||
|
failed = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now),
|
||||||
|
acquire=_acquirer(sysfs_path, 1010),
|
||||||
|
)
|
||||||
|
assert failed["ok"] is False
|
||||||
|
assert "capacity full (1 observations)" in failed["error"]
|
||||||
|
|
||||||
|
called = False
|
||||||
|
|
||||||
|
def must_not_acquire():
|
||||||
|
nonlocal called
|
||||||
|
called = True
|
||||||
|
return _smartctl(1020), sysfs_path
|
||||||
|
|
||||||
|
retry = collector.run_collection(
|
||||||
|
config=config,
|
||||||
|
clock=FakeClock(now + timedelta(minutes=5)),
|
||||||
|
acquire=must_not_acquire,
|
||||||
|
)
|
||||||
|
assert retry["ok"] is False
|
||||||
|
assert "capacity full (1 observations)" in retry["error"]
|
||||||
|
assert called is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_v3_readers_ignore_pending_table_until_store_migrates(tmp_path):
|
||||||
|
store_path = tmp_path / "observations.db"
|
||||||
|
conn = init_store(store_path)
|
||||||
|
conn.execute("DROP TABLE pending_publications")
|
||||||
|
conn.execute("PRAGMA user_version=3")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
with read_status(store_path, datetime.now(timezone.utc), query_services=False) as (reader, comp):
|
||||||
|
assert reader is not None
|
||||||
|
assert comp.store_fault is None
|
||||||
|
assert comp.pending_publication_count == 0
|
||||||
|
|
||||||
|
migrated = init_store(store_path)
|
||||||
|
try:
|
||||||
|
assert migrated.execute("PRAGMA user_version").fetchone()[0] == 6
|
||||||
|
assert migrated.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0] == 0
|
||||||
|
finally:
|
||||||
|
migrated.close()
|
||||||
@@ -0,0 +1,520 @@
|
|||||||
|
"""Historical local-day migration and repair acceptance tests for issue #99."""
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from fenris.local_day import (
|
||||||
|
query_local_day_summary,
|
||||||
|
record_local_activity_interval,
|
||||||
|
)
|
||||||
|
from fenris.status import read_status
|
||||||
|
from fenris.store import init_store
|
||||||
|
|
||||||
|
SOURCE_SCHEMA_VERSION = 5
|
||||||
|
LOCAL_DATE = "2026-09-02"
|
||||||
|
TIMEZONE = "Asia/Kolkata"
|
||||||
|
UTC_START = "2026-09-01T18:30:00+00:00"
|
||||||
|
UTC_END = "2026-09-02T18:30:00+00:00"
|
||||||
|
|
||||||
|
|
||||||
|
def _legacy_store(path: Path) -> sqlite3.Connection:
|
||||||
|
"""Build a v5 store with one old coarse local-day summary."""
|
||||||
|
conn = init_store(path)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_days "
|
||||||
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, bytes_written, "
|
||||||
|
" bytes_read, coverage, sample_count, complete, activity_incomplete, "
|
||||||
|
" activity_precision) "
|
||||||
|
"VALUES (?, ?, '+05:30', ?, ?, 999999, 888888, 1.0, 24, 1, 0, 'legacy')",
|
||||||
|
(LOCAL_DATE, TIMEZONE, UTC_START, UTC_END),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO monitoring_periods (started_at, ended_at) VALUES (?, ?)",
|
||||||
|
(UTC_START, "2026-09-03T00:00:00+00:00"),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO controller_segments "
|
||||||
|
"(opened_at, identity_key, identity_degraded) VALUES (?, ?, 0)",
|
||||||
|
(UTC_START, "controller-a"),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO hour_observations "
|
||||||
|
"(hour, bytes_written_delta, bytes_read_delta, sample_count, coverage) "
|
||||||
|
"VALUES ('2026-09-02T00:00:00+00:00', 123, 45, 2, 1.0)"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO day_aggregates "
|
||||||
|
"(day, bytes_written_delta, bytes_read_delta, coverage) "
|
||||||
|
"VALUES ('2026-09-02', 678, 90, 1.0)"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, power_on_hours, "
|
||||||
|
" segment_id, local_tz) VALUES (?, '/dev/nvme0', 1000, 2000, 100, 1, ?)",
|
||||||
|
("2026-09-01T19:00:00+00:00", TIMEZONE),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, power_on_hours, "
|
||||||
|
" segment_id, local_tz) VALUES (?, '/dev/nvme0', 11000, 5000, 115, 1, ?)",
|
||||||
|
("2026-09-02T10:00:00+00:00", TIMEZONE),
|
||||||
|
)
|
||||||
|
conn.execute(f"PRAGMA user_version={SOURCE_SCHEMA_VERSION}")
|
||||||
|
conn.commit()
|
||||||
|
return conn
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_rebuilds_legacy_day_from_samples_and_keeps_recorded_bounds(
|
||||||
|
tmp_path, monkeypatch,
|
||||||
|
):
|
||||||
|
db = tmp_path / "legacy.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.close()
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
with read_status(
|
||||||
|
db, datetime(2026, 9, 4, tzinfo=timezone.utc), query_services=False,
|
||||||
|
) as (reader, composition):
|
||||||
|
assert reader is not None
|
||||||
|
assert composition.store_fault is None
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
reader, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary is not None
|
||||||
|
assert summary["local_date"] == LOCAL_DATE
|
||||||
|
assert summary["tz_name"] == TIMEZONE
|
||||||
|
assert summary["tz_offset"] == "+05:30"
|
||||||
|
assert summary["utc_start"] == UTC_START
|
||||||
|
assert summary["utc_end"] == UTC_END
|
||||||
|
assert summary["bytes_written"] == 10_000
|
||||||
|
assert summary["bytes_read"] == 3_000
|
||||||
|
assert summary["activity_precision"] == "measured"
|
||||||
|
assert summary["segments"] == [{
|
||||||
|
"segment_id": 1,
|
||||||
|
"bytes_written": 10_000,
|
||||||
|
"bytes_read": 3_000,
|
||||||
|
"activity_seconds": 54_000,
|
||||||
|
"activity_intervals": 1,
|
||||||
|
}]
|
||||||
|
assert summary["activity_state"] == "incomplete"
|
||||||
|
assert reader.execute(
|
||||||
|
"SELECT bytes_written_delta FROM hour_observations"
|
||||||
|
).fetchone()[0] == 123
|
||||||
|
assert reader.execute(
|
||||||
|
"SELECT bytes_written_delta FROM day_aggregates"
|
||||||
|
).fetchone()[0] == 678
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
conn.close()
|
||||||
|
with read_status(
|
||||||
|
db, datetime(2026, 9, 4, tzinfo=timezone.utc), query_services=False,
|
||||||
|
) as (reader, _composition):
|
||||||
|
assert reader is not None
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
reader, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 10_000
|
||||||
|
assert summary["bytes_read"] == 3_000
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_rebuilds_legacy_local_day_once_across_restarts(tmp_path):
|
||||||
|
from fenris.repair import repair_derivation
|
||||||
|
|
||||||
|
db = tmp_path / "repair.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
|
||||||
|
first = repair_derivation(conn)
|
||||||
|
assert first.ok
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 10_000
|
||||||
|
assert summary["bytes_read"] == 3_000
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
conn = sqlite3.connect(db)
|
||||||
|
second = repair_derivation(conn)
|
||||||
|
assert second.ok
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 10_000
|
||||||
|
assert summary["bytes_read"] == 3_000
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT bytes_written, bytes_read, activity_intervals "
|
||||||
|
"FROM local_day_segment_totals"
|
||||||
|
).fetchall() == [(10_000, 3_000, 1)]
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_interrupted_repair_keeps_local_day_rebuild_retryable(tmp_path):
|
||||||
|
from fenris.repair import repair_derivation
|
||||||
|
|
||||||
|
db = tmp_path / "repair-interrupted.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TRIGGER interrupt_repair_local_rebuild "
|
||||||
|
"BEFORE INSERT ON local_day_segment_totals "
|
||||||
|
"BEGIN SELECT RAISE(ABORT, 'simulated repair interruption'); END"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
failed = repair_derivation(conn)
|
||||||
|
assert not failed.ok
|
||||||
|
assert query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)["bytes_written"] is None
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT activity_precision, bytes_written FROM local_days"
|
||||||
|
).fetchone() == ("legacy", 999_999)
|
||||||
|
conn.execute("DROP TRIGGER interrupt_repair_local_rebuild")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db)
|
||||||
|
retried = repair_derivation(conn)
|
||||||
|
assert retried.ok
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 10_000
|
||||||
|
assert summary["bytes_read"] == 3_000
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_repair_marker_does_not_block_crash_retry(tmp_path):
|
||||||
|
from fenris.repair import repair_derivation
|
||||||
|
|
||||||
|
conn = _legacy_store(tmp_path / "stale-repair.db")
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO store_metadata (key, value) "
|
||||||
|
"VALUES ('repair_in_progress', 'true')"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
result = repair_derivation(conn)
|
||||||
|
|
||||||
|
assert result.ok
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT value FROM store_metadata WHERE key = 'repair_in_progress'"
|
||||||
|
).fetchone() == ("false",)
|
||||||
|
assert query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)["bytes_written"] == 10_000
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_uses_whole_utc_hours_but_skips_midnight_straddling_hours(
|
||||||
|
tmp_path,
|
||||||
|
):
|
||||||
|
db = tmp_path / "coarse.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute("DELETE FROM samples")
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO hour_observations "
|
||||||
|
"(hour, bytes_written_delta, bytes_read_delta, sample_count, "
|
||||||
|
" active_seconds, coverage) "
|
||||||
|
"VALUES ('2026-09-01T18:00:00+00:00', 99999, 9999, 2, 3600, 1.0)"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO hour_observations "
|
||||||
|
"(hour, bytes_written_delta, bytes_read_delta, sample_count, "
|
||||||
|
" active_seconds, coverage) "
|
||||||
|
"VALUES ('2026-09-02T18:00:00+00:00', 88888, 8888, 2, 3600, 1.0)"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 123
|
||||||
|
assert summary["bytes_read"] == 45
|
||||||
|
assert summary["activity_precision"] == "coarse"
|
||||||
|
assert summary["activity_state"] == "incomplete"
|
||||||
|
assert summary["utc_start"] == UTC_START
|
||||||
|
assert summary["utc_end"] == UTC_END
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples "
|
||||||
|
"(id, ts, device, bytes_written, bytes_read, segment_id, local_tz) "
|
||||||
|
"VALUES (10, '2026-09-02T12:00:00+00:00', '/dev/nvme0', 1000, 2000, 1, ?), "
|
||||||
|
" (11, '2026-09-02T12:05:00+00:00', '/dev/nvme0', 2000, 2500, 1, ?)",
|
||||||
|
(TIMEZONE, TIMEZONE),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
assert record_local_activity_interval(
|
||||||
|
conn,
|
||||||
|
{
|
||||||
|
"id": 10, "ts": "2026-09-02T12:00:00+00:00",
|
||||||
|
"bytes_written": 1_000, "bytes_read": 2_000,
|
||||||
|
"local_tz": TIMEZONE,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11, "ts": "2026-09-02T12:05:00+00:00",
|
||||||
|
"bytes_written": 2_000, "bytes_read": 2_500,
|
||||||
|
"local_tz": TIMEZONE,
|
||||||
|
},
|
||||||
|
start_sample_id=10, end_sample_id=11, segment_id=1,
|
||||||
|
) == "known"
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 1_000
|
||||||
|
assert summary["bytes_read"] == 500
|
||||||
|
assert summary["activity_precision"] == "measured"
|
||||||
|
assert summary["segments"] == [{
|
||||||
|
"segment_id": 1,
|
||||||
|
"bytes_written": 1_000,
|
||||||
|
"bytes_read": 500,
|
||||||
|
"activity_seconds": 300,
|
||||||
|
"activity_intervals": 1,
|
||||||
|
}]
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_does_not_double_count_coarse_hours_in_overlapping_days(tmp_path):
|
||||||
|
db = tmp_path / "overlapping-coarse-days.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute("DELETE FROM samples")
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_days "
|
||||||
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, bytes_written, "
|
||||||
|
" bytes_read, coverage, sample_count, complete, activity_precision) "
|
||||||
|
"VALUES ('2026-09-02', 'UTC', '+00:00', "
|
||||||
|
"'2026-09-02T00:00:00+00:00', '2026-09-03T00:00:00+00:00', "
|
||||||
|
"999999, 888888, 1.0, 24, 1, 'legacy')"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
|
||||||
|
for tz_name in (TIMEZONE, "UTC"):
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, tz_name,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] is None
|
||||||
|
assert summary["bytes_read"] is None
|
||||||
|
assert summary["activity_precision"] == "unavailable"
|
||||||
|
assert summary["activity_state"] == "unavailable"
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT bytes_written_delta, bytes_read_delta FROM hour_observations "
|
||||||
|
"WHERE hour = '2026-09-02T00:00:00+00:00'"
|
||||||
|
).fetchone() == (123, 45)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_retains_local_midnight_interval_once_as_shared_evidence(tmp_path):
|
||||||
|
db = tmp_path / "shared.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_days "
|
||||||
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, bytes_written, "
|
||||||
|
" bytes_read, coverage, sample_count, complete, activity_precision) "
|
||||||
|
"VALUES ('2026-09-01', ?, '+05:30', "
|
||||||
|
"'2026-08-31T18:30:00+00:00', '2026-09-01T18:30:00+00:00', "
|
||||||
|
"500, 600, 1.0, 24, 1, 'legacy')",
|
||||||
|
(TIMEZONE,),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE monitoring_periods SET started_at = '2026-08-31T18:00:00+00:00'"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE controller_segments SET opened_at = '2026-08-31T18:00:00+00:00'"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE samples SET ts = '2026-09-01T18:25:00+00:00', "
|
||||||
|
"bytes_written = 1000, bytes_read = 2000 WHERE id = 1"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE samples SET ts = '2026-09-01T18:35:00+00:00', "
|
||||||
|
"bytes_written = 1100, bytes_read = 2050 WHERE id = 2"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
shared = conn.execute(
|
||||||
|
"SELECT COUNT(*), SUM(bytes_written), SUM(bytes_read) "
|
||||||
|
"FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone()
|
||||||
|
assert shared == (1, 100, 50)
|
||||||
|
for local_date in ("2026-09-01", "2026-09-02"):
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, local_date, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] is None
|
||||||
|
assert summary["shared_bytes_written"] == 100
|
||||||
|
assert summary["shared_bytes_read"] == 50
|
||||||
|
assert summary["activity_state"] == "incomplete"
|
||||||
|
from fenris.repair import repair_derivation
|
||||||
|
assert repair_derivation(conn).ok
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*), SUM(bytes_written), SUM(bytes_read) "
|
||||||
|
"FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone() == (1, 100, 50)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_interrupted_upgrade_keeps_legacy_day_retryable(tmp_path):
|
||||||
|
db = tmp_path / "interrupted.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TRIGGER interrupt_local_rebuild "
|
||||||
|
"BEFORE INSERT ON local_day_segment_totals "
|
||||||
|
"BEGIN SELECT RAISE(ABORT, 'simulated interruption'); END"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
with pytest.raises(sqlite3.IntegrityError, match="simulated interruption"):
|
||||||
|
init_store(db)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db)
|
||||||
|
assert conn.execute("PRAGMA user_version").fetchone() == (SOURCE_SCHEMA_VERSION,)
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT activity_precision, bytes_written FROM local_days"
|
||||||
|
).fetchone() == ("legacy", 999_999)
|
||||||
|
conn.execute("DROP TRIGGER interrupt_local_rebuild")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 10_000
|
||||||
|
assert summary["bytes_read"] == 3_000
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_prior_migration_step_commits_before_local_day_repair(tmp_path):
|
||||||
|
db = tmp_path / "migration-steps.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute("PRAGMA user_version=4")
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TRIGGER interrupt_local_rebuild BEFORE UPDATE OF bytes_written "
|
||||||
|
"ON local_days BEGIN SELECT RAISE(ABORT, 'simulated interruption'); END"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
with pytest.raises(sqlite3.IntegrityError, match="simulated interruption"):
|
||||||
|
init_store(db)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db)
|
||||||
|
assert conn.execute("PRAGMA user_version").fetchone() == (5,)
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT 1 FROM sqlite_master WHERE type='table' "
|
||||||
|
"AND name='local_day_segment_totals'"
|
||||||
|
).fetchone() is not None
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT bytes_written FROM local_days WHERE local_date=?",
|
||||||
|
(LOCAL_DATE,),
|
||||||
|
).fetchone() == (999_999,)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_does_not_allocate_unzoned_interval_to_overlapping_timezones(
|
||||||
|
tmp_path,
|
||||||
|
):
|
||||||
|
db = tmp_path / "overlapping-timezones.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_days "
|
||||||
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, bytes_written, "
|
||||||
|
" bytes_read, coverage, sample_count, complete, activity_precision) "
|
||||||
|
"VALUES ('2026-09-02', 'UTC', '+00:00', "
|
||||||
|
"'2026-09-02T00:00:00+00:00', '2026-09-03T00:00:00+00:00', "
|
||||||
|
"999999, 888888, 1.0, 24, 1, 'legacy')"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE samples SET ts = '2026-09-02T01:00:00+00:00', local_tz = NULL "
|
||||||
|
"WHERE id = 1"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE samples SET ts = '2026-09-02T10:00:00+00:00', local_tz = NULL "
|
||||||
|
"WHERE id = 2"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
for tz_name in (TIMEZONE, "UTC"):
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, tz_name,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] is None
|
||||||
|
assert summary["bytes_read"] is None
|
||||||
|
assert summary["activity_precision"] == "unavailable"
|
||||||
|
assert summary["activity_state"] == "unavailable"
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_rejects_interval_before_recorded_controller_segment(
|
||||||
|
tmp_path,
|
||||||
|
):
|
||||||
|
db = tmp_path / "segment-boundary.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE controller_segments "
|
||||||
|
"SET opened_at = '2026-09-01T20:00:00+00:00' WHERE id = 1"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] is None
|
||||||
|
assert summary["bytes_read"] is None
|
||||||
|
assert summary["activity_precision"] == "unavailable"
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sample_at_local_midnight_does_not_hide_coarse_reconstruction(tmp_path):
|
||||||
|
db = tmp_path / "midnight-sample.db"
|
||||||
|
conn = _legacy_store(db)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE samples SET ts = '2026-09-01T17:00:00+00:00' WHERE id = 1"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE samples SET ts = ?, local_tz = ? WHERE id = 2",
|
||||||
|
(UTC_END, TIMEZONE),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, LOCAL_DATE, TIMEZONE,
|
||||||
|
datetime(2026, 9, 4, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert summary["bytes_written"] == 123
|
||||||
|
assert summary["bytes_read"] == 45
|
||||||
|
assert summary["activity_precision"] == "coarse"
|
||||||
|
assert summary["activity_state"] == "incomplete"
|
||||||
|
conn.close()
|
||||||
+108
-7
@@ -6,26 +6,24 @@ Covers:
|
|||||||
- Three-minute cadence constants
|
- Three-minute cadence constants
|
||||||
- TUI integration with live graph
|
- TUI integration with live graph
|
||||||
"""
|
"""
|
||||||
|
import sys
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import sys
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
from fenris.store import init_store
|
from fenris.activity_selection import IntervalIdentity
|
||||||
from fenris.monitoring_periods import ensure_period_open
|
from fenris.monitoring_periods import ensure_period_open
|
||||||
|
from fenris.status import ACCURACY_SEC, CADENCE_DEFAULT_S, FRESH_THRESHOLD_S
|
||||||
|
from fenris.store import init_store
|
||||||
from fenris.tui import (
|
from fenris.tui import (
|
||||||
|
LIVE_WINDOW_H,
|
||||||
FenrisTuiApp,
|
FenrisTuiApp,
|
||||||
LiveActivityGraph,
|
LiveActivityGraph,
|
||||||
_query_live_graph_data,
|
_query_live_graph_data,
|
||||||
LIVE_WINDOW_H,
|
|
||||||
_query_daily_graph_data,
|
|
||||||
_RANGE_OPTIONS,
|
|
||||||
)
|
)
|
||||||
from fenris.status import CADENCE_DEFAULT_S, FRESH_THRESHOLD_S, ACCURACY_SEC
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Helpers
|
# Helpers
|
||||||
@@ -63,6 +61,27 @@ def _open_period(conn, start="2026-09-01T00:00:00+00:00"):
|
|||||||
ensure_period_open(conn, datetime.fromisoformat(start))
|
ensure_period_open(conn, datetime.fromisoformat(start))
|
||||||
|
|
||||||
|
|
||||||
|
def _live_intervals(count=4):
|
||||||
|
first = datetime(2026, 9, 28, 12, 0, tzinfo=timezone.utc)
|
||||||
|
intervals = []
|
||||||
|
for index in range(count):
|
||||||
|
start = first + timedelta(minutes=3 * index)
|
||||||
|
end = start + timedelta(minutes=3)
|
||||||
|
intervals.append({
|
||||||
|
"start_ts": start.isoformat(),
|
||||||
|
"end_ts": end.isoformat(),
|
||||||
|
"start_label": start.strftime("%H:%M"),
|
||||||
|
"end_label": end.strftime("%H:%M"),
|
||||||
|
"bytes_written": (index + 1) * 1000,
|
||||||
|
"bytes_read": (index + 1) * 500,
|
||||||
|
"elapsed_s": 180,
|
||||||
|
"is_gap": False,
|
||||||
|
"is_zero": False,
|
||||||
|
"is_segment_boundary": False,
|
||||||
|
})
|
||||||
|
return intervals
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Cadence constants (issue #91 AC1)
|
# Cadence constants (issue #91 AC1)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -322,3 +341,85 @@ class TestTUILiveIntegration:
|
|||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
main_grid = app.query_one("#main-grid")
|
main_grid = app.query_one("#main-grid")
|
||||||
assert main_grid.has_class("constrained")
|
assert main_grid.has_class("constrained")
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("size", [(100, 40), (70, 20)])
|
||||||
|
async def test_follow_pin_expiry_and_today_use_visible_readout(self, tmp_path, size):
|
||||||
|
"""Live selection follows, pins, expires, and resets at both layout sizes."""
|
||||||
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
|
intervals = _live_intervals()
|
||||||
|
async with app.run_test(size=size) as pilot:
|
||||||
|
await pilot.pause()
|
||||||
|
graph = app.query_one("#live-activity")
|
||||||
|
graph.set_data(intervals[:3])
|
||||||
|
|
||||||
|
readout = str(app.query_one("#live-readout").render())
|
||||||
|
assert "12:09 UTC" in readout
|
||||||
|
assert graph.selection.following_live
|
||||||
|
assert graph.selection.selected_live_interval == IntervalIdentity(
|
||||||
|
intervals[2]["start_ts"], intervals[2]["end_ts"]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Refresh while following selects the newly measured newest point.
|
||||||
|
graph.set_data(intervals)
|
||||||
|
assert "12:12 UTC" in str(app.query_one("#live-readout").render())
|
||||||
|
|
||||||
|
# Live graph receives focus at launch; first arrow pins immediately.
|
||||||
|
await pilot.press("left")
|
||||||
|
pinned = graph.selection.selected_live_interval
|
||||||
|
readout = str(app.query_one("#live-readout").render())
|
||||||
|
assert "12:09 UTC" in readout
|
||||||
|
assert not graph.selection.following_live
|
||||||
|
|
||||||
|
graph.set_data(intervals)
|
||||||
|
readout = str(app.query_one("#live-readout").render())
|
||||||
|
assert "12:09 UTC" in readout
|
||||||
|
assert graph.selection.selected_live_interval == pinned
|
||||||
|
|
||||||
|
# Read/write changes presentation only; pinned point stays stable.
|
||||||
|
await pilot.press("w")
|
||||||
|
graph.set_data(intervals)
|
||||||
|
assert graph.measure == "read"
|
||||||
|
assert graph.selection.selected_live_interval == pinned
|
||||||
|
assert "12:09 UTC" in str(app.query_one("#live-readout").render())
|
||||||
|
|
||||||
|
# Remove pinned interval from rolling window. Explain expiration.
|
||||||
|
graph.set_data(intervals[3:])
|
||||||
|
readout = str(app.query_one("#live-readout").render())
|
||||||
|
assert "Inspected interval expired; following live" in readout
|
||||||
|
assert "12:12 UTC" in readout
|
||||||
|
assert graph.selection.following_live
|
||||||
|
|
||||||
|
# Today action clears expiration and resumes newest-point following.
|
||||||
|
await pilot.press("t")
|
||||||
|
assert graph.selection.following_live
|
||||||
|
assert not graph.selection.live_interval_expired
|
||||||
|
graph.set_data(intervals)
|
||||||
|
readout = str(app.query_one("#live-readout").render())
|
||||||
|
assert "12:12 UTC" in readout
|
||||||
|
assert "Inspected interval expired" not in readout
|
||||||
|
assert graph.selection.following_live
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_mouse_inspection_pins_same_interval_identity(self, tmp_path):
|
||||||
|
"""Mouse inspection pins the interval at its visible graph location."""
|
||||||
|
app = FenrisTuiApp(store_path=tmp_path / "test.db", refresh_interval_s=999)
|
||||||
|
intervals = _live_intervals()
|
||||||
|
async with app.run_test(size=(100, 40)) as pilot:
|
||||||
|
await pilot.pause()
|
||||||
|
graph = app.query_one("#live-activity")
|
||||||
|
graph.set_data(intervals[:3])
|
||||||
|
await pilot.pause()
|
||||||
|
render = app.query_one("#live-render")
|
||||||
|
await pilot.click(
|
||||||
|
"#live-render",
|
||||||
|
offset=(8, max(0, render.content_size.height // 2)),
|
||||||
|
)
|
||||||
|
await pilot.pause()
|
||||||
|
|
||||||
|
expected = IntervalIdentity(
|
||||||
|
intervals[0]["start_ts"], intervals[0]["end_ts"]
|
||||||
|
)
|
||||||
|
assert graph.selection.selected_live_interval == expected
|
||||||
|
assert not graph.selection.following_live
|
||||||
|
assert "12:03 UTC" in str(app.query_one("#live-readout").render())
|
||||||
|
|||||||
+520
-14
@@ -25,6 +25,7 @@ from fenris.store import init_store, SCHEMA_VERSION
|
|||||||
from fenris.local_day import (
|
from fenris.local_day import (
|
||||||
derive_local_day_summary,
|
derive_local_day_summary,
|
||||||
persist_local_day,
|
persist_local_day,
|
||||||
|
record_local_activity_interval,
|
||||||
query_local_day_summary,
|
query_local_day_summary,
|
||||||
query_current_local_day,
|
query_current_local_day,
|
||||||
LocalDaySummary,
|
LocalDaySummary,
|
||||||
@@ -135,13 +136,56 @@ class TestSchemaMigration:
|
|||||||
assert count == 0
|
assert count == 0
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
def test_v4_totals_keep_values_but_are_not_treated_as_local_precision(
|
||||||
|
self, tmp_path,
|
||||||
|
):
|
||||||
|
db = tmp_path / "v4.db"
|
||||||
|
conn = sqlite3.connect(db)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE samples (id INTEGER PRIMARY KEY, ts TEXT, device TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE pending_publications "
|
||||||
|
"(id INTEGER PRIMARY KEY, sample_ts TEXT, payload TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE local_days ("
|
||||||
|
"id INTEGER PRIMARY KEY, local_date TEXT NOT NULL, tz_name TEXT NOT NULL, "
|
||||||
|
"tz_offset TEXT NOT NULL, utc_start TEXT NOT NULL, utc_end TEXT NOT NULL, "
|
||||||
|
"bytes_written INTEGER DEFAULT 0, bytes_read INTEGER DEFAULT 0, "
|
||||||
|
"coverage REAL DEFAULT 0, sample_count INTEGER DEFAULT 0, "
|
||||||
|
"complete BOOLEAN DEFAULT 0, UNIQUE(local_date, tz_name))"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO local_days VALUES "
|
||||||
|
"(1, '2026-09-01', 'UTC', '+00:00', "
|
||||||
|
"'2026-09-01T00:00:00+00:00', '2026-09-02T00:00:00+00:00', "
|
||||||
|
"5120000, 2048000, 0.5, 10, 0)"
|
||||||
|
)
|
||||||
|
conn.execute("PRAGMA user_version=4")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
conn = init_store(db)
|
||||||
|
stored = conn.execute(
|
||||||
|
"SELECT bytes_written, activity_precision FROM local_days"
|
||||||
|
).fetchone()
|
||||||
|
assert stored == (5_120_000, "unavailable")
|
||||||
|
visible = query_local_day_summary(conn, "2026-09-01", "UTC")
|
||||||
|
assert visible["bytes_written"] is None
|
||||||
|
assert visible["activity_state"] == "unavailable"
|
||||||
|
assert "local_tz" in {
|
||||||
|
row[1] for row in conn.execute("PRAGMA table_info(samples)")
|
||||||
|
}
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Local-day derivation from UTC hours
|
# Local-day coverage derivation from UTC hours
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
class TestDeriveLocalDay:
|
class TestDeriveLocalDay:
|
||||||
"""Derive local-day summaries from UTC hour observations."""
|
"""Derive coverage from UTC hours without allocating local-day bytes."""
|
||||||
|
|
||||||
def test_utc_plus_zero(self, tmp_path):
|
def test_utc_plus_zero(self, tmp_path):
|
||||||
"""UTC timezone: local day boundaries = UTC day boundaries."""
|
"""UTC timezone: local day boundaries = UTC day boundaries."""
|
||||||
@@ -158,8 +202,9 @@ class TestDeriveLocalDay:
|
|||||||
assert summary.local_date == "2026-09-01"
|
assert summary.local_date == "2026-09-01"
|
||||||
assert summary.tz_name == "UTC"
|
assert summary.tz_name == "UTC"
|
||||||
assert summary.tz_offset == "+00:00"
|
assert summary.tz_offset == "+00:00"
|
||||||
assert summary.bytes_written == 300
|
assert summary.bytes_written == 0
|
||||||
assert summary.bytes_read == 130
|
assert summary.bytes_read == 0
|
||||||
|
assert summary.activity_precision == "unavailable"
|
||||||
assert summary.complete is False # not all 24 hours covered
|
assert summary.complete is False # not all 24 hours covered
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
@@ -179,12 +224,13 @@ class TestDeriveLocalDay:
|
|||||||
assert summary.local_date == "2026-09-01"
|
assert summary.local_date == "2026-09-01"
|
||||||
assert summary.tz_name == "Asia/Kolkata"
|
assert summary.tz_name == "Asia/Kolkata"
|
||||||
assert summary.tz_offset == "+05:30"
|
assert summary.tz_offset == "+05:30"
|
||||||
assert summary.bytes_written == 300
|
assert summary.bytes_written == 0
|
||||||
assert summary.bytes_read == 130
|
assert summary.bytes_read == 0
|
||||||
|
assert summary.activity_precision == "unavailable"
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
def test_midnight_spanning_hour_included(self, tmp_path):
|
def test_hourly_bytes_cannot_be_allocated_to_local_midnight(self, tmp_path):
|
||||||
"""UTC hour straddling local midnight is included in the local day."""
|
"""UTC-hour totals do not establish exact local-day byte totals."""
|
||||||
conn = init_store(tmp_path / "obs.db")
|
conn = init_store(tmp_path / "obs.db")
|
||||||
|
|
||||||
# For UTC+5:30, local day 2026-09-01 starts at 2026-08-31T18:30 UTC
|
# For UTC+5:30, local day 2026-09-01 starts at 2026-08-31T18:30 UTC
|
||||||
@@ -196,9 +242,9 @@ class TestDeriveLocalDay:
|
|||||||
summary = derive_local_day_summary(conn, "Asia/Kolkata", clock)
|
summary = derive_local_day_summary(conn, "Asia/Kolkata", clock)
|
||||||
|
|
||||||
assert summary is not None
|
assert summary is not None
|
||||||
# The midnight-spanning hour (18:00) is included
|
assert summary.bytes_written == 0
|
||||||
assert summary.bytes_written == 300
|
assert summary.bytes_read == 0
|
||||||
assert summary.bytes_read == 130
|
assert summary.activity_precision == "unavailable"
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
def test_no_hours_returns_none(self, tmp_path):
|
def test_no_hours_returns_none(self, tmp_path):
|
||||||
@@ -387,6 +433,411 @@ class TestCollectorIntegration:
|
|||||||
assert row[1] == 30 * 512000 # reads
|
assert row[1] == 30 * 512000 # reads
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
def test_interval_crossing_utc_hour_stays_wholly_in_local_day(
|
||||||
|
self, tmp_path, sysfs_tree, monkeypatch,
|
||||||
|
):
|
||||||
|
"""A measured interval inside one local date keeps its full delta."""
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "Asia/Kolkata")
|
||||||
|
store = str(tmp_path / "obs.db")
|
||||||
|
cfg = {"device": "/dev/nvme0", "store_path": store}
|
||||||
|
sysfs_nvme = sysfs_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
first_at = datetime(2026, 9, 1, 12, 59, tzinfo=timezone.utc)
|
||||||
|
second_at = datetime(2026, 9, 1, 13, 1, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
first = run_collection(
|
||||||
|
_make_smartctl(10_000_000, 8_000_000),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(first_at),
|
||||||
|
)
|
||||||
|
second = run_collection(
|
||||||
|
_make_smartctl(10_000_010, 8_000_004),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(second_at),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert first["ok"] and second["ok"]
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
summary = query_local_day_summary(conn, "2026-09-01")
|
||||||
|
|
||||||
|
assert summary is not None
|
||||||
|
assert summary["bytes_written"] == 5_120_000
|
||||||
|
assert summary["bytes_read"] == 2_048_000
|
||||||
|
assert summary["segments"] == [{
|
||||||
|
"segment_id": 1,
|
||||||
|
"bytes_written": 5_120_000,
|
||||||
|
"bytes_read": 2_048_000,
|
||||||
|
"activity_seconds": 120,
|
||||||
|
"activity_intervals": 1,
|
||||||
|
}]
|
||||||
|
segment_totals = conn.execute(
|
||||||
|
"SELECT segment_id, bytes_written, bytes_read, activity_intervals "
|
||||||
|
"FROM local_day_segment_totals"
|
||||||
|
).fetchone()
|
||||||
|
assert segment_totals == (1, 5_120_000, 2_048_000, 1)
|
||||||
|
|
||||||
|
samples = conn.execute(
|
||||||
|
"SELECT id, ts, bytes_written, bytes_read, local_tz "
|
||||||
|
"FROM samples ORDER BY id"
|
||||||
|
).fetchall()
|
||||||
|
first_sample, second_sample = samples
|
||||||
|
assert record_local_activity_interval(
|
||||||
|
conn,
|
||||||
|
dict(zip(("id", "ts", "bytes_written", "bytes_read", "local_tz"), first_sample)),
|
||||||
|
dict(zip(("id", "ts", "bytes_written", "bytes_read", "local_tz"), second_sample)),
|
||||||
|
start_sample_id=first_sample[0],
|
||||||
|
end_sample_id=second_sample[0],
|
||||||
|
segment_id=1,
|
||||||
|
) == "known"
|
||||||
|
repeated = query_local_day_summary(conn, "2026-09-01", "Asia/Kolkata")
|
||||||
|
assert repeated["bytes_written"] == 5_120_000
|
||||||
|
assert repeated["bytes_read"] == 2_048_000
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT SUM(bytes_written), SUM(bytes_read), SUM(activity_intervals) "
|
||||||
|
"FROM local_day_segment_totals"
|
||||||
|
).fetchone() == (5_120_000, 2_048_000, 1)
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
(
|
||||||
|
"first_at", "second_at", "local_date", "expected_hours",
|
||||||
|
"expected_start", "expected_end", "expected_offset",
|
||||||
|
),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
datetime(2026, 3, 8, 6, 30, tzinfo=timezone.utc),
|
||||||
|
datetime(2026, 3, 8, 7, 30, tzinfo=timezone.utc),
|
||||||
|
"2026-03-08", 23,
|
||||||
|
"2026-03-08T05:00:00+00:00",
|
||||||
|
"2026-03-09T04:00:00+00:00", "-05:00",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
datetime(2026, 11, 1, 5, 30, tzinfo=timezone.utc),
|
||||||
|
datetime(2026, 11, 1, 6, 30, tzinfo=timezone.utc),
|
||||||
|
"2026-11-01", 25,
|
||||||
|
"2026-11-01T04:00:00+00:00",
|
||||||
|
"2026-11-02T05:00:00+00:00", "-04:00",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
ids=("spring-forward", "fall-back-repeated-hour"),
|
||||||
|
)
|
||||||
|
def test_collector_retains_exact_interval_on_dst_local_day(
|
||||||
|
self,
|
||||||
|
tmp_path,
|
||||||
|
sysfs_tree,
|
||||||
|
monkeypatch,
|
||||||
|
first_at,
|
||||||
|
second_at,
|
||||||
|
local_date,
|
||||||
|
expected_hours,
|
||||||
|
expected_start,
|
||||||
|
expected_end,
|
||||||
|
expected_offset,
|
||||||
|
):
|
||||||
|
"""Collection preserves exact activity on short and long local days."""
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
zone = "America/New_York"
|
||||||
|
monkeypatch.setenv("TZ", zone)
|
||||||
|
store = str(tmp_path / "obs.db")
|
||||||
|
cfg = {"device": "/dev/nvme0", "store_path": store}
|
||||||
|
sysfs_nvme = sysfs_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_000, 8_000_000),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(first_at),
|
||||||
|
)["ok"]
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_001, 8_000_002),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(second_at),
|
||||||
|
)["ok"]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, local_date, zone, second_at + timedelta(minutes=1)
|
||||||
|
)
|
||||||
|
assert summary is not None
|
||||||
|
assert summary["bytes_written"] == 512_000
|
||||||
|
assert summary["bytes_read"] == 1_024_000
|
||||||
|
assert summary["activity_seconds"] == 3_600
|
||||||
|
assert summary["utc_start"] == expected_start
|
||||||
|
assert summary["utc_end"] == expected_end
|
||||||
|
assert summary["tz_offset"] == expected_offset
|
||||||
|
assert (
|
||||||
|
datetime.fromisoformat(summary["utc_end"])
|
||||||
|
- datetime.fromisoformat(summary["utc_start"])
|
||||||
|
).total_seconds() == expected_hours * 3_600
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_local_midnight_interval_is_shared_once(
|
||||||
|
self, tmp_path, sysfs_tree, monkeypatch,
|
||||||
|
):
|
||||||
|
"""A real counter interval crossing local midnight stays unallocated."""
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "Asia/Kolkata")
|
||||||
|
store = str(tmp_path / "obs.db")
|
||||||
|
cfg = {"device": "/dev/nvme0", "store_path": store}
|
||||||
|
sysfs_nvme = sysfs_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
first_at = datetime(2026, 9, 1, 18, 25, tzinfo=timezone.utc)
|
||||||
|
second_at = datetime(2026, 9, 1, 18, 35, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
first = run_collection(
|
||||||
|
_make_smartctl(10_000_000, 8_000_000),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(first_at),
|
||||||
|
)
|
||||||
|
second = run_collection(
|
||||||
|
_make_smartctl(10_000_100, 8_000_060),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(second_at),
|
||||||
|
)
|
||||||
|
assert first["ok"] and second["ok"]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
evidence = conn.execute(
|
||||||
|
"SELECT COUNT(*), SUM(bytes_written), SUM(bytes_read) "
|
||||||
|
"FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone()
|
||||||
|
assert evidence == (1, 51_200_000, 30_720_000)
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT SUM(bytes_written), SUM(bytes_read) FROM local_days"
|
||||||
|
).fetchone() == (0, 0)
|
||||||
|
|
||||||
|
for local_date in ("2026-09-01", "2026-09-02"):
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, local_date, "Asia/Kolkata", second_at
|
||||||
|
)
|
||||||
|
assert summary is not None
|
||||||
|
assert summary["bytes_written"] is None
|
||||||
|
assert summary["shared_bytes_written"] == 51_200_000
|
||||||
|
assert summary["shared_bytes_read"] == 30_720_000
|
||||||
|
assert summary["activity_state"] in ("incomplete", "so_far")
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
from fenris.status import read_status
|
||||||
|
from fenris.tui import FenrisTuiApp
|
||||||
|
|
||||||
|
app = FenrisTuiApp(store_path=Path(store), refresh_interval_s=999)
|
||||||
|
async with app.run_test(size=(100, 32)):
|
||||||
|
app._browse_date = "2026-09-02"
|
||||||
|
with read_status(
|
||||||
|
Path(store), second_at, query_services=False
|
||||||
|
) as (reader, _):
|
||||||
|
assert reader is not None
|
||||||
|
app._render_local_day(reader)
|
||||||
|
visible = str(app.query_one("#local-day").render())
|
||||||
|
assert "2026-09-02" in visible
|
||||||
|
assert "Asia/Kolkata +05:30" in visible
|
||||||
|
assert "W unavailable" in visible
|
||||||
|
assert "shared at midnight W 0.051 GB" in visible
|
||||||
|
|
||||||
|
def test_synthetic_shared_bytes_conserve_one_hundred_bytes(self, tmp_path):
|
||||||
|
"""Shared evidence conserves byte values before display rounding."""
|
||||||
|
conn = init_store(tmp_path / "obs.db")
|
||||||
|
first_at = datetime(2026, 9, 1, 18, 25, tzinfo=timezone.utc)
|
||||||
|
second_at = datetime(2026, 9, 3, 18, 35, tzinfo=timezone.utc)
|
||||||
|
ensure_period_open(conn, first_at)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, local_tz) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(first_at.isoformat(), "/dev/nvme0", 1_000, 2_000, "Asia/Kolkata"),
|
||||||
|
)
|
||||||
|
first_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO samples (ts, device, bytes_written, bytes_read, local_tz) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(second_at.isoformat(), "/dev/nvme0", 1_100, 2_050, "Asia/Kolkata"),
|
||||||
|
)
|
||||||
|
second_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
|
||||||
|
|
||||||
|
result = record_local_activity_interval(
|
||||||
|
conn,
|
||||||
|
{
|
||||||
|
"id": first_id,
|
||||||
|
"ts": first_at.isoformat(),
|
||||||
|
"bytes_written": 1_000,
|
||||||
|
"bytes_read": 2_000,
|
||||||
|
"local_tz": "Asia/Kolkata",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": second_id,
|
||||||
|
"ts": second_at.isoformat(),
|
||||||
|
"bytes_written": 1_100,
|
||||||
|
"bytes_read": 2_050,
|
||||||
|
"local_tz": "Asia/Kolkata",
|
||||||
|
},
|
||||||
|
start_sample_id=first_id,
|
||||||
|
end_sample_id=second_id,
|
||||||
|
segment_id=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == "local_midnight"
|
||||||
|
stored = conn.execute(
|
||||||
|
"SELECT COUNT(*), SUM(bytes_written), SUM(bytes_read) "
|
||||||
|
"FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone()
|
||||||
|
assert stored == (1, 100, 50)
|
||||||
|
record_local_activity_interval(
|
||||||
|
conn,
|
||||||
|
{
|
||||||
|
"id": first_id,
|
||||||
|
"ts": first_at.isoformat(),
|
||||||
|
"bytes_written": 1_000,
|
||||||
|
"bytes_read": 2_000,
|
||||||
|
"local_tz": "Asia/Kolkata",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": second_id,
|
||||||
|
"ts": second_at.isoformat(),
|
||||||
|
"bytes_written": 1_100,
|
||||||
|
"bytes_read": 2_050,
|
||||||
|
"local_tz": "Asia/Kolkata",
|
||||||
|
},
|
||||||
|
start_sample_id=first_id,
|
||||||
|
end_sample_id=second_id,
|
||||||
|
segment_id=1,
|
||||||
|
)
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*), SUM(bytes_written), SUM(bytes_read) "
|
||||||
|
"FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone() == (1, 100, 50)
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT SUM(bytes_written), SUM(bytes_read) FROM local_days"
|
||||||
|
).fetchone() == (0, 0)
|
||||||
|
middle_day = query_local_day_summary(
|
||||||
|
conn, "2026-09-03", "Asia/Kolkata", second_at + timedelta(days=1)
|
||||||
|
)
|
||||||
|
assert middle_day["shared_bytes_written"] == 100
|
||||||
|
assert middle_day["shared_bytes_read"] == 50
|
||||||
|
assert middle_day["shared_evidence_count"] == 1
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_timezone_change_keeps_interval_unallocated(
|
||||||
|
self, tmp_path, sysfs_tree, monkeypatch,
|
||||||
|
):
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
store = str(tmp_path / "obs.db")
|
||||||
|
cfg = {"device": "/dev/nvme0", "store_path": store}
|
||||||
|
sysfs_nvme = sysfs_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
first_at = datetime(2026, 9, 1, 22, 0, tzinfo=timezone.utc)
|
||||||
|
second_at = datetime(2026, 9, 1, 22, 5, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_000, 8_000_000),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(first_at),
|
||||||
|
)["ok"]
|
||||||
|
monkeypatch.setenv("TZ", "Asia/Kolkata")
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_010, 8_000_004),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(second_at),
|
||||||
|
)["ok"]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
event = conn.execute(
|
||||||
|
"SELECT COUNT(*), SUM(bytes_written), MIN(reason) "
|
||||||
|
"FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone()
|
||||||
|
assert event == (1, 5_120_000, "timezone_change")
|
||||||
|
old_zone = query_local_day_summary(conn, "2026-09-01", "UTC")
|
||||||
|
new_zone = query_local_day_summary(conn, "2026-09-02", "Asia/Kolkata")
|
||||||
|
assert old_zone["bytes_written"] is None
|
||||||
|
assert new_zone["bytes_written"] is None
|
||||||
|
assert old_zone["unallocated_bytes_written"] == 5_120_000
|
||||||
|
assert new_zone["unallocated_bytes_written"] == 5_120_000
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_pause_boundary_keeps_interval_unallocated(
|
||||||
|
self, tmp_path, sysfs_tree, monkeypatch,
|
||||||
|
):
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
store = str(tmp_path / "obs.db")
|
||||||
|
cfg = {"device": "/dev/nvme0", "store_path": store}
|
||||||
|
sysfs_nvme = sysfs_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
first_at = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc)
|
||||||
|
second_at = datetime(2026, 9, 1, 12, 10, tzinfo=timezone.utc)
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_000, 8_000_000),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(first_at),
|
||||||
|
)["ok"]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
close_period(conn, first_at + timedelta(minutes=5), "user_disabled")
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_010, 8_000_004),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(second_at),
|
||||||
|
)["ok"]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
reason = conn.execute(
|
||||||
|
"SELECT reason FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone()
|
||||||
|
summary = query_local_day_summary(conn, "2026-09-01", "UTC")
|
||||||
|
assert reason == ("monitoring_period",)
|
||||||
|
assert summary["bytes_written"] is None
|
||||||
|
assert summary["unallocated_bytes_written"] == 5_120_000
|
||||||
|
assert summary["shared_bytes_written"] == 0
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_controller_segment_boundary_marks_local_activity_incomplete(
|
||||||
|
self, tmp_path, sysfs_tree, monkeypatch,
|
||||||
|
):
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "UTC")
|
||||||
|
store = str(tmp_path / "obs.db")
|
||||||
|
cfg = {"device": "/dev/nvme0", "store_path": store}
|
||||||
|
sysfs_nvme = sysfs_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
first_at = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc)
|
||||||
|
second_at = datetime(2026, 9, 1, 12, 5, tzinfo=timezone.utc)
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_000, 8_000_000),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(first_at),
|
||||||
|
)["ok"]
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10, 8_000_004),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(second_at),
|
||||||
|
)["ok"]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, "2026-09-01", "UTC", second_at
|
||||||
|
)
|
||||||
|
assert summary is not None
|
||||||
|
assert summary["bytes_written"] is None
|
||||||
|
assert summary["activity_state"] == "incomplete"
|
||||||
|
assert conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM local_day_unallocated_evidence"
|
||||||
|
).fetchone()[0] == 0
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# DST handling (23/25-hour days)
|
# DST handling (23/25-hour days)
|
||||||
@@ -402,17 +853,72 @@ class TestDSTHandling:
|
|||||||
# Simulate a 23-hour day in a timezone with DST
|
# Simulate a 23-hour day in a timezone with DST
|
||||||
# For simplicity, just verify the summary records the correct UTC range
|
# For simplicity, just verify the summary records the correct UTC range
|
||||||
clock = datetime(2026, 3, 8, 12, 0, 0, tzinfo=timezone.utc)
|
clock = datetime(2026, 3, 8, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
# US/Eastern springs forward on 2026-03-08
|
# New York springs forward on 2026-03-08
|
||||||
# Local day 2026-03-08 is 23 hours: UTC [07:00, 06:00+1d)
|
# Local day 2026-03-08 is 23 hours: UTC [07:00, 06:00+1d)
|
||||||
_insert_hour(conn, "2026-03-08T08:00:00+00:00", bw=100, active=3600)
|
_insert_hour(conn, "2026-03-08T08:00:00+00:00", bw=100, active=3600)
|
||||||
_insert_hour(conn, "2026-03-08T12:00:00+00:00", bw=200, active=3600)
|
_insert_hour(conn, "2026-03-08T12:00:00+00:00", bw=200, active=3600)
|
||||||
|
|
||||||
summary = derive_local_day_summary(conn, "US/Eastern", clock)
|
summary = derive_local_day_summary(conn, "America/New_York", clock)
|
||||||
assert summary is not None
|
assert summary is not None
|
||||||
assert summary.local_date == "2026-03-08"
|
assert summary.local_date == "2026-03-08"
|
||||||
# UTC range should be approximately 23 hours
|
# UTC range should be approximately 23 hours
|
||||||
utc_start = datetime.fromisoformat(summary.utc_start)
|
utc_start = datetime.fromisoformat(summary.utc_start)
|
||||||
utc_end = datetime.fromisoformat(summary.utc_end)
|
utc_end = datetime.fromisoformat(summary.utc_end)
|
||||||
duration = (utc_end - utc_start).total_seconds()
|
duration = (utc_end - utc_start).total_seconds()
|
||||||
assert 22 * 3600 <= duration <= 24 * 3600 # ~23h ± tolerance
|
assert duration == 23 * 3600
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_long_day_25_hours(self):
|
||||||
|
from fenris.local_day import local_day_boundaries
|
||||||
|
|
||||||
|
start, end, offset = local_day_boundaries(
|
||||||
|
"2026-11-01", "America/New_York"
|
||||||
|
)
|
||||||
|
assert (end - start).total_seconds() == 25 * 3600
|
||||||
|
assert offset == "-04:00"
|
||||||
|
|
||||||
|
def test_half_hour_offset_records_exact_local_boundaries(self):
|
||||||
|
from fenris.local_day import local_day_boundaries
|
||||||
|
|
||||||
|
start, end, offset = local_day_boundaries(
|
||||||
|
"2026-09-01", "Asia/Kolkata"
|
||||||
|
)
|
||||||
|
assert start.isoformat() == "2026-08-31T18:30:00+00:00"
|
||||||
|
assert end.isoformat() == "2026-09-01T18:30:00+00:00"
|
||||||
|
assert offset == "+05:30"
|
||||||
|
|
||||||
|
def test_repeated_local_clock_labels_remain_one_recorded_day(
|
||||||
|
self, tmp_path, sysfs_tree, monkeypatch,
|
||||||
|
):
|
||||||
|
from fenris.collector import run_collection
|
||||||
|
|
||||||
|
monkeypatch.setenv("TZ", "America/New_York")
|
||||||
|
store = str(tmp_path / "obs.db")
|
||||||
|
cfg = {"device": "/dev/nvme0", "store_path": store}
|
||||||
|
sysfs_nvme = sysfs_tree / "sys" / "class" / "nvme" / "nvme0"
|
||||||
|
# Both endpoints display as 01:30 locally, on opposite UTC offsets.
|
||||||
|
first_at = datetime(2026, 11, 1, 5, 30, tzinfo=timezone.utc)
|
||||||
|
second_at = datetime(2026, 11, 1, 6, 30, tzinfo=timezone.utc)
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_000, 8_000_000),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(first_at),
|
||||||
|
)["ok"]
|
||||||
|
assert run_collection(
|
||||||
|
_make_smartctl(10_000_001, 8_000_002),
|
||||||
|
sysfs_nvme,
|
||||||
|
cfg,
|
||||||
|
_Clock(second_at),
|
||||||
|
)["ok"]
|
||||||
|
|
||||||
|
conn = sqlite3.connect(store)
|
||||||
|
summary = query_local_day_summary(
|
||||||
|
conn, "2026-11-01", "America/New_York", second_at
|
||||||
|
)
|
||||||
|
assert summary is not None
|
||||||
|
assert summary["bytes_written"] == 512_000
|
||||||
|
assert summary["bytes_read"] == 1_024_000
|
||||||
|
assert summary["utc_start"] == "2026-11-01T04:00:00+00:00"
|
||||||
|
assert summary["utc_end"] == "2026-11-02T05:00:00+00:00"
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|||||||
@@ -100,8 +100,8 @@ def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO local_days "
|
"INSERT INTO local_days "
|
||||||
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
||||||
" bytes_written, bytes_read, coverage, sample_count, complete) "
|
" bytes_written, bytes_read, coverage, sample_count, complete, activity_intervals) "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)",
|
||||||
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
||||||
bw, br, coverage, samples, complete),
|
bw, br, coverage, samples, complete),
|
||||||
)
|
)
|
||||||
|
|||||||
+17
-15
@@ -27,9 +27,14 @@ def store_conn(tmp_path: Path):
|
|||||||
|
|
||||||
|
|
||||||
def _insert_sample(conn, ts_iso, device="/dev/nvme0"):
|
def _insert_sample(conn, ts_iso, device="/dev/nvme0"):
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO controller_segments "
|
||||||
|
"(id, opened_at, identity_key) VALUES (1, '2026-01-01T00:00:00+00:00', 'test')"
|
||||||
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO samples (ts, device, data_units_written, data_units_read, "
|
"INSERT INTO samples (ts, device, data_units_written, data_units_read, "
|
||||||
" bytes_written, bytes_read, percentage_used) VALUES (?, ?, 0, 0, 0, 0, 0)",
|
" bytes_written, bytes_read, percentage_used, segment_id, local_tz) "
|
||||||
|
"VALUES (?, ?, 0, 0, 0, 0, 0, 1, 'UTC')",
|
||||||
(ts_iso, device),
|
(ts_iso, device),
|
||||||
)
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
@@ -50,33 +55,30 @@ class TestPruneOldSamples:
|
|||||||
cursor = store_conn.execute("SELECT COUNT(*) FROM samples")
|
cursor = store_conn.execute("SELECT COUNT(*) FROM samples")
|
||||||
assert cursor.fetchone()[0] == 1
|
assert cursor.fetchone()[0] == 1
|
||||||
|
|
||||||
def test_removes_old_samples(self, store_conn):
|
def test_keeps_old_samples_without_replacement_evidence(self, store_conn):
|
||||||
now = datetime(2026, 9, 15, 12, 0, 0, tzinfo=timezone.utc)
|
now = datetime(2026, 9, 15, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
# Insert samples at 10, 14, and 15 days ago
|
# Old samples stay until UTC and local-day evidence replace them.
|
||||||
# All three are before the cutoff (2026-09-01T12:00:00)
|
|
||||||
# The 15-day-old sample is not a boundary anchor because
|
|
||||||
# the next sample (14 days ago) is also before the cutoff
|
|
||||||
for days_ago in [10, 14, 15]:
|
for days_ago in [10, 14, 15]:
|
||||||
ts = (now - timedelta(days=days_ago)).isoformat()
|
ts = (now - timedelta(days=days_ago)).isoformat()
|
||||||
_insert_sample(store_conn, ts)
|
_insert_sample(store_conn, ts)
|
||||||
|
|
||||||
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
||||||
assert pruned == 1 # Only the 15-day-old sample removed
|
assert pruned == 0
|
||||||
|
|
||||||
cursor = store_conn.execute("SELECT COUNT(*) FROM samples")
|
cursor = store_conn.execute("SELECT COUNT(*) FROM samples")
|
||||||
assert cursor.fetchone()[0] == 2
|
assert cursor.fetchone()[0] == 3
|
||||||
|
|
||||||
def test_removes_many_old_samples(self, store_conn):
|
def test_keeps_many_old_samples_without_replacement_evidence(self, store_conn):
|
||||||
now = datetime(2026, 9, 15, 12, 0, 0, tzinfo=timezone.utc)
|
now = datetime(2026, 9, 15, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
for days_ago in range(1, 30):
|
for days_ago in range(1, 30):
|
||||||
ts = (now - timedelta(days=days_ago)).isoformat()
|
ts = (now - timedelta(days=days_ago)).isoformat()
|
||||||
_insert_sample(store_conn, ts)
|
_insert_sample(store_conn, ts)
|
||||||
|
|
||||||
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
||||||
assert pruned == 15 # Days 15-29 removed
|
assert pruned == 0
|
||||||
|
|
||||||
cursor = store_conn.execute("SELECT COUNT(*) FROM samples")
|
cursor = store_conn.execute("SELECT COUNT(*) FROM samples")
|
||||||
assert cursor.fetchone()[0] == 14 # Days 1-14 kept
|
assert cursor.fetchone()[0] == 29
|
||||||
|
|
||||||
def test_empty_store_no_error(self, store_conn):
|
def test_empty_store_no_error(self, store_conn):
|
||||||
now = datetime(2026, 9, 15, 12, 0, 0, tzinfo=timezone.utc)
|
now = datetime(2026, 9, 15, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
@@ -130,8 +132,8 @@ class TestPruneOldSamples:
|
|||||||
)
|
)
|
||||||
assert cursor.fetchone()[0] == 1
|
assert cursor.fetchone()[0] == 1
|
||||||
|
|
||||||
def test_old_sample_with_derived_interval_removed(self, store_conn):
|
def test_hour_rows_alone_do_not_replace_local_day_evidence(self, store_conn):
|
||||||
"""Old samples with fully derived intervals are removed."""
|
"""An hour row alone cannot authorize source-sample deletion."""
|
||||||
now = datetime(2026, 9, 30, 12, 0, 0, tzinfo=timezone.utc)
|
now = datetime(2026, 9, 30, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
# Old sample with derived interval
|
# Old sample with derived interval
|
||||||
@@ -148,8 +150,8 @@ class TestPruneOldSamples:
|
|||||||
# Run pruning
|
# Run pruning
|
||||||
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
||||||
|
|
||||||
# Old sample should be removed (interval is derived)
|
# Keep the source samples until local-day replacement evidence exists.
|
||||||
cursor = store_conn.execute(
|
cursor = store_conn.execute(
|
||||||
"SELECT COUNT(*) FROM samples WHERE ts = '2026-09-10T10:00:00+00:00'"
|
"SELECT COUNT(*) FROM samples WHERE ts = '2026-09-10T10:00:00+00:00'"
|
||||||
)
|
)
|
||||||
assert cursor.fetchone()[0] == 0
|
assert cursor.fetchone()[0] == 1
|
||||||
|
|||||||
@@ -235,8 +235,8 @@ class TestBoundaryAnchorRetention:
|
|||||||
# the interval spans the retention boundary
|
# the interval spans the retention boundary
|
||||||
assert needs_boundary_anchor(store_conn, "2026-09-15T23:55:00+00:00", now)
|
assert needs_boundary_anchor(store_conn, "2026-09-15T23:55:00+00:00", now)
|
||||||
|
|
||||||
def test_not_boundary_anchor_if_fully_derived(self, store_conn):
|
def test_hour_only_derivation_does_not_replace_local_day_evidence(self, store_conn):
|
||||||
"""Sample that's fully derived is not a boundary anchor."""
|
"""UTC hour evidence alone does not make a sample safe to prune."""
|
||||||
now = datetime(2026, 9, 30, 12, 0, 0, tzinfo=timezone.utc)
|
now = datetime(2026, 9, 30, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
# Insert sample and fully derive its interval
|
# Insert sample and fully derive its interval
|
||||||
@@ -247,8 +247,7 @@ class TestBoundaryAnchorRetention:
|
|||||||
_insert_hour(store_conn, "2026-09-14T10:00:00+00:00",
|
_insert_hour(store_conn, "2026-09-14T10:00:00+00:00",
|
||||||
bytes_written_delta=1000000)
|
bytes_written_delta=1000000)
|
||||||
|
|
||||||
# Not a boundary anchor
|
assert needs_boundary_anchor(store_conn, "2026-09-14T10:00:00+00:00", now)
|
||||||
assert not needs_boundary_anchor(store_conn, "2026-09-14T10:00:00+00:00", now)
|
|
||||||
|
|
||||||
def test_pruning_retains_boundary_anchors(self, store_conn):
|
def test_pruning_retains_boundary_anchors(self, store_conn):
|
||||||
"""Pruning keeps samples needed as boundary anchors."""
|
"""Pruning keeps samples needed as boundary anchors."""
|
||||||
@@ -274,8 +273,8 @@ class TestBoundaryAnchorRetention:
|
|||||||
)
|
)
|
||||||
assert cursor.fetchone()[0] == 1
|
assert cursor.fetchone()[0] == 1
|
||||||
|
|
||||||
def test_pruning_removes_old_sample_with_derived_interval(self, store_conn):
|
def test_pruning_keeps_samples_without_local_day_replacement(self, store_conn):
|
||||||
"""Pruning removes old samples when interval is fully derived."""
|
"""Pruning keeps samples when only the UTC-hour row exists."""
|
||||||
now = datetime(2026, 9, 30, 12, 0, 0, tzinfo=timezone.utc)
|
now = datetime(2026, 9, 30, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
# Old sample with derived interval
|
# Old sample with derived interval
|
||||||
@@ -289,11 +288,11 @@ class TestBoundaryAnchorRetention:
|
|||||||
# Run pruning
|
# Run pruning
|
||||||
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
pruned = prune_old_samples(store_conn, now, retention_days=14)
|
||||||
|
|
||||||
# Old sample should be removed (interval is derived)
|
# The source remains until local-day evidence is also durable.
|
||||||
cursor = store_conn.execute(
|
cursor = store_conn.execute(
|
||||||
"SELECT COUNT(*) FROM samples WHERE ts = '2026-09-10T10:00:00+00:00'"
|
"SELECT COUNT(*) FROM samples WHERE ts = '2026-09-10T10:00:00+00:00'"
|
||||||
)
|
)
|
||||||
assert cursor.fetchone()[0] == 0
|
assert cursor.fetchone()[0] == 1
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
+22
-21
@@ -109,6 +109,7 @@ def _insert_sample(conn, ts, pu=5, device="/dev/nvme0n1"):
|
|||||||
|
|
||||||
def _open_period(conn, start="2026-09-01T00:00:00+00:00"):
|
def _open_period(conn, start="2026-09-01T00:00:00+00:00"):
|
||||||
ensure_period_open(conn, datetime.fromisoformat(start))
|
ensure_period_open(conn, datetime.fromisoformat(start))
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
||||||
@@ -123,8 +124,8 @@ def _insert_local_day(conn, local_date, tz_name="UTC", tz_offset="+00:00",
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO local_days "
|
"INSERT INTO local_days "
|
||||||
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
"(local_date, tz_name, tz_offset, utc_start, utc_end, "
|
||||||
" bytes_written, bytes_read, coverage, sample_count, complete) "
|
" bytes_written, bytes_read, coverage, sample_count, complete, activity_intervals) "
|
||||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)",
|
||||||
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
(local_date, tz_name, tz_offset, utc_start, utc_end,
|
||||||
bw, br, coverage, samples, complete),
|
bw, br, coverage, samples, complete),
|
||||||
)
|
)
|
||||||
@@ -926,6 +927,7 @@ class TestBarGraphTUI:
|
|||||||
for i in range(14):
|
for i in range(14):
|
||||||
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
_insert_day(conn, d, bw=1024*1024*100)
|
_insert_day(conn, d, bw=1024*1024*100)
|
||||||
|
_insert_local_day(conn, d, bw=1024*1024*100)
|
||||||
_insert_sample(conn, "2026-09-30T10:00:00+00:00")
|
_insert_sample(conn, "2026-09-30T10:00:00+00:00")
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
@@ -940,11 +942,9 @@ class TestBarGraphTUI:
|
|||||||
assert "Unalloc" in legend
|
assert "Unalloc" in legend
|
||||||
assert "Zero" in legend
|
assert "Zero" in legend
|
||||||
range_label = str(app.query_one("#bar-range").render())
|
range_label = str(app.query_one("#bar-range").render())
|
||||||
assert "14 days" in range_label
|
assert "14 local days" in range_label
|
||||||
assert "UTC" in range_label
|
|
||||||
plotted = str(app.query_one("#bar-render").render())
|
plotted = str(app.query_one("#bar-render").render())
|
||||||
assert "Writes (" in legend
|
assert "Writes (" in legend
|
||||||
assert "UTC" in range_label
|
|
||||||
assert "─" in plotted
|
assert "─" in plotted
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -956,6 +956,7 @@ class TestBarGraphTUI:
|
|||||||
for i in range(14):
|
for i in range(14):
|
||||||
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
_insert_day(conn, d, bw=1024*1024*100)
|
_insert_day(conn, d, bw=1024*1024*100)
|
||||||
|
_insert_local_day(conn, d, bw=1024*1024*100)
|
||||||
_insert_sample(conn, "2026-09-30T10:00:00+00:00")
|
_insert_sample(conn, "2026-09-30T10:00:00+00:00")
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
@@ -1029,6 +1030,7 @@ class TestBarGraphTUI:
|
|||||||
for i in range(14):
|
for i in range(14):
|
||||||
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
_insert_day(conn, d, bw=1024*1024*100)
|
_insert_day(conn, d, bw=1024*1024*100)
|
||||||
|
_insert_local_day(conn, d, bw=1024*1024*100)
|
||||||
_insert_sample(conn, "2026-09-30T10:00:00+00:00")
|
_insert_sample(conn, "2026-09-30T10:00:00+00:00")
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
@@ -1042,15 +1044,16 @@ class TestBarGraphTUI:
|
|||||||
|
|
||||||
# Newest day is selected initially.
|
# Newest day is selected initially.
|
||||||
readout = str(app.query_one("#bar-readout").render())
|
readout = str(app.query_one("#bar-readout").render())
|
||||||
assert "UTC" in readout
|
assert "totals so far" in readout
|
||||||
assert "GB" in readout
|
assert "W 0.105 GB known" in readout
|
||||||
|
|
||||||
# Moving left updates the selected-day readout.
|
# Moving left updates the selected-day readout.
|
||||||
await pilot.press("left")
|
await pilot.press("left")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
readout = str(app.query_one("#bar-readout").render())
|
readout = str(app.query_one("#bar-readout").render())
|
||||||
assert "2026-09" in readout
|
assert "2026-09" in readout
|
||||||
assert "GB" in readout
|
assert "incomplete" in readout
|
||||||
|
assert "W 0.105 GB known" in readout
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_hourly_drill_down_and_back(self, tmp_path):
|
async def test_hourly_drill_down_and_back(self, tmp_path):
|
||||||
@@ -1061,6 +1064,7 @@ class TestBarGraphTUI:
|
|||||||
for i in range(14):
|
for i in range(14):
|
||||||
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
d = (datetime(2026, 9, 15) + timedelta(days=i)).strftime("%Y-%m-%d")
|
||||||
_insert_day(conn, d, bw=1024*1024*100)
|
_insert_day(conn, d, bw=1024*1024*100)
|
||||||
|
_insert_local_day(conn, d, bw=1024*1024*100)
|
||||||
# Insert hours for each day
|
# Insert hours for each day
|
||||||
for h in range(24):
|
for h in range(24):
|
||||||
hour = "%sT%02d:00:00+00:00" % (d, h)
|
hour = "%sT%02d:00:00+00:00" % (d, h)
|
||||||
@@ -1077,30 +1081,27 @@ class TestBarGraphTUI:
|
|||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
|
|
||||||
# The newest day is selected automatically.
|
assert app.query_one("#activity-tabs").active == "view-history"
|
||||||
assert graph.selected_index == len(graph._day_data) - 1
|
daily_readout = str(app.query_one("#bar-readout").render())
|
||||||
assert graph.view_mode == "daily"
|
|
||||||
|
|
||||||
# Enter drill-down
|
# Enter drill-down
|
||||||
await pilot.press("enter")
|
await pilot.press("enter")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
assert graph.view_mode == "hourly"
|
assert app.query_one("#activity-tabs").active == "view-day"
|
||||||
assert graph.drill_day is not None
|
assert "UTC" in str(app.query_one("#bar-readout").render())
|
||||||
assert len(graph._hour_data) == 24
|
|
||||||
|
|
||||||
# Five-minute data refreshes retain the selected hour.
|
# Inspection identity survives refresh through visible readout.
|
||||||
graph._hourly_selected = 12
|
await pilot.press("left")
|
||||||
selected_hour = graph._hour_data[12]["hour"]
|
selected_hour = str(app.query_one("#bar-readout").render())
|
||||||
app._refresh()
|
app._refresh()
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
assert graph.view_mode == "hourly"
|
assert str(app.query_one("#bar-readout").render()) == selected_hour
|
||||||
assert graph._hour_data[graph._hourly_selected]["hour"] == selected_hour
|
|
||||||
|
|
||||||
# Esc returns to daily
|
# Esc returns to daily
|
||||||
await pilot.press("escape")
|
await pilot.press("escape")
|
||||||
await pilot.pause()
|
await pilot.pause()
|
||||||
assert graph.view_mode == "daily"
|
assert app.query_one("#activity-tabs").active == "view-history"
|
||||||
assert graph.drill_day is None
|
assert str(app.query_one("#bar-readout").render()) == daily_readout
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_empty_store_graph(self, tmp_path):
|
async def test_empty_store_graph(self, tmp_path):
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
"""Timezone detection tests."""
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from fenris import tz_util
|
||||||
|
|
||||||
|
|
||||||
|
def test_relative_localtime_symlink_returns_zoneinfo_key(tmp_path, monkeypatch):
|
||||||
|
zoneinfo = tmp_path / "usr" / "share" / "zoneinfo" / "Asia" / "Kolkata"
|
||||||
|
zoneinfo.parent.mkdir(parents=True)
|
||||||
|
zoneinfo.write_bytes(b"zoneinfo")
|
||||||
|
localtime = tmp_path / "etc" / "localtime"
|
||||||
|
localtime.parent.mkdir()
|
||||||
|
localtime.symlink_to("../usr/share/zoneinfo/Asia/Kolkata")
|
||||||
|
|
||||||
|
real_path = tz_util.Path
|
||||||
|
monkeypatch.setattr(
|
||||||
|
tz_util,
|
||||||
|
"Path",
|
||||||
|
lambda path: localtime if path == "/etc/localtime" else real_path(path),
|
||||||
|
)
|
||||||
|
monkeypatch.delenv("TZ", raising=False)
|
||||||
|
|
||||||
|
assert tz_util.detect_system_tz() == "Asia/Kolkata"
|
||||||
Reference in New Issue
Block a user