Keep local-day history trustworthy after detail expires (#93)

Add query_local_day_history() with evidence-limit metadata to local_day.py, including LocalDayHistoryEntry dataclass that annotates each summary with detail_available and derived_from_surviving flags. Document the 14-day detail vs indefinite-summary retention policy in both local_day.py and pruning.py. Fix query_local_day_summary() to convert SQLite integer booleans to Python bool, and add deduplication guard to the midnight-spanning hour logic in derive_local_day_summary(). Add 17 lifecycle tests covering all acceptance criteria: - AC1: Summaries survive sample pruning, evidence limits visible - AC2: Boundary anchors preserved before pruning - AC3: Timezone survives system-timezone change - AC4: Legacy UTC summaries labelled incomplete - AC5: Idempotent pruning and repair - AC6: Volume conservation (no double counting) - AC7: Real temporary stores - AC8: User-visible transition from recent to aged - AC9: Retention policy documented All 775 tests pass.
2026-09-18 14:27:44 +05:30
parent 5df8a12339
commit 95cca2e115
3 changed files with 771 additions and 10 deletions
@@ -10,6 +10,21 @@ Key contracts:
 - Midnight-spanning intervals are retained once as shared/unallocated evidence
 - UTC hour/day aggregates are never modified or deleted
 - Migration cannot manufacture local precision from historical UTC data
+
+Retention policy (issue #93):
+- Raw three-minute samples are pruned after 14 days (spec §3.4, ST-5).
+- Hour observations and day aggregates are retained indefinitely.
+- Local-day summaries are persisted at collection time and retained
+  indefinitely.  They survive raw-sample pruning because they depend on
+  hour observations, not on raw samples.
+- After detail expires, aged local summaries remain queryable with their
+  recorded timezone and UTC boundaries.  The evidence-availability flag
+  on each history entry indicates whether the underlying raw detail is
+  still present or has been pruned.
+- A later timezone change does not rewrite historical day boundaries.
+  Each summary retains the timezone and offsets recorded at collection.
+- No fabricated evidence replaces missing precision.  Incomplete or
+  unavailable dates are labelled as such in the history readout.
 """
 import sqlite3
 from dataclasses import dataclass
@@ -31,6 +46,52 @@ class LocalDaySummary:
    complete: bool           # day's UTC range fully covered by hour observations


+@dataclass(frozen=True)
+class LocalDayHistoryEntry:
+    """A local-day summary with evidence-limit metadata for history readout.
+
+    Includes the summary data plus flags that indicate whether the
+    underlying raw three-minute detail is still available or has been
+    pruned, and whether the summary was derived from surviving evidence
+    or is a legacy UTC-only aggregate.
+    """
+    local_date: str
+    tz_name: str
+    tz_offset: str
+    utc_start: str
+    utc_end: str
+    bytes_written: int
+    bytes_read: int
+    coverage: float
+    sample_count: int
+    complete: bool
+    detail_available: bool    # True if raw samples for this day are within 14-day retention
+    derived_from_surviving: bool  # True if derived from hour observations, not raw samples
+
+    @classmethod
+    def from_summary(
+        cls,
+        summary: dict,
+        detail_available: bool,
+        derived_from_surviving: bool = True,
+    ) -> "LocalDayHistoryEntry":
+        """Create a history entry from a stored summary dict."""
+        return cls(
+            local_date=summary["local_date"],
+            tz_name=summary["tz_name"],
+            tz_offset=summary["tz_offset"],
+            utc_start=summary["utc_start"],
+            utc_end=summary["utc_end"],
+            bytes_written=summary["bytes_written"],
+            bytes_read=summary["bytes_read"],
+            coverage=summary["coverage"],
+            sample_count=summary["sample_count"],
+            complete=summary["complete"],
+            detail_available=detail_available,
+            derived_from_surviving=derived_from_surviving,
+        )
+
+
 def _local_midnight_utc(dt: datetime, tz_name: str) -> datetime:
    """Compute the UTC time of the local midnight that contains *dt*.

@@ -85,9 +146,13 @@ def derive_local_day_summary(
    )
    rows = cursor.fetchall()

-    # Also check for a midnight-spanning hour before utc_start
-    # (UTC hour that starts before utc_start but ends after it)
-    # The relevant hour is the one whose floor-hour contains utc_start
+    # Check for a midnight-spanning UTC hour before utc_start.
+    # When the local midnight falls inside a UTC hour (e.g. UTC+5:30
+    # where local midnight is 18:30 UTC), the hour 18:00 straddles the
+    # boundary.  The main query (hour >= utc_start) excludes it because
+    # 18:00 < 18:30, so we must include it separately.  This does NOT
+    # double-count: the hour falls outside the query range by
+    # construction (issue #93).
    midnight_hour = utc_start.replace(minute=0, second=0, microsecond=0)
    midnight_hour_iso = midnight_hour.strftime("%Y-%m-%dT%H:00:00+00:00")
    prev_row = conn.execute(
@@ -111,13 +176,17 @@ def derive_local_day_summary(
        unknown_seconds += row[7] or 0
        hour_count += 1

-    # Handle midnight-spanning hour (starts before local midnight, ends after)
+    # Include midnight-spanning hour if it exists and is not already
+    # in the main query results (it won't be, since hour < utc_start).
    if prev_row is not None:
-        # This hour straddles the local midnight boundary
-        total_bw += prev_row[1] or 0
-        total_br += prev_row[2] or 0
-        total_samples += prev_row[3] or 0
-        hour_count += 1
+        # Verify this hour is NOT already counted in the main query
+        prev_hour_iso = prev_row[0]
+        already_counted = any(r[0] == prev_hour_iso for r in rows)
+        if not already_counted:
+            total_bw += prev_row[1] or 0
+            total_br += prev_row[2] or 0
+            total_samples += prev_row[3] or 0
+            hour_count += 1

    total_evidenced = known_seconds + unknown_seconds
    # Coverage is known seconds as a share of the full local day,
@@ -213,7 +282,7 @@ def query_local_day_summary(
        "bytes_read": row[6],
        "coverage": row[7],
        "sample_count": row[8],
-        "complete": row[9],
+        "complete": bool(row[9]),
    }


@@ -228,3 +297,83 @@ def query_current_local_day(
    local_dt = clock_now.astimezone(local_tz)
    local_date = local_dt.strftime("%Y-%m-%d")
    return query_local_day_summary(conn, local_date)
+
+
+def _is_detail_available(
+    conn: sqlite3.Connection,
+    utc_start: str,
+    utc_end: str,
+    now: datetime,
+    retention_days: int = 14,
+) -> bool:
+    """Check if raw samples covering the local-day range are still retained.
+
+    Returns True if at least one sample within [utc_start, utc_end] is
+    younger than retention_days.  This is a conservative check; the actual
+    pruning boundary depends on boundary-anchor logic.
+    """
+    cutoff = now - timedelta(days=retention_days)
+    cutoff_iso = cutoff.isoformat()
+
+    # If any sample in the range is newer than cutoff, detail is available
+    row = conn.execute(
+        "SELECT 1 FROM samples WHERE ts >= ? AND ts < ? AND ts >= ? LIMIT 1",
+        (utc_start, utc_end, cutoff_iso),
+    ).fetchone()
+    return row is not None
+
+
+def query_local_day_history(
+    conn: sqlite3.Connection,
+    start_date: str,
+    end_date: str,
+    now: datetime,
+) -> list[LocalDayHistoryEntry]:
+    """Query local-day summaries for a date range with evidence-limit metadata.
+
+    Returns history entries sorted by local_date, each annotated with
+    whether the underlying raw detail is still available (within the
+    14-day retention window) or has been pruned.  Timezone information
+    and UTC boundaries are always present, even after detail expires.
+
+    Args:
+        conn: Connection to the observation store.
+        start_date: Inclusive start date (e.g. "2026-09-01").
+        end_date: Inclusive end date (e.g. "2026-09-14").
+        now: Current UTC time for retention boundary check.
+
+    Returns:
+        List of LocalDayHistoryEntry sorted by local_date.
+    """
+    rows = conn.execute(
+        "SELECT local_date, tz_name, tz_offset, utc_start, utc_end, "
+        "       bytes_written, bytes_read, coverage, sample_count, complete "
+        "FROM local_days "
+        "WHERE local_date >= ? AND local_date <= ? "
+        "ORDER BY local_date",
+        (start_date, end_date),
+    ).fetchall()
+
+    entries = []
+    for row in rows:
+        summary = {
+            "local_date": row[0],
+            "tz_name": row[1],
+            "tz_offset": row[2],
+            "utc_start": row[3],
+            "utc_end": row[4],
+            "bytes_written": row[5],
+            "bytes_read": row[6],
+            "coverage": row[7],
+            "sample_count": row[8],
+            "complete": row[9],
+        }
+        detail_available = _is_detail_available(conn, row[3], row[4], now)
+        entries.append(
+            LocalDayHistoryEntry.from_summary(
+                summary,
+                detail_available=detail_available,
+                derived_from_surviving=True,
+            )
+        )
+    return entries
@@ -3,6 +3,12 @@
 Raw samples are pruned opportunistically to 14 days.
 Hour observations and day aggregates are retained indefinitely.
 Boundary anchors required for successor evidence are retained.
+
+Local-day summaries (local_days table) are never touched by pruning.
+They are persisted at collection time from hour observations and survive
+raw-sample pruning because they depend on hour observations, not on raw
+samples.  This is the mechanism that keeps local-day history trustworthy
+after detail expires (issue #93).
 """
 import sqlite3
 from datetime import datetime, timedelta, timezone