fix: rebuild trustworthy local-day evidence (#99)
This commit is contained in:
xavierk
+117
-89
@@ -11,7 +11,7 @@ from typing import Optional
|
||||
|
||||
|
||||
# Schema version - increment on each migration
|
||||
SCHEMA_VERSION = 5
|
||||
SCHEMA_VERSION = 6
|
||||
|
||||
|
||||
# Packaged default placement (spec §8.3). The config may override it, but a
|
||||
@@ -72,9 +72,11 @@ def init_store(store_path: Path) -> sqlite3.Connection:
|
||||
)
|
||||
elif current_version < SCHEMA_VERSION:
|
||||
# Older version - apply migrations
|
||||
_apply_migrations(conn, current_version)
|
||||
conn.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
|
||||
conn.commit()
|
||||
try:
|
||||
_apply_migrations(conn, current_version)
|
||||
except Exception:
|
||||
conn.close()
|
||||
raise
|
||||
|
||||
return conn
|
||||
|
||||
@@ -291,95 +293,119 @@ def _create_local_day_segment_totals(conn: sqlite3.Connection) -> None:
|
||||
|
||||
def _apply_migrations(conn: sqlite3.Connection, current_version: int):
|
||||
"""Apply forward-only migrations from current_version to SCHEMA_VERSION.
|
||||
|
||||
Each migration step is a transactional block. Add new steps as sequential
|
||||
elif branches when SCHEMA_VERSION increases.
|
||||
|
||||
Spec: §3.6, §10.2
|
||||
|
||||
Commit each version transition independently. A failed step rolls back in
|
||||
full while earlier successful steps remain versioned and retryable.
|
||||
"""
|
||||
# Migration 1→2: add segment_id provenance to samples,
|
||||
# unattributed byte tracking to day_aggregates (issue #73)
|
||||
if current_version < 2:
|
||||
# Defensive: only ALTER if table exists (handles minimal v1 stores)
|
||||
tables = {row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
).fetchall()}
|
||||
if "samples" in tables:
|
||||
# Check if column already exists (idempotent)
|
||||
cols = {row[1] for row in conn.execute("PRAGMA table_info(samples)").fetchall()}
|
||||
if "segment_id" not in cols:
|
||||
conn.execute("ALTER TABLE samples ADD COLUMN segment_id INTEGER")
|
||||
if "day_aggregates" in tables:
|
||||
cols = {row[1] for row in conn.execute("PRAGMA table_info(day_aggregates)").fetchall()}
|
||||
if "unattributed_bytes_written" not in cols:
|
||||
conn.execute("ALTER TABLE day_aggregates ADD COLUMN unattributed_bytes_written INTEGER DEFAULT 0")
|
||||
if "unattributed_bytes_read" not in cols:
|
||||
conn.execute("ALTER TABLE day_aggregates ADD COLUMN unattributed_bytes_read INTEGER DEFAULT 0")
|
||||
current_version = 2
|
||||
migrations = {
|
||||
2: _migrate_1_to_2,
|
||||
3: _migrate_2_to_3,
|
||||
4: _migrate_3_to_4,
|
||||
5: _migrate_4_to_5,
|
||||
6: _migrate_5_to_6,
|
||||
}
|
||||
while current_version < SCHEMA_VERSION:
|
||||
target_version = current_version + 1
|
||||
migration = migrations.get(target_version)
|
||||
if migration is None:
|
||||
raise ValueError(f"No migration registered for schema {target_version}")
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
migration(conn)
|
||||
conn.execute(f"PRAGMA user_version={target_version}")
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
current_version = target_version
|
||||
|
||||
# Migration 2→3: add local_days table for local-day activity totals
|
||||
# (issue #90, ADR 0010). Pure addition — no existing rows touched.
|
||||
if current_version < 3:
|
||||
tables = {row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
|
||||
def _migrate_1_to_2(conn: sqlite3.Connection) -> None:
|
||||
"""Add segment provenance and unattributed UTC byte tracking."""
|
||||
tables = {row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
).fetchall()}
|
||||
if "samples" in tables:
|
||||
cols = {row[1] for row in conn.execute(
|
||||
"PRAGMA table_info(samples)"
|
||||
).fetchall()}
|
||||
if "local_days" not in tables:
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS local_days (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
local_date TEXT NOT NULL,
|
||||
tz_name TEXT NOT NULL,
|
||||
tz_offset TEXT NOT NULL,
|
||||
utc_start TEXT NOT NULL,
|
||||
utc_end TEXT NOT NULL,
|
||||
bytes_written INTEGER DEFAULT 0,
|
||||
bytes_read INTEGER DEFAULT 0,
|
||||
coverage REAL DEFAULT 0.0,
|
||||
sample_count INTEGER DEFAULT 0,
|
||||
complete BOOLEAN DEFAULT 0,
|
||||
UNIQUE(local_date, tz_name)
|
||||
if "segment_id" not in cols:
|
||||
conn.execute("ALTER TABLE samples ADD COLUMN segment_id INTEGER")
|
||||
if "day_aggregates" in tables:
|
||||
cols = {row[1] for row in conn.execute(
|
||||
"PRAGMA table_info(day_aggregates)"
|
||||
).fetchall()}
|
||||
for column in ("unattributed_bytes_written", "unattributed_bytes_read"):
|
||||
if column not in cols:
|
||||
conn.execute(
|
||||
f"ALTER TABLE day_aggregates ADD COLUMN {column} INTEGER DEFAULT 0"
|
||||
)
|
||||
""")
|
||||
current_version = 3
|
||||
|
||||
# Migration 3→4: retain acquired observations until derived evidence can
|
||||
# be published atomically (issue #97, ADR 0011).
|
||||
if current_version < 4:
|
||||
_create_pending_publications(conn)
|
||||
current_version = 4
|
||||
|
||||
# Migration 4→5: retain precise measured local-day activity and once-only
|
||||
# unallocated intervals. Existing local-day totals came from UTC-hour
|
||||
# aggregates, so preserve their rows but mark their local precision legacy.
|
||||
if current_version < 5:
|
||||
tables = {row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
def _migrate_2_to_3(conn: sqlite3.Connection) -> None:
|
||||
"""Add local-day activity summaries."""
|
||||
tables = {row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
).fetchall()}
|
||||
if "local_days" not in tables:
|
||||
conn.execute("""
|
||||
CREATE TABLE local_days (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
local_date TEXT NOT NULL,
|
||||
tz_name TEXT NOT NULL,
|
||||
tz_offset TEXT NOT NULL,
|
||||
utc_start TEXT NOT NULL,
|
||||
utc_end TEXT NOT NULL,
|
||||
bytes_written INTEGER DEFAULT 0,
|
||||
bytes_read INTEGER DEFAULT 0,
|
||||
coverage REAL DEFAULT 0.0,
|
||||
sample_count INTEGER DEFAULT 0,
|
||||
complete BOOLEAN DEFAULT 0,
|
||||
UNIQUE(local_date, tz_name)
|
||||
)
|
||||
""")
|
||||
|
||||
|
||||
def _migrate_3_to_4(conn: sqlite3.Connection) -> None:
|
||||
"""Add private publication staging for acquired observations."""
|
||||
_create_pending_publications(conn)
|
||||
|
||||
|
||||
def _migrate_4_to_5(conn: sqlite3.Connection) -> None:
|
||||
"""Add measured local-day evidence storage and mark old totals legacy."""
|
||||
tables = {row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
).fetchall()}
|
||||
if "samples" in tables:
|
||||
sample_cols = {row[1] for row in conn.execute(
|
||||
"PRAGMA table_info(samples)"
|
||||
).fetchall()}
|
||||
if "samples" in tables:
|
||||
sample_cols = {row[1] for row in conn.execute(
|
||||
"PRAGMA table_info(samples)"
|
||||
).fetchall()}
|
||||
if "local_tz" not in sample_cols:
|
||||
conn.execute("ALTER TABLE samples ADD COLUMN local_tz TEXT")
|
||||
if "local_days" in tables:
|
||||
local_cols = {row[1] for row in conn.execute(
|
||||
"PRAGMA table_info(local_days)"
|
||||
).fetchall()}
|
||||
for column, declaration in (
|
||||
("activity_seconds", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("activity_intervals", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("activity_incomplete", "BOOLEAN NOT NULL DEFAULT 0"),
|
||||
("activity_precision", "TEXT NOT NULL DEFAULT 'legacy'"),
|
||||
("last_sample_id", "INTEGER"),
|
||||
):
|
||||
if column not in local_cols:
|
||||
conn.execute(
|
||||
"ALTER TABLE local_days ADD COLUMN %s %s" %
|
||||
(column, declaration)
|
||||
)
|
||||
_create_local_day_shared_evidence(conn)
|
||||
_create_local_day_segment_totals(conn)
|
||||
current_version = 5
|
||||
if "local_tz" not in sample_cols:
|
||||
conn.execute("ALTER TABLE samples ADD COLUMN local_tz TEXT")
|
||||
if "local_days" in tables:
|
||||
local_cols = {row[1] for row in conn.execute(
|
||||
"PRAGMA table_info(local_days)"
|
||||
).fetchall()}
|
||||
for column, declaration in (
|
||||
("activity_seconds", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("activity_intervals", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("activity_incomplete", "BOOLEAN NOT NULL DEFAULT 0"),
|
||||
("activity_precision", "TEXT NOT NULL DEFAULT 'legacy'"),
|
||||
("last_sample_id", "INTEGER"),
|
||||
):
|
||||
if column not in local_cols:
|
||||
conn.execute(
|
||||
f"ALTER TABLE local_days ADD COLUMN {column} {declaration}"
|
||||
)
|
||||
_create_local_day_shared_evidence(conn)
|
||||
_create_local_day_segment_totals(conn)
|
||||
|
||||
|
||||
def _migrate_5_to_6(conn: sqlite3.Connection) -> None:
|
||||
"""Rebuild local-day summaries from surviving trustworthy evidence."""
|
||||
from .local_day import repair_legacy_local_day_evidence
|
||||
|
||||
repair_legacy_local_day_evidence(conn)
|
||||
|
||||
|
||||
def migrate_to_latest(store_path: Path) -> int:
|
||||
@@ -416,9 +442,11 @@ def migrate_to_latest(store_path: Path) -> int:
|
||||
return SCHEMA_VERSION
|
||||
|
||||
steps = SCHEMA_VERSION - current_version
|
||||
_apply_migrations(conn, current_version)
|
||||
conn.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
|
||||
conn.commit()
|
||||
try:
|
||||
_apply_migrations(conn, current_version)
|
||||
except Exception:
|
||||
conn.close()
|
||||
raise
|
||||
conn.close()
|
||||
return steps
|
||||
|
||||
|
||||
Reference in New Issue
Block a user