feat: implement legacy history migration (closes #24)
This commit is contained in:
@@ -295,6 +295,13 @@ def run_collection(
|
|||||||
store_path = get_store_path(config)
|
store_path = get_store_path(config)
|
||||||
conn = init_store(store_path)
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Run legacy import if needed (idempotent)
|
||||||
|
from .legacy import import_legacy_history
|
||||||
|
history_path = Path(config.get("data_dir", ".")) / "history.jsonl"
|
||||||
|
if history_path.exists():
|
||||||
|
import_legacy_history(conn, history_path, clock=clock)
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Validate invariants
|
# Validate invariants
|
||||||
validate_sample_invariants(sample, conn)
|
validate_sample_invariants(sample, conn)
|
||||||
|
|||||||
@@ -0,0 +1,440 @@
|
|||||||
|
"""Legacy migration: import history.jsonl into the observation store.
|
||||||
|
|
||||||
|
Spec §3.5, ADR 0001 §6. Idempotent and interruption-safe.
|
||||||
|
|
||||||
|
Entry points:
|
||||||
|
- Installer import detection at ./data/history.jsonl (§10.1)
|
||||||
|
- fenris import <path> (§8.8)
|
||||||
|
- Collector's first new-version run (ADR 0001 §6)
|
||||||
|
|
||||||
|
The import is a single transaction — a scripted kill mid-import leaves the
|
||||||
|
store fully pre- or fully post-migration. history.jsonl is the sole authority;
|
||||||
|
hourly.jsonl is diffed and logged but never trusted. Malformed lines are
|
||||||
|
quarantined with a logged count, never silently dropped. Legacy files are
|
||||||
|
renamed *.migrated only after commit and never deleted.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import sqlite3
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from .hour_classify import classify_hour
|
||||||
|
from .segment import open_segment, normalize_identity
|
||||||
|
from .monitoring_periods import close_period
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Legacy import marker — stored in a metadata table
|
||||||
|
LEGACY_IMPORT_MARKER = "legacy_imported"
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_metadata_table(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Create the metadata table if it doesn't exist."""
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS store_metadata (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def is_legacy_imported(conn: sqlite3.Connection) -> bool:
|
||||||
|
"""Check if legacy history has already been imported.
|
||||||
|
|
||||||
|
Spec §3.5.1: If the store already carries the legacy-import marker, do nothing.
|
||||||
|
"""
|
||||||
|
_ensure_metadata_table(conn)
|
||||||
|
cursor = conn.execute(
|
||||||
|
"SELECT value FROM store_metadata WHERE key = ?",
|
||||||
|
(LEGACY_IMPORT_MARKER,),
|
||||||
|
)
|
||||||
|
row = cursor.fetchone()
|
||||||
|
return row is not None and row[0] == "true"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_history_line(line: str, line_num: int) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Parse a single line from history.jsonl.
|
||||||
|
|
||||||
|
Returns None for malformed lines (quarantined, not silently dropped).
|
||||||
|
"""
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
record = json.loads(line)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
logger.warning("Malformed JSON at line %d: %s", line_num, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Validate required fields
|
||||||
|
required_fields = [
|
||||||
|
"timestamp", "model", "serial", "firmware_version",
|
||||||
|
"data_units_written", "data_units_read", "percentage_used",
|
||||||
|
"power_on_hours", "temperature",
|
||||||
|
]
|
||||||
|
|
||||||
|
for field in required_fields:
|
||||||
|
if field not in record:
|
||||||
|
logger.warning("Missing field '%s' at line %d", field, line_num)
|
||||||
|
return None
|
||||||
|
|
||||||
|
return record
|
||||||
|
|
||||||
|
|
||||||
|
def _record_to_sample(record: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Convert a legacy history.jsonl record to a sample dict."""
|
||||||
|
# Legacy records use different field names
|
||||||
|
duw = record["data_units_written"]
|
||||||
|
dur = record["data_units_read"]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"ts": record["timestamp"],
|
||||||
|
"device": record.get("device", "/dev/nvme0"),
|
||||||
|
"subnqn": record.get("subsystem_nqn", ""),
|
||||||
|
"sn": record["serial"],
|
||||||
|
"mn": record["model"],
|
||||||
|
"fr": record["firmware_version"],
|
||||||
|
"capacity_bytes": record.get("capacity_bytes", 0),
|
||||||
|
"percentage_used": record["percentage_used"],
|
||||||
|
"available_spare": record.get("available_spare"),
|
||||||
|
"media_errors": record.get("media_errors", 0),
|
||||||
|
"power_on_hours": record["power_on_hours"],
|
||||||
|
"power_cycles": record.get("power_cycles"),
|
||||||
|
"unsafe_shutdowns": record.get("unsafe_shutdowns"),
|
||||||
|
"temperature_c": record["temperature"],
|
||||||
|
"data_units_written": duw,
|
||||||
|
"data_units_read": dur,
|
||||||
|
"bytes_written": duw * 512000,
|
||||||
|
"bytes_read": dur * 512000,
|
||||||
|
"critical_warning": record.get("critical_warning", 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _derive_hour_observation(
|
||||||
|
samples: List[Dict[str, Any]],
|
||||||
|
hour_start: datetime,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Derive a single hour observation from samples in that hour.
|
||||||
|
|
||||||
|
Pre-migration hours carry an unknown activity split except directly
|
||||||
|
evidenced facts — a sample present means powered on; a DUW delta means
|
||||||
|
writes occurred (§3.5.4).
|
||||||
|
"""
|
||||||
|
hour_end = hour_start.replace(hour=hour_start.hour + 1) if hour_start.hour < 23 else hour_start.replace(hour=0, day=hour_start.day + 1)
|
||||||
|
|
||||||
|
# Filter samples in this hour
|
||||||
|
hour_samples = []
|
||||||
|
for s in samples:
|
||||||
|
ts = datetime.fromisoformat(s["ts"])
|
||||||
|
if hour_start <= ts < hour_end:
|
||||||
|
hour_samples.append(s)
|
||||||
|
|
||||||
|
if not hour_samples:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Sort by timestamp
|
||||||
|
hour_samples.sort(key=lambda x: x["ts"])
|
||||||
|
|
||||||
|
# Compute deltas from first to last sample in the hour
|
||||||
|
first = hour_samples[0]
|
||||||
|
last = hour_samples[-1]
|
||||||
|
|
||||||
|
duw_delta = last["bytes_written"] - first["bytes_written"]
|
||||||
|
dur_delta = last["bytes_read"] - first["bytes_read"]
|
||||||
|
poh_delta = (last["power_on_hours"] - first["power_on_hours"]) * 3600
|
||||||
|
|
||||||
|
# Temperature stats
|
||||||
|
temps = [s["temperature_c"] for s in hour_samples]
|
||||||
|
|
||||||
|
# Classify the hour
|
||||||
|
split = classify_hour(
|
||||||
|
wall_clock_seconds=3600,
|
||||||
|
poh_delta=poh_delta,
|
||||||
|
duw_delta=duw_delta,
|
||||||
|
dur_delta=dur_delta,
|
||||||
|
sampled_seconds=3600, # Legacy samples cover the full hour
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"hour": hour_start.strftime("%Y-%m-%dT%H:00:00Z"),
|
||||||
|
"active_seconds": split.seconds_active,
|
||||||
|
"idle_seconds": split.seconds_idle,
|
||||||
|
"powered_off_seconds": split.seconds_powered_off,
|
||||||
|
"unknown_seconds": split.seconds_unknown,
|
||||||
|
"bytes_written_delta": duw_delta,
|
||||||
|
"bytes_read_delta": dur_delta,
|
||||||
|
"temperature_min": min(temps),
|
||||||
|
"temperature_avg": sum(temps) / len(temps),
|
||||||
|
"temperature_max": max(temps),
|
||||||
|
"sample_count": len(hour_samples),
|
||||||
|
"coverage": 1.0 if split.seconds_unknown == 0 else (3600 - split.seconds_unknown) / 3600,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _diff_hourly_jsonl(
|
||||||
|
hourly_path: Path,
|
||||||
|
derived_hours: Dict[str, Dict[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
"""Diff hourly.jsonl against derived data and log mismatches.
|
||||||
|
|
||||||
|
Spec §3.5.3: hourly.jsonl is never trusted; mismatches are diffed and logged.
|
||||||
|
"""
|
||||||
|
if not hourly_path.exists():
|
||||||
|
logger.info("No hourly.jsonl found for diffing")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(hourly_path, "r") as f:
|
||||||
|
for line_num, line in enumerate(f, 1):
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
record = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
logger.warning("Malformed hourly.jsonl at line %d", line_num)
|
||||||
|
continue
|
||||||
|
|
||||||
|
hour_key = record.get("hour")
|
||||||
|
if hour_key not in derived_hours:
|
||||||
|
logger.info("Hourly.jsonl has hour %s not in derived data", hour_key)
|
||||||
|
continue
|
||||||
|
|
||||||
|
derived = derived_hours[hour_key]
|
||||||
|
mismatches = []
|
||||||
|
|
||||||
|
for field in ["bytes_written_delta", "bytes_read_delta", "sample_count"]:
|
||||||
|
if field in record and record[field] != derived.get(field):
|
||||||
|
mismatches.append(
|
||||||
|
f"{field}: hourly={record[field]} derived={derived.get(field)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if mismatches:
|
||||||
|
logger.info(
|
||||||
|
"Hourly.jsonl mismatch for %s: %s",
|
||||||
|
hour_key,
|
||||||
|
"; ".join(mismatches),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to diff hourly.jsonl: %s", e)
|
||||||
|
|
||||||
|
|
||||||
|
def import_legacy_history(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
history_path: Path,
|
||||||
|
hourly_path: Optional[Path] = None,
|
||||||
|
clock=None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Import legacy history.jsonl into the observation store.
|
||||||
|
|
||||||
|
This is the main entry point for legacy migration. It is:
|
||||||
|
- Idempotent: second run no-ops on the legacy-import marker
|
||||||
|
- Interruption-safe: single transaction
|
||||||
|
- Never creates synthetic baselines
|
||||||
|
|
||||||
|
Args:
|
||||||
|
conn: Connection to the observation store
|
||||||
|
history_path: Path to history.jsonl
|
||||||
|
hourly_path: Optional path to hourly.jsonl for diffing
|
||||||
|
clock: Injected clock (for testing)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with migration outcome
|
||||||
|
"""
|
||||||
|
# Check idempotency
|
||||||
|
if is_legacy_imported(conn):
|
||||||
|
return {"ok": True, "skipped": True, "reason": "already_imported"}
|
||||||
|
|
||||||
|
# Read and parse history.jsonl
|
||||||
|
samples = []
|
||||||
|
malformed_count = 0
|
||||||
|
|
||||||
|
if not history_path.exists():
|
||||||
|
return {"ok": False, "error": f"History file not found: {history_path}"}
|
||||||
|
|
||||||
|
with open(history_path, "r") as f:
|
||||||
|
for line_num, line in enumerate(f, 1):
|
||||||
|
record = _parse_history_line(line, line_num)
|
||||||
|
if record is None:
|
||||||
|
malformed_count += 1
|
||||||
|
continue
|
||||||
|
samples.append(_record_to_sample(record))
|
||||||
|
|
||||||
|
if malformed_count > 0:
|
||||||
|
logger.warning("Quarantined %d malformed lines from history.jsonl", malformed_count)
|
||||||
|
|
||||||
|
if not samples:
|
||||||
|
return {"ok": False, "error": "No valid samples found in history.jsonl"}
|
||||||
|
|
||||||
|
# Sort samples by timestamp
|
||||||
|
samples.sort(key=lambda x: x["ts"])
|
||||||
|
|
||||||
|
# Derive hour observations
|
||||||
|
hour_observations = {}
|
||||||
|
for sample in samples:
|
||||||
|
ts = datetime.fromisoformat(sample["ts"])
|
||||||
|
hour_start = ts.replace(minute=0, second=0, microsecond=0)
|
||||||
|
hour_key = hour_start.strftime("%Y-%m-%dT%H:00:00Z")
|
||||||
|
|
||||||
|
if hour_key not in hour_observations:
|
||||||
|
hour_observations[hour_key] = {
|
||||||
|
"hour": hour_start,
|
||||||
|
"samples": [],
|
||||||
|
}
|
||||||
|
hour_observations[hour_key]["samples"].append(sample)
|
||||||
|
|
||||||
|
derived_hours = {}
|
||||||
|
for hour_key, hour_data in hour_observations.items():
|
||||||
|
obs = _derive_hour_observation(hour_data["samples"], hour_data["hour"])
|
||||||
|
if obs is not None:
|
||||||
|
derived_hours[hour_key] = obs
|
||||||
|
|
||||||
|
# Diff against hourly.jsonl if provided
|
||||||
|
if hourly_path:
|
||||||
|
_diff_hourly_jsonl(hourly_path, derived_hours)
|
||||||
|
|
||||||
|
# Single transaction for the entire import
|
||||||
|
try:
|
||||||
|
# Begin transaction
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
|
||||||
|
# 1. Insert raw samples
|
||||||
|
for sample in samples:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO samples (
|
||||||
|
ts, device, subnqn, sn, mn, fr, capacity_bytes,
|
||||||
|
percentage_used, available_spare, media_errors, power_on_hours,
|
||||||
|
power_cycles, unsafe_shutdowns, temperature_c,
|
||||||
|
data_units_written, data_units_read, bytes_written, bytes_read,
|
||||||
|
critical_warning
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
sample["ts"],
|
||||||
|
sample["device"],
|
||||||
|
sample["subnqn"],
|
||||||
|
sample["sn"],
|
||||||
|
sample["mn"],
|
||||||
|
sample["fr"],
|
||||||
|
sample["capacity_bytes"],
|
||||||
|
sample["percentage_used"],
|
||||||
|
sample["available_spare"],
|
||||||
|
sample["media_errors"],
|
||||||
|
sample["power_on_hours"],
|
||||||
|
sample["power_cycles"],
|
||||||
|
sample["unsafe_shutdowns"],
|
||||||
|
sample["temperature_c"],
|
||||||
|
sample["data_units_written"],
|
||||||
|
sample["data_units_read"],
|
||||||
|
sample["bytes_written"],
|
||||||
|
sample["bytes_read"],
|
||||||
|
sample["critical_warning"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Insert derived hour observations
|
||||||
|
for hour_key, obs in sorted(derived_hours.items()):
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT OR REPLACE INTO hour_observations (
|
||||||
|
hour, active_seconds, idle_seconds, powered_off_seconds,
|
||||||
|
unknown_seconds, bytes_written_delta, bytes_read_delta,
|
||||||
|
temperature_min, temperature_avg, temperature_max,
|
||||||
|
sample_count, coverage
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
obs["hour"],
|
||||||
|
obs["active_seconds"],
|
||||||
|
obs["idle_seconds"],
|
||||||
|
obs["powered_off_seconds"],
|
||||||
|
obs["unknown_seconds"],
|
||||||
|
obs["bytes_written_delta"],
|
||||||
|
obs["bytes_read_delta"],
|
||||||
|
obs["temperature_min"],
|
||||||
|
obs["temperature_avg"],
|
||||||
|
obs["temperature_max"],
|
||||||
|
obs["sample_count"],
|
||||||
|
obs["coverage"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3. Open implicit monitoring period at first legacy sample
|
||||||
|
first_sample_ts = datetime.fromisoformat(samples[0]["ts"])
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO monitoring_periods (started_at) VALUES (?)",
|
||||||
|
(first_sample_ts.isoformat(),),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4. Close period with end_cause = migrated at migration moment
|
||||||
|
if clock:
|
||||||
|
migration_time = clock.utcnow()
|
||||||
|
else:
|
||||||
|
migration_time = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE monitoring_periods SET ended_at = ?, end_cause = ? WHERE ended_at IS NULL",
|
||||||
|
(migration_time.isoformat(), "migrated"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 5. Open legacy controller segment (mn-only)
|
||||||
|
# Legacy identity is model-scoped only (§4.4)
|
||||||
|
first_sample = samples[0]
|
||||||
|
legacy_identity = {
|
||||||
|
"mn": first_sample["mn"],
|
||||||
|
"sn": "", # Legacy segments are mn-only
|
||||||
|
"subnqn": "",
|
||||||
|
"fr": "",
|
||||||
|
}
|
||||||
|
legacy_identity_key = f"legacy|{first_sample['mn']}"
|
||||||
|
open_segment(
|
||||||
|
conn,
|
||||||
|
migration_time,
|
||||||
|
legacy_identity,
|
||||||
|
legacy_identity_key,
|
||||||
|
identity_degraded=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 6. Set legacy import marker
|
||||||
|
_ensure_metadata_table(conn)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO store_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(LEGACY_IMPORT_MARKER, "true"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Commit
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
conn.rollback()
|
||||||
|
raise RuntimeError(f"Migration failed: {e}") from e
|
||||||
|
|
||||||
|
# 7. Rename legacy files to *.migrated (only after commit)
|
||||||
|
try:
|
||||||
|
migrated_path = history_path.with_suffix(history_path.suffix + ".migrated")
|
||||||
|
history_path.rename(migrated_path)
|
||||||
|
logger.info("Renamed %s to %s", history_path, migrated_path)
|
||||||
|
|
||||||
|
if hourly_path and hourly_path.exists():
|
||||||
|
hourly_migrated = hourly_path.with_suffix(hourly_path.suffix + ".migrated")
|
||||||
|
hourly_path.rename(hourly_migrated)
|
||||||
|
logger.info("Renamed %s to %s", hourly_path, hourly_migrated)
|
||||||
|
except Exception as e:
|
||||||
|
# Non-fatal: files weren't renamed but migration succeeded
|
||||||
|
logger.warning("Failed to rename legacy files: %s", e)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"skipped": False,
|
||||||
|
"samples_imported": len(samples),
|
||||||
|
"hours_imported": len(derived_hours),
|
||||||
|
"malformed_lines": malformed_count,
|
||||||
|
"first_sample": samples[0]["ts"],
|
||||||
|
"last_sample": samples[-1]["ts"],
|
||||||
|
"legacy_identity_key": legacy_identity_key,
|
||||||
|
}
|
||||||
@@ -166,6 +166,15 @@ def _create_schema(conn: sqlite3.Connection):
|
|||||||
""")
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
# Metadata table for store state (e.g., legacy import marker)
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS store_metadata (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
def _apply_migrations(conn: sqlite3.Connection, current_version: int):
|
def _apply_migrations(conn: sqlite3.Connection, current_version: int):
|
||||||
"""Apply forward-only migrations from current_version to SCHEMA_VERSION."""
|
"""Apply forward-only migrations from current_version to SCHEMA_VERSION."""
|
||||||
# Future migrations will go here
|
# Future migrations will go here
|
||||||
|
|||||||
@@ -247,6 +247,7 @@ def test_store_initialization(config_fixture: Dict[str, Any]):
|
|||||||
|
|
||||||
# sqlite_sequence is a system table created by AUTOINCREMENT
|
# sqlite_sequence is a system table created by AUTOINCREMENT
|
||||||
expected_tables.add("sqlite_sequence")
|
expected_tables.add("sqlite_sequence")
|
||||||
|
expected_tables.add("store_metadata")
|
||||||
assert expected_tables == tables
|
assert expected_tables == tables
|
||||||
|
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|||||||
@@ -0,0 +1,404 @@
|
|||||||
|
"""Legacy migration tests.
|
||||||
|
|
||||||
|
Tests the idempotent, interruption-safe import of history.jsonl into the
|
||||||
|
observation store.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# Add src to path for imports
|
||||||
|
import sys
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
|
||||||
|
from fenris.legacy import import_legacy_history, is_legacy_imported, _parse_history_line
|
||||||
|
from fenris.store import init_store
|
||||||
|
|
||||||
|
|
||||||
|
# Fixtures
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def history_fixture() -> str:
|
||||||
|
"""Minimal history.jsonl content with two samples."""
|
||||||
|
samples = [
|
||||||
|
{
|
||||||
|
"timestamp": "2026-08-30T10:00:00Z",
|
||||||
|
"device": "/dev/nvme0",
|
||||||
|
"model": "Samsung SSD 970 EVO Plus 1TB",
|
||||||
|
"serial": "S4EWNX0N123456",
|
||||||
|
"firmware_version": "2B2QEXM7",
|
||||||
|
"capacity_bytes": 1024000000000,
|
||||||
|
"data_units_written": 1000000,
|
||||||
|
"data_units_read": 500000,
|
||||||
|
"percentage_used": 5,
|
||||||
|
"power_on_hours": 8765,
|
||||||
|
"temperature": 35,
|
||||||
|
"available_spare": 100,
|
||||||
|
"media_errors": 0,
|
||||||
|
"power_cycles": 1234,
|
||||||
|
"unsafe_shutdowns": 5,
|
||||||
|
"critical_warning": 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"timestamp": "2026-08-30T11:00:00Z",
|
||||||
|
"device": "/dev/nvme0",
|
||||||
|
"model": "Samsung SSD 970 EVO Plus 1TB",
|
||||||
|
"serial": "S4EWNX0N123456",
|
||||||
|
"firmware_version": "2B2QEXM7",
|
||||||
|
"capacity_bytes": 1024000000000,
|
||||||
|
"data_units_written": 1001000,
|
||||||
|
"data_units_read": 501000,
|
||||||
|
"percentage_used": 5,
|
||||||
|
"power_on_hours": 8766,
|
||||||
|
"temperature": 36,
|
||||||
|
"available_spare": 100,
|
||||||
|
"media_errors": 0,
|
||||||
|
"power_cycles": 1234,
|
||||||
|
"unsafe_shutdowns": 5,
|
||||||
|
"critical_warning": 0,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
return "\n".join(json.dumps(s) for s in samples)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def hourly_fixture() -> str:
|
||||||
|
"""Minimal hourly.jsonl content for diffing."""
|
||||||
|
hours = [
|
||||||
|
{
|
||||||
|
"hour": "2026-08-30T10:00:00Z",
|
||||||
|
"bytes_written_delta": 512000000,
|
||||||
|
"bytes_read_delta": 256000000,
|
||||||
|
"sample_count": 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hour": "2026-08-30T11:00:00Z",
|
||||||
|
"bytes_written_delta": 513000000,
|
||||||
|
"bytes_read_delta": 257000000,
|
||||||
|
"sample_count": 1,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
return "\n".join(json.dumps(h) for h in hours)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def config_fixture(tmp_path: Path) -> Dict[str, Any]:
|
||||||
|
"""Configuration fixture."""
|
||||||
|
return {
|
||||||
|
"device": "/dev/nvme0",
|
||||||
|
"store_path": str(tmp_path / "observations.db"),
|
||||||
|
"data_dir": str(tmp_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def clock_fixture():
|
||||||
|
"""Injected clock returning fixed time."""
|
||||||
|
class FakeClock:
|
||||||
|
def __init__(self):
|
||||||
|
self.now = datetime(2026, 9, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
def utcnow(self):
|
||||||
|
return self.now
|
||||||
|
|
||||||
|
return FakeClock()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: Idempotency - second run no-ops
|
||||||
|
|
||||||
|
def test_import_idempotent(
|
||||||
|
history_fixture: str,
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given a store with legacy import marker,
|
||||||
|
when import is called again,
|
||||||
|
then it no-ops."""
|
||||||
|
# Create history file
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_fixture)
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# First import
|
||||||
|
result1 = import_legacy_history(conn, history_path, clock=clock_fixture)
|
||||||
|
assert result1["ok"] is True
|
||||||
|
assert result1["skipped"] is False
|
||||||
|
|
||||||
|
# Second import (should no-op)
|
||||||
|
history_path2 = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path2.write_text(history_fixture)
|
||||||
|
result2 = import_legacy_history(conn, history_path2, clock=clock_fixture)
|
||||||
|
assert result2["ok"] is True
|
||||||
|
assert result2["skipped"] is True
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: Interruption safety - single transaction
|
||||||
|
|
||||||
|
def test_import_single_transaction(
|
||||||
|
history_fixture: str,
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given history.jsonl,
|
||||||
|
when import runs,
|
||||||
|
then the entire import is a single transaction."""
|
||||||
|
# Create history file
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_fixture)
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Import
|
||||||
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
# Verify all data was imported atomically
|
||||||
|
cursor = conn.execute("SELECT COUNT(*) FROM samples")
|
||||||
|
assert cursor.fetchone()[0] == 2
|
||||||
|
|
||||||
|
cursor = conn.execute("SELECT COUNT(*) FROM hour_observations")
|
||||||
|
assert cursor.fetchone()[0] == 2
|
||||||
|
|
||||||
|
cursor = conn.execute("SELECT COUNT(*) FROM monitoring_periods")
|
||||||
|
assert cursor.fetchone()[0] == 1
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: Legacy files renamed to *.migrated after commit
|
||||||
|
|
||||||
|
def test_legacy_files_renamed(
|
||||||
|
history_fixture: str,
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given history.jsonl and hourly.jsonl,
|
||||||
|
when import commits,
|
||||||
|
then files are renamed to *.migrated."""
|
||||||
|
# Create files
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_fixture)
|
||||||
|
|
||||||
|
hourly_path = Path(config_fixture["data_dir"]) / "hourly.jsonl"
|
||||||
|
hourly_path.write_text("{}")
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Import
|
||||||
|
result = import_legacy_history(conn, history_path, hourly_path=hourly_path, clock=clock_fixture)
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
# Verify files renamed
|
||||||
|
assert not history_path.exists()
|
||||||
|
assert history_path.with_suffix(history_path.suffix + ".migrated").exists()
|
||||||
|
|
||||||
|
assert not hourly_path.exists()
|
||||||
|
assert hourly_path.with_suffix(hourly_path.suffix + ".migrated").exists()
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: Malformed lines quarantined with logged count
|
||||||
|
|
||||||
|
def test_malformed_lines_quarantined(
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given history.jsonl with malformed lines,
|
||||||
|
when import runs,
|
||||||
|
then malformed lines are quarantined with logged count."""
|
||||||
|
# Create history with malformed lines
|
||||||
|
history_content = "\n".join([
|
||||||
|
'{"timestamp": "2026-08-30T10:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1000, "data_units_read": 500, "percentage_used": 5, "power_on_hours": 100, "temperature": 35}',
|
||||||
|
'NOT JSON',
|
||||||
|
'{"timestamp": "2026-08-30T11:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1001, "data_units_read": 501, "percentage_used": 5, "power_on_hours": 101, "temperature": 36}',
|
||||||
|
])
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_content)
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Import
|
||||||
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
||||||
|
assert result["ok"] is True
|
||||||
|
assert result["malformed_lines"] == 1
|
||||||
|
assert result["samples_imported"] == 2
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: hourly.jsonl diffed and logged but never trusted
|
||||||
|
|
||||||
|
def test_hourly_jsonl_diffed(
|
||||||
|
history_fixture: str,
|
||||||
|
hourly_fixture: str,
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given history.jsonl and hourly.jsonl with mismatches,
|
||||||
|
when import runs,
|
||||||
|
then mismatches are diffed and logged."""
|
||||||
|
# Create files
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_fixture)
|
||||||
|
|
||||||
|
hourly_path = Path(config_fixture["data_dir"]) / "hourly.jsonl"
|
||||||
|
hourly_path.write_text(hourly_fixture)
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Import (should not fail even with mismatches)
|
||||||
|
result = import_legacy_history(conn, history_path, hourly_path=hourly_path, clock=clock_fixture)
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
# Verify data was imported from history.jsonl, not hourly.jsonl
|
||||||
|
cursor = conn.execute("SELECT bytes_written_delta FROM hour_observations ORDER BY hour")
|
||||||
|
deltas = [row[0] for row in cursor.fetchall()]
|
||||||
|
|
||||||
|
# Should match history.jsonl derived values, not hourly.jsonl
|
||||||
|
assert len(deltas) == 2
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: Legacy identity is mn-only
|
||||||
|
|
||||||
|
def test_legacy_identity_mn_only(
|
||||||
|
history_fixture: str,
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given history.jsonl,
|
||||||
|
when import runs,
|
||||||
|
then legacy segment has mn-only identity."""
|
||||||
|
# Create history file
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_fixture)
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Import
|
||||||
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
||||||
|
assert result["ok"] is True
|
||||||
|
assert result["legacy_identity_key"] == "legacy|Samsung SSD 970 EVO Plus 1TB"
|
||||||
|
|
||||||
|
# Verify segment has mn-only identity
|
||||||
|
cursor = conn.execute("SELECT identity_key, mn, sn, subnqn FROM controller_segments")
|
||||||
|
row = cursor.fetchone()
|
||||||
|
assert row[0] == "legacy|Samsung SSD 970 EVO Plus 1TB"
|
||||||
|
assert row[1] == "Samsung SSD 970 EVO Plus 1TB"
|
||||||
|
assert row[2] is None # sn is None for legacy
|
||||||
|
assert row[3] is None # subnqn is None for legacy
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: No synthetic baseline created
|
||||||
|
|
||||||
|
def test_no_synthetic_baseline(
|
||||||
|
history_fixture: str,
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given history.jsonl,
|
||||||
|
when import runs,
|
||||||
|
then no endurance baseline is created."""
|
||||||
|
# Create history file
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_fixture)
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Import
|
||||||
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
# Verify no baseline created
|
||||||
|
cursor = conn.execute("SELECT COUNT(*) FROM endurance_baseline")
|
||||||
|
assert cursor.fetchone()[0] == 0
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: Monitoring period opened and closed
|
||||||
|
|
||||||
|
def test_monitoring_period_opened_closed(
|
||||||
|
history_fixture: str,
|
||||||
|
config_fixture: Dict[str, Any],
|
||||||
|
clock_fixture,
|
||||||
|
):
|
||||||
|
"""Given history.jsonl,
|
||||||
|
when import runs,
|
||||||
|
then one monitoring period is opened at first sample and closed at migration."""
|
||||||
|
# Create history file
|
||||||
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
||||||
|
history_path.write_text(history_fixture)
|
||||||
|
|
||||||
|
# Initialize store
|
||||||
|
store_path = Path(config_fixture["store_path"])
|
||||||
|
conn = init_store(store_path)
|
||||||
|
|
||||||
|
# Import
|
||||||
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
# Verify monitoring period
|
||||||
|
cursor = conn.execute("SELECT started_at, ended_at, end_cause FROM monitoring_periods")
|
||||||
|
row = cursor.fetchone()
|
||||||
|
assert row[0] == "2026-08-30T10:00:00+00:00" # First sample time
|
||||||
|
assert row[1] == clock_fixture.now.isoformat() # Migration time
|
||||||
|
assert row[2] == "migrated"
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Test: Parse history line
|
||||||
|
|
||||||
|
def test_parse_history_line_valid():
|
||||||
|
"""Given a valid history line,
|
||||||
|
when parsed,
|
||||||
|
then returns the record."""
|
||||||
|
line = '{"timestamp": "2026-08-30T10:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1000, "data_units_read": 500, "percentage_used": 5, "power_on_hours": 100, "temperature": 35}'
|
||||||
|
result = _parse_history_line(line, 1)
|
||||||
|
assert result is not None
|
||||||
|
assert result["model"] == "Test"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_history_line_malformed():
|
||||||
|
"""Given a malformed JSON line,
|
||||||
|
when parsed,
|
||||||
|
then returns None."""
|
||||||
|
result = _parse_history_line("NOT JSON", 1)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_history_line_missing_field():
|
||||||
|
"""Given a line with missing required field,
|
||||||
|
when parsed,
|
||||||
|
then returns None."""
|
||||||
|
line = '{"timestamp": "2026-08-30T10:00:00Z", "model": "Test"}'
|
||||||
|
result = _parse_history_line(line, 1)
|
||||||
|
assert result is None
|
||||||
Reference in New Issue
Block a user