feat: implement legacy history migration (closes #24)

2026-09-01 23:09:05 +05:30
parent 6a1841c447
commit d005412c0d
5 changed files with 861 additions and 0 deletions
@@ -0,0 +1,404 @@
+"""Legacy migration tests.
+
+Tests the idempotent, interruption-safe import of history.jsonl into the
+observation store.
+"""
+import json
+import os
+import sqlite3
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict
+
+import pytest
+
+# Add src to path for imports
+import sys
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from fenris.legacy import import_legacy_history, is_legacy_imported, _parse_history_line
+from fenris.store import init_store
+
+
+# Fixtures
+
+@pytest.fixture
+def history_fixture() -> str:
+    """Minimal history.jsonl content with two samples."""
+    samples = [
+        {
+            "timestamp": "2026-08-30T10:00:00Z",
+            "device": "/dev/nvme0",
+            "model": "Samsung SSD 970 EVO Plus 1TB",
+            "serial": "S4EWNX0N123456",
+            "firmware_version": "2B2QEXM7",
+            "capacity_bytes": 1024000000000,
+            "data_units_written": 1000000,
+            "data_units_read": 500000,
+            "percentage_used": 5,
+            "power_on_hours": 8765,
+            "temperature": 35,
+            "available_spare": 100,
+            "media_errors": 0,
+            "power_cycles": 1234,
+            "unsafe_shutdowns": 5,
+            "critical_warning": 0,
+        },
+        {
+            "timestamp": "2026-08-30T11:00:00Z",
+            "device": "/dev/nvme0",
+            "model": "Samsung SSD 970 EVO Plus 1TB",
+            "serial": "S4EWNX0N123456",
+            "firmware_version": "2B2QEXM7",
+            "capacity_bytes": 1024000000000,
+            "data_units_written": 1001000,
+            "data_units_read": 501000,
+            "percentage_used": 5,
+            "power_on_hours": 8766,
+            "temperature": 36,
+            "available_spare": 100,
+            "media_errors": 0,
+            "power_cycles": 1234,
+            "unsafe_shutdowns": 5,
+            "critical_warning": 0,
+        },
+    ]
+    return "\n".join(json.dumps(s) for s in samples)
+
+
+@pytest.fixture
+def hourly_fixture() -> str:
+    """Minimal hourly.jsonl content for diffing."""
+    hours = [
+        {
+            "hour": "2026-08-30T10:00:00Z",
+            "bytes_written_delta": 512000000,
+            "bytes_read_delta": 256000000,
+            "sample_count": 1,
+        },
+        {
+            "hour": "2026-08-30T11:00:00Z",
+            "bytes_written_delta": 513000000,
+            "bytes_read_delta": 257000000,
+            "sample_count": 1,
+        },
+    ]
+    return "\n".join(json.dumps(h) for h in hours)
+
+
+@pytest.fixture
+def config_fixture(tmp_path: Path) -> Dict[str, Any]:
+    """Configuration fixture."""
+    return {
+        "device": "/dev/nvme0",
+        "store_path": str(tmp_path / "observations.db"),
+        "data_dir": str(tmp_path),
+    }
+
+
+@pytest.fixture
+def clock_fixture():
+    """Injected clock returning fixed time."""
+    class FakeClock:
+        def __init__(self):
+            self.now = datetime(2026, 9, 1, 12, 0, 0, tzinfo=timezone.utc)
+        
+        def utcnow(self):
+            return self.now
+    
+    return FakeClock()
+
+
+# Test: Idempotency - second run no-ops
+
+def test_import_idempotent(
+    history_fixture: str,
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given a store with legacy import marker,
+    when import is called again,
+    then it no-ops."""
+    # Create history file
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_fixture)
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # First import
+    result1 = import_legacy_history(conn, history_path, clock=clock_fixture)
+    assert result1["ok"] is True
+    assert result1["skipped"] is False
+    
+    # Second import (should no-op)
+    history_path2 = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path2.write_text(history_fixture)
+    result2 = import_legacy_history(conn, history_path2, clock=clock_fixture)
+    assert result2["ok"] is True
+    assert result2["skipped"] is True
+    
+    conn.close()
+
+
+# Test: Interruption safety - single transaction
+
+def test_import_single_transaction(
+    history_fixture: str,
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given history.jsonl,
+    when import runs,
+    then the entire import is a single transaction."""
+    # Create history file
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_fixture)
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # Import
+    result = import_legacy_history(conn, history_path, clock=clock_fixture)
+    assert result["ok"] is True
+    
+    # Verify all data was imported atomically
+    cursor = conn.execute("SELECT COUNT(*) FROM samples")
+    assert cursor.fetchone()[0] == 2
+    
+    cursor = conn.execute("SELECT COUNT(*) FROM hour_observations")
+    assert cursor.fetchone()[0] == 2
+    
+    cursor = conn.execute("SELECT COUNT(*) FROM monitoring_periods")
+    assert cursor.fetchone()[0] == 1
+    
+    conn.close()
+
+
+# Test: Legacy files renamed to *.migrated after commit
+
+def test_legacy_files_renamed(
+    history_fixture: str,
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given history.jsonl and hourly.jsonl,
+    when import commits,
+    then files are renamed to *.migrated."""
+    # Create files
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_fixture)
+    
+    hourly_path = Path(config_fixture["data_dir"]) / "hourly.jsonl"
+    hourly_path.write_text("{}")
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # Import
+    result = import_legacy_history(conn, history_path, hourly_path=hourly_path, clock=clock_fixture)
+    assert result["ok"] is True
+    
+    # Verify files renamed
+    assert not history_path.exists()
+    assert history_path.with_suffix(history_path.suffix + ".migrated").exists()
+    
+    assert not hourly_path.exists()
+    assert hourly_path.with_suffix(hourly_path.suffix + ".migrated").exists()
+    
+    conn.close()
+
+
+# Test: Malformed lines quarantined with logged count
+
+def test_malformed_lines_quarantined(
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given history.jsonl with malformed lines,
+    when import runs,
+    then malformed lines are quarantined with logged count."""
+    # Create history with malformed lines
+    history_content = "\n".join([
+        '{"timestamp": "2026-08-30T10:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1000, "data_units_read": 500, "percentage_used": 5, "power_on_hours": 100, "temperature": 35}',
+        'NOT JSON',
+        '{"timestamp": "2026-08-30T11:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1001, "data_units_read": 501, "percentage_used": 5, "power_on_hours": 101, "temperature": 36}',
+    ])
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_content)
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # Import
+    result = import_legacy_history(conn, history_path, clock=clock_fixture)
+    assert result["ok"] is True
+    assert result["malformed_lines"] == 1
+    assert result["samples_imported"] == 2
+    
+    conn.close()
+
+
+# Test: hourly.jsonl diffed and logged but never trusted
+
+def test_hourly_jsonl_diffed(
+    history_fixture: str,
+    hourly_fixture: str,
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given history.jsonl and hourly.jsonl with mismatches,
+    when import runs,
+    then mismatches are diffed and logged."""
+    # Create files
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_fixture)
+    
+    hourly_path = Path(config_fixture["data_dir"]) / "hourly.jsonl"
+    hourly_path.write_text(hourly_fixture)
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # Import (should not fail even with mismatches)
+    result = import_legacy_history(conn, history_path, hourly_path=hourly_path, clock=clock_fixture)
+    assert result["ok"] is True
+    
+    # Verify data was imported from history.jsonl, not hourly.jsonl
+    cursor = conn.execute("SELECT bytes_written_delta FROM hour_observations ORDER BY hour")
+    deltas = [row[0] for row in cursor.fetchall()]
+    
+    # Should match history.jsonl derived values, not hourly.jsonl
+    assert len(deltas) == 2
+    
+    conn.close()
+
+
+# Test: Legacy identity is mn-only
+
+def test_legacy_identity_mn_only(
+    history_fixture: str,
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given history.jsonl,
+    when import runs,
+    then legacy segment has mn-only identity."""
+    # Create history file
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_fixture)
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # Import
+    result = import_legacy_history(conn, history_path, clock=clock_fixture)
+    assert result["ok"] is True
+    assert result["legacy_identity_key"] == "legacy|Samsung SSD 970 EVO Plus 1TB"
+    
+    # Verify segment has mn-only identity
+    cursor = conn.execute("SELECT identity_key, mn, sn, subnqn FROM controller_segments")
+    row = cursor.fetchone()
+    assert row[0] == "legacy|Samsung SSD 970 EVO Plus 1TB"
+    assert row[1] == "Samsung SSD 970 EVO Plus 1TB"
+    assert row[2] is None  # sn is None for legacy
+    assert row[3] is None  # subnqn is None for legacy
+    
+    conn.close()
+
+
+# Test: No synthetic baseline created
+
+def test_no_synthetic_baseline(
+    history_fixture: str,
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given history.jsonl,
+    when import runs,
+    then no endurance baseline is created."""
+    # Create history file
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_fixture)
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # Import
+    result = import_legacy_history(conn, history_path, clock=clock_fixture)
+    assert result["ok"] is True
+    
+    # Verify no baseline created
+    cursor = conn.execute("SELECT COUNT(*) FROM endurance_baseline")
+    assert cursor.fetchone()[0] == 0
+    
+    conn.close()
+
+
+# Test: Monitoring period opened and closed
+
+def test_monitoring_period_opened_closed(
+    history_fixture: str,
+    config_fixture: Dict[str, Any],
+    clock_fixture,
+):
+    """Given history.jsonl,
+    when import runs,
+    then one monitoring period is opened at first sample and closed at migration."""
+    # Create history file
+    history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
+    history_path.write_text(history_fixture)
+    
+    # Initialize store
+    store_path = Path(config_fixture["store_path"])
+    conn = init_store(store_path)
+    
+    # Import
+    result = import_legacy_history(conn, history_path, clock=clock_fixture)
+    assert result["ok"] is True
+    
+    # Verify monitoring period
+    cursor = conn.execute("SELECT started_at, ended_at, end_cause FROM monitoring_periods")
+    row = cursor.fetchone()
+    assert row[0] == "2026-08-30T10:00:00+00:00"  # First sample time
+    assert row[1] == clock_fixture.now.isoformat()  # Migration time
+    assert row[2] == "migrated"
+    
+    conn.close()
+
+
+# Test: Parse history line
+
+def test_parse_history_line_valid():
+    """Given a valid history line,
+    when parsed,
+    then returns the record."""
+    line = '{"timestamp": "2026-08-30T10:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1000, "data_units_read": 500, "percentage_used": 5, "power_on_hours": 100, "temperature": 35}'
+    result = _parse_history_line(line, 1)
+    assert result is not None
+    assert result["model"] == "Test"
+
+
+def test_parse_history_line_malformed():
+    """Given a malformed JSON line,
+    when parsed,
+    then returns None."""
+    result = _parse_history_line("NOT JSON", 1)
+    assert result is None
+
+
+def test_parse_history_line_missing_field():
+    """Given a line with missing required field,
+    when parsed,
+    then returns None."""
+    line = '{"timestamp": "2026-08-30T10:00:00Z", "model": "Test"}'
+    result = _parse_history_line(line, 1)
+    assert result is None