405 lines
12 KiB
Python
405 lines
12 KiB
Python
"""Legacy migration tests.
|
|
|
|
Tests the idempotent, interruption-safe import of history.jsonl into the
|
|
observation store.
|
|
"""
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Dict
|
|
|
|
import pytest
|
|
|
|
# Add src to path for imports
|
|
import sys
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
|
|
from fenris.legacy import import_legacy_history, is_legacy_imported, _parse_history_line
|
|
from fenris.store import init_store
|
|
|
|
|
|
# Fixtures
|
|
|
|
@pytest.fixture
|
|
def history_fixture() -> str:
|
|
"""Minimal history.jsonl content with two samples."""
|
|
samples = [
|
|
{
|
|
"timestamp": "2026-08-30T10:00:00Z",
|
|
"device": "/dev/nvme0",
|
|
"model": "Samsung SSD 970 EVO Plus 1TB",
|
|
"serial": "S4EWNX0N123456",
|
|
"firmware_version": "2B2QEXM7",
|
|
"capacity_bytes": 1024000000000,
|
|
"data_units_written": 1000000,
|
|
"data_units_read": 500000,
|
|
"percentage_used": 5,
|
|
"power_on_hours": 8765,
|
|
"temperature": 35,
|
|
"available_spare": 100,
|
|
"media_errors": 0,
|
|
"power_cycles": 1234,
|
|
"unsafe_shutdowns": 5,
|
|
"critical_warning": 0,
|
|
},
|
|
{
|
|
"timestamp": "2026-08-30T11:00:00Z",
|
|
"device": "/dev/nvme0",
|
|
"model": "Samsung SSD 970 EVO Plus 1TB",
|
|
"serial": "S4EWNX0N123456",
|
|
"firmware_version": "2B2QEXM7",
|
|
"capacity_bytes": 1024000000000,
|
|
"data_units_written": 1001000,
|
|
"data_units_read": 501000,
|
|
"percentage_used": 5,
|
|
"power_on_hours": 8766,
|
|
"temperature": 36,
|
|
"available_spare": 100,
|
|
"media_errors": 0,
|
|
"power_cycles": 1234,
|
|
"unsafe_shutdowns": 5,
|
|
"critical_warning": 0,
|
|
},
|
|
]
|
|
return "\n".join(json.dumps(s) for s in samples)
|
|
|
|
|
|
@pytest.fixture
|
|
def hourly_fixture() -> str:
|
|
"""Minimal hourly.jsonl content for diffing."""
|
|
hours = [
|
|
{
|
|
"hour": "2026-08-30T10:00:00Z",
|
|
"bytes_written_delta": 512000000,
|
|
"bytes_read_delta": 256000000,
|
|
"sample_count": 1,
|
|
},
|
|
{
|
|
"hour": "2026-08-30T11:00:00Z",
|
|
"bytes_written_delta": 513000000,
|
|
"bytes_read_delta": 257000000,
|
|
"sample_count": 1,
|
|
},
|
|
]
|
|
return "\n".join(json.dumps(h) for h in hours)
|
|
|
|
|
|
@pytest.fixture
|
|
def config_fixture(tmp_path: Path) -> Dict[str, Any]:
|
|
"""Configuration fixture."""
|
|
return {
|
|
"device": "/dev/nvme0",
|
|
"store_path": str(tmp_path / "observations.db"),
|
|
"data_dir": str(tmp_path),
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def clock_fixture():
|
|
"""Injected clock returning fixed time."""
|
|
class FakeClock:
|
|
def __init__(self):
|
|
self.now = datetime(2026, 9, 1, 12, 0, 0, tzinfo=timezone.utc)
|
|
|
|
def utcnow(self):
|
|
return self.now
|
|
|
|
return FakeClock()
|
|
|
|
|
|
# Test: Idempotency - second run no-ops
|
|
|
|
def test_import_idempotent(
|
|
history_fixture: str,
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given a store with legacy import marker,
|
|
when import is called again,
|
|
then it no-ops."""
|
|
# Create history file
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_fixture)
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# First import
|
|
result1 = import_legacy_history(conn, history_path, clock=clock_fixture)
|
|
assert result1["ok"] is True
|
|
assert result1["skipped"] is False
|
|
|
|
# Second import (should no-op)
|
|
history_path2 = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path2.write_text(history_fixture)
|
|
result2 = import_legacy_history(conn, history_path2, clock=clock_fixture)
|
|
assert result2["ok"] is True
|
|
assert result2["skipped"] is True
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: Interruption safety - single transaction
|
|
|
|
def test_import_single_transaction(
|
|
history_fixture: str,
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given history.jsonl,
|
|
when import runs,
|
|
then the entire import is a single transaction."""
|
|
# Create history file
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_fixture)
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# Import
|
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
|
assert result["ok"] is True
|
|
|
|
# Verify all data was imported atomically
|
|
cursor = conn.execute("SELECT COUNT(*) FROM samples")
|
|
assert cursor.fetchone()[0] == 2
|
|
|
|
cursor = conn.execute("SELECT COUNT(*) FROM hour_observations")
|
|
assert cursor.fetchone()[0] == 2
|
|
|
|
cursor = conn.execute("SELECT COUNT(*) FROM monitoring_periods")
|
|
assert cursor.fetchone()[0] == 1
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: Legacy files renamed to *.migrated after commit
|
|
|
|
def test_legacy_files_renamed(
|
|
history_fixture: str,
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given history.jsonl and hourly.jsonl,
|
|
when import commits,
|
|
then files are renamed to *.migrated."""
|
|
# Create files
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_fixture)
|
|
|
|
hourly_path = Path(config_fixture["data_dir"]) / "hourly.jsonl"
|
|
hourly_path.write_text("{}")
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# Import
|
|
result = import_legacy_history(conn, history_path, hourly_path=hourly_path, clock=clock_fixture)
|
|
assert result["ok"] is True
|
|
|
|
# Verify files renamed
|
|
assert not history_path.exists()
|
|
assert history_path.with_suffix(history_path.suffix + ".migrated").exists()
|
|
|
|
assert not hourly_path.exists()
|
|
assert hourly_path.with_suffix(hourly_path.suffix + ".migrated").exists()
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: Malformed lines quarantined with logged count
|
|
|
|
def test_malformed_lines_quarantined(
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given history.jsonl with malformed lines,
|
|
when import runs,
|
|
then malformed lines are quarantined with logged count."""
|
|
# Create history with malformed lines
|
|
history_content = "\n".join([
|
|
'{"timestamp": "2026-08-30T10:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1000, "data_units_read": 500, "percentage_used": 5, "power_on_hours": 100, "temperature": 35}',
|
|
'NOT JSON',
|
|
'{"timestamp": "2026-08-30T11:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1001, "data_units_read": 501, "percentage_used": 5, "power_on_hours": 101, "temperature": 36}',
|
|
])
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_content)
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# Import
|
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
|
assert result["ok"] is True
|
|
assert result["malformed_lines"] == 1
|
|
assert result["samples_imported"] == 2
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: hourly.jsonl diffed and logged but never trusted
|
|
|
|
def test_hourly_jsonl_diffed(
|
|
history_fixture: str,
|
|
hourly_fixture: str,
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given history.jsonl and hourly.jsonl with mismatches,
|
|
when import runs,
|
|
then mismatches are diffed and logged."""
|
|
# Create files
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_fixture)
|
|
|
|
hourly_path = Path(config_fixture["data_dir"]) / "hourly.jsonl"
|
|
hourly_path.write_text(hourly_fixture)
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# Import (should not fail even with mismatches)
|
|
result = import_legacy_history(conn, history_path, hourly_path=hourly_path, clock=clock_fixture)
|
|
assert result["ok"] is True
|
|
|
|
# Verify data was imported from history.jsonl, not hourly.jsonl
|
|
cursor = conn.execute("SELECT bytes_written_delta FROM hour_observations ORDER BY hour")
|
|
deltas = [row[0] for row in cursor.fetchall()]
|
|
|
|
# Should match history.jsonl derived values, not hourly.jsonl
|
|
assert len(deltas) == 2
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: Legacy identity is mn-only
|
|
|
|
def test_legacy_identity_mn_only(
|
|
history_fixture: str,
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given history.jsonl,
|
|
when import runs,
|
|
then legacy segment has mn-only identity."""
|
|
# Create history file
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_fixture)
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# Import
|
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
|
assert result["ok"] is True
|
|
assert result["legacy_identity_key"] == "legacy|Samsung SSD 970 EVO Plus 1TB"
|
|
|
|
# Verify segment has mn-only identity
|
|
cursor = conn.execute("SELECT identity_key, mn, sn, subnqn FROM controller_segments")
|
|
row = cursor.fetchone()
|
|
assert row[0] == "legacy|Samsung SSD 970 EVO Plus 1TB"
|
|
assert row[1] == "Samsung SSD 970 EVO Plus 1TB"
|
|
assert row[2] is None # sn is None for legacy
|
|
assert row[3] is None # subnqn is None for legacy
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: No synthetic baseline created
|
|
|
|
def test_no_synthetic_baseline(
|
|
history_fixture: str,
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given history.jsonl,
|
|
when import runs,
|
|
then no endurance baseline is created."""
|
|
# Create history file
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_fixture)
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# Import
|
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
|
assert result["ok"] is True
|
|
|
|
# Verify no baseline created
|
|
cursor = conn.execute("SELECT COUNT(*) FROM endurance_baseline")
|
|
assert cursor.fetchone()[0] == 0
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: Monitoring period opened and closed
|
|
|
|
def test_monitoring_period_opened_closed(
|
|
history_fixture: str,
|
|
config_fixture: Dict[str, Any],
|
|
clock_fixture,
|
|
):
|
|
"""Given history.jsonl,
|
|
when import runs,
|
|
then one monitoring period is opened at first sample and closed at migration."""
|
|
# Create history file
|
|
history_path = Path(config_fixture["data_dir"]) / "history.jsonl"
|
|
history_path.write_text(history_fixture)
|
|
|
|
# Initialize store
|
|
store_path = Path(config_fixture["store_path"])
|
|
conn = init_store(store_path)
|
|
|
|
# Import
|
|
result = import_legacy_history(conn, history_path, clock=clock_fixture)
|
|
assert result["ok"] is True
|
|
|
|
# Verify monitoring period
|
|
cursor = conn.execute("SELECT started_at, ended_at, end_cause FROM monitoring_periods")
|
|
row = cursor.fetchone()
|
|
assert row[0] == "2026-08-30T10:00:00+00:00" # First sample time
|
|
assert row[1] == clock_fixture.now.isoformat() # Migration time
|
|
assert row[2] == "migrated"
|
|
|
|
conn.close()
|
|
|
|
|
|
# Test: Parse history line
|
|
|
|
def test_parse_history_line_valid():
|
|
"""Given a valid history line,
|
|
when parsed,
|
|
then returns the record."""
|
|
line = '{"timestamp": "2026-08-30T10:00:00Z", "model": "Test", "serial": "123", "firmware_version": "1.0", "data_units_written": 1000, "data_units_read": 500, "percentage_used": 5, "power_on_hours": 100, "temperature": 35}'
|
|
result = _parse_history_line(line, 1)
|
|
assert result is not None
|
|
assert result["model"] == "Test"
|
|
|
|
|
|
def test_parse_history_line_malformed():
|
|
"""Given a malformed JSON line,
|
|
when parsed,
|
|
then returns None."""
|
|
result = _parse_history_line("NOT JSON", 1)
|
|
assert result is None
|
|
|
|
|
|
def test_parse_history_line_missing_field():
|
|
"""Given a line with missing required field,
|
|
when parsed,
|
|
then returns None."""
|
|
line = '{"timestamp": "2026-08-30T10:00:00Z", "model": "Test"}'
|
|
result = _parse_history_line(line, 1)
|
|
assert result is None
|