feat: retain pending publications (#97)
This commit is contained in:
xavierk
+181
-65
@@ -5,19 +5,21 @@ This module implements the thinnest complete write path:
|
||||
- Acquire controller identity from sysfs
|
||||
- Normalize identity exactly once at write time
|
||||
- Validate every row against store invariants
|
||||
- Stage valid observations privately before derivation
|
||||
- Publish the sample and derived evidence in one collection-owned transaction
|
||||
|
||||
No code path outside the collector interrogates the device.
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .store import init_store, get_store_path
|
||||
from .derive import derive_hours_from_interval, find_previous_sample
|
||||
from .monitoring_periods import ensure_period_open
|
||||
from .derive import find_previous_sample, derive_hours_from_interval
|
||||
from .store import init_store, get_store_path
|
||||
|
||||
|
||||
class AcquisitionError(Exception):
|
||||
@@ -30,6 +32,9 @@ class InvariantViolationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
PENDING_PUBLICATION_LIMIT = 6720
|
||||
|
||||
|
||||
def acquire_from_smartctl(smartctl_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Acquire counters and thermal evidence from smartctl -a -j data.
|
||||
|
||||
@@ -195,7 +200,7 @@ def write_sample(
|
||||
sample: Dict[str, Any],
|
||||
identity: Dict[str, Any],
|
||||
conn: sqlite3.Connection,
|
||||
clock,
|
||||
observed_at: datetime,
|
||||
) -> Dict[str, Any]:
|
||||
"""Write one sample to the observation store.
|
||||
|
||||
@@ -219,8 +224,7 @@ def write_sample(
|
||||
# Open new segment if needed
|
||||
segment_opened = False
|
||||
if should_open:
|
||||
now = clock.utcnow()
|
||||
open_segment(conn, now, identity, identity_key, identity_degraded)
|
||||
open_segment(conn, observed_at, identity, identity_key, identity_degraded)
|
||||
segment_opened = True
|
||||
|
||||
# Get current segment_id for provenance
|
||||
@@ -271,91 +275,203 @@ def write_sample(
|
||||
}
|
||||
|
||||
|
||||
def _observation_time(sample: Dict[str, Any]) -> datetime:
|
||||
"""Read an observation's original timestamp for ordered recovery."""
|
||||
observed_at = datetime.fromisoformat(sample["ts"])
|
||||
if observed_at.tzinfo is None:
|
||||
return observed_at.replace(tzinfo=timezone.utc)
|
||||
return observed_at.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _publish_observation(
|
||||
conn: sqlite3.Connection,
|
||||
sample: Dict[str, Any],
|
||||
identity: Dict[str, Any],
|
||||
tz_name: str,
|
||||
) -> None:
|
||||
"""Publish one staged sample and all dependent evidence in caller transaction."""
|
||||
observed_at = _observation_time(sample)
|
||||
validate_sample_invariants(sample, conn)
|
||||
ensure_period_open(conn, observed_at)
|
||||
seg_info = write_sample(sample, identity, conn, observed_at)
|
||||
|
||||
cursor = conn.execute("SELECT id FROM samples ORDER BY id DESC LIMIT 1")
|
||||
current_id = cursor.fetchone()[0]
|
||||
prev = find_previous_sample(conn, seg_info.get("segment_id"), current_id)
|
||||
if prev is not None:
|
||||
current = {
|
||||
"id": current_id,
|
||||
"ts": sample["ts"],
|
||||
"bytes_written": sample["bytes_written"],
|
||||
"bytes_read": sample["bytes_read"],
|
||||
"power_on_hours": sample["power_on_hours"],
|
||||
"temperature_c": sample["temperature_c"],
|
||||
"data_units_written": sample["data_units_written"],
|
||||
"data_units_read": sample["data_units_read"],
|
||||
}
|
||||
derive_hours_from_interval(conn, prev, current)
|
||||
|
||||
from .day_aggregate import derive_all_days, persist_day_aggregate
|
||||
for aggregate in derive_all_days(conn):
|
||||
persist_day_aggregate(conn, aggregate)
|
||||
|
||||
from .local_day import derive_local_day_summary, persist_local_day
|
||||
local_summary = derive_local_day_summary(conn, tz_name, observed_at)
|
||||
if local_summary is not None:
|
||||
persist_local_day(conn, local_summary)
|
||||
|
||||
|
||||
def _pending_count(conn: sqlite3.Connection) -> int:
|
||||
return conn.execute("SELECT COUNT(*) FROM pending_publications").fetchone()[0]
|
||||
|
||||
|
||||
def _stage_observation(
|
||||
conn: sqlite3.Connection,
|
||||
sample: Dict[str, Any],
|
||||
identity: Dict[str, Any],
|
||||
tz_name: str,
|
||||
) -> int:
|
||||
"""Durably stage valid acquired evidence before attempting publication."""
|
||||
payload = json.dumps(
|
||||
{"sample": sample, "identity": identity, "tz_name": tz_name},
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO pending_publications (sample_ts, payload) VALUES (?, ?)",
|
||||
(sample["ts"], payload),
|
||||
)
|
||||
return cursor.lastrowid
|
||||
|
||||
|
||||
def _recover_pending(conn: sqlite3.Connection) -> int:
|
||||
"""Publish pending observations oldest first; stop at first failure."""
|
||||
recovered = 0
|
||||
while True:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT id, payload FROM pending_publications ORDER BY id LIMIT 1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
conn.rollback()
|
||||
return recovered
|
||||
|
||||
pending_id, payload = row
|
||||
observation = json.loads(payload)
|
||||
_publish_observation(
|
||||
conn,
|
||||
observation["sample"],
|
||||
observation["identity"],
|
||||
observation["tz_name"],
|
||||
)
|
||||
conn.execute("DELETE FROM pending_publications WHERE id = ?", (pending_id,))
|
||||
conn.commit()
|
||||
recovered += 1
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def _recover_pending_for_collection(conn: sqlite3.Connection) -> int:
|
||||
"""Retry queued work and report exhausted capacity without masking store faults."""
|
||||
try:
|
||||
return _recover_pending(conn)
|
||||
except Exception as exc:
|
||||
if isinstance(exc, sqlite3.Error) and not isinstance(exc, sqlite3.IntegrityError):
|
||||
raise
|
||||
if isinstance(exc, InvariantViolationError):
|
||||
raise
|
||||
try:
|
||||
capacity_full = _pending_count(conn) >= PENDING_PUBLICATION_LIMIT
|
||||
except sqlite3.Error:
|
||||
raise exc
|
||||
if capacity_full:
|
||||
raise RuntimeError(
|
||||
"pending publication capacity full "
|
||||
f"({PENDING_PUBLICATION_LIMIT} observations); no new observation acquired; "
|
||||
f"recovery failed: {exc}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
|
||||
def run_collection(
|
||||
smartctl_data: Dict[str, Any],
|
||||
sysfs_path: Path,
|
||||
config: Dict[str, Any],
|
||||
clock,
|
||||
smartctl_data: Optional[Dict[str, Any]] = None,
|
||||
sysfs_path: Optional[Path] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
clock=None,
|
||||
*,
|
||||
acquire: Optional[Callable[[], tuple[Dict[str, Any], Path]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run one collection run.
|
||||
|
||||
This is the main entry point for the collector.
|
||||
Returns the run outcome.
|
||||
"""Recover old work, acquire one observation, then publish it atomically.
|
||||
|
||||
Production callers pass ``acquire`` so recovery and capacity checks run
|
||||
before device interrogation. Direct sample arguments remain useful for
|
||||
deterministic collector tests.
|
||||
"""
|
||||
conn = None
|
||||
try:
|
||||
# Acquire counters and thermal evidence
|
||||
counters = acquire_from_smartctl(smartctl_data)
|
||||
|
||||
# Acquire controller identity
|
||||
identity = acquire_from_sysfs(sysfs_path)
|
||||
|
||||
# Build sample with injected clock
|
||||
sample = {
|
||||
"ts": clock.utcnow().isoformat(),
|
||||
"device": config["device"],
|
||||
**counters,
|
||||
**identity,
|
||||
}
|
||||
|
||||
# Initialize store if needed
|
||||
if config is None or clock is None:
|
||||
raise ValueError("config and clock are required")
|
||||
|
||||
store_path = get_store_path(config)
|
||||
conn = init_store(store_path)
|
||||
|
||||
# Run legacy import if needed (idempotent)
|
||||
from .legacy import import_legacy_history
|
||||
history_path = Path(config.get("data_dir", ".")) / "history.jsonl"
|
||||
if history_path.exists():
|
||||
import_legacy_history(conn, history_path, clock=clock)
|
||||
|
||||
# Collection owns one transaction for the sample and its evidence.
|
||||
ensure_period_open(conn, clock.utcnow())
|
||||
recovered_count = _recover_pending_for_collection(conn)
|
||||
|
||||
validate_sample_invariants(sample, conn)
|
||||
seg_info = write_sample(sample, identity, conn, clock)
|
||||
# Hold the writer reservation across the capacity check and acquisition.
|
||||
# A concurrent collector will recheck pending work before it acquires.
|
||||
while True:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
waiting = _pending_count(conn)
|
||||
if waiting >= PENDING_PUBLICATION_LIMIT:
|
||||
conn.rollback()
|
||||
recovered_count += _recover_pending_for_collection(conn)
|
||||
continue
|
||||
if waiting:
|
||||
conn.rollback()
|
||||
recovered_count += _recover_pending_for_collection(conn)
|
||||
continue
|
||||
|
||||
cursor = conn.execute("SELECT id FROM samples ORDER BY id DESC LIMIT 1")
|
||||
current_id = cursor.fetchone()[0]
|
||||
prev = find_previous_sample(conn, seg_info.get("segment_id"), current_id)
|
||||
if prev is not None:
|
||||
current = {
|
||||
"id": current_id,
|
||||
"ts": sample["ts"],
|
||||
"bytes_written": sample["bytes_written"],
|
||||
"bytes_read": sample["bytes_read"],
|
||||
"power_on_hours": sample["power_on_hours"],
|
||||
"temperature_c": sample["temperature_c"],
|
||||
"data_units_written": sample["data_units_written"],
|
||||
"data_units_read": sample["data_units_read"],
|
||||
from .tz_util import detect_system_tz
|
||||
tz_name = detect_system_tz()
|
||||
if acquire is not None:
|
||||
smartctl_data, sysfs_path = acquire()
|
||||
if smartctl_data is None or sysfs_path is None:
|
||||
raise AcquisitionError("No acquired SMART data or sysfs identity")
|
||||
|
||||
counters = acquire_from_smartctl(smartctl_data)
|
||||
identity = acquire_from_sysfs(sysfs_path)
|
||||
sample = {
|
||||
"ts": clock.utcnow().isoformat(),
|
||||
"device": config["device"],
|
||||
**counters,
|
||||
**identity,
|
||||
}
|
||||
derive_hours_from_interval(conn, prev, current)
|
||||
|
||||
from .day_aggregate import derive_all_days, persist_day_aggregate
|
||||
for agg in derive_all_days(conn):
|
||||
persist_day_aggregate(conn, agg)
|
||||
|
||||
from .tz_util import detect_system_tz
|
||||
from .local_day import derive_local_day_summary, persist_local_day
|
||||
tz_name = detect_system_tz()
|
||||
local_summary = derive_local_day_summary(conn, tz_name, clock.utcnow())
|
||||
if local_summary is not None:
|
||||
persist_local_day(conn, local_summary)
|
||||
|
||||
conn.commit()
|
||||
validate_sample_invariants(sample, conn)
|
||||
_stage_observation(conn, sample, identity, tz_name)
|
||||
conn.commit()
|
||||
break
|
||||
|
||||
recovered_count += _recover_pending_for_collection(conn)
|
||||
return {
|
||||
"ok": True,
|
||||
"sample_count": 1,
|
||||
"sample_count": recovered_count,
|
||||
"store_path": str(store_path),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
except Exception as exc:
|
||||
if conn is not None:
|
||||
conn.rollback()
|
||||
return {
|
||||
"ok": False,
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"error": str(exc),
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
finally:
|
||||
if conn is not None:
|
||||
|
||||
Reference in New Issue
Block a user