xavierk/Fenris
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases 11 Wiki Activity

Implement runit support for Fenris monitoring (issue #84)

Browse Source 
Deliver end-to-end native monitoring path under runit with existing
CLI/TUI controls and truthful status, preserving systemd behavior.

Changes:
- Add init system abstraction layer (src/fenris/init_system.py) that
  detects systemd vs runit and provides unified interface for timer
  control, on-demand collection, and service state queries
- Create runit service files (units/runit/) with completion-relative
  5-minute cadence, 2-minute boot delay, bounded execution (90s),
  no catch-up, and serialized runs via flock
- Update monitor.py to use abstraction layer instead of direct systemctl
- Update status.py to use abstraction layer for service state queries
- Update all packaging scripts (deb, rpm) for init-system-aware setup
- Update Makefile to install runit service files alongside systemd units
- Add 46 tests for init system abstraction layer

Spec: ADR 0008, §8.4, §8.5, §8.6, §8.7, §8.8
Closes #84

Co-authored-by: CommandCodeBot <noreply@commandcode.ai>
This commit is contained in:
xavierk
2026-09-15 00:15:24 +05:30
co-authored by CommandCodeBot
parent 37a0ed7030
commit e27052d09a
15 changed files with 1390 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fenris/init_system.py
+479
View File
@@ -0,0 +1,479 @@
"""Init system abstraction for Fenris monitoring (issue #84).
Detects the active init system (systemd or runit) and provides a unified
interface for timer/collection control and service-state queries. This keeps
the privileged helper and status composition init-system agnostic without
introducing a generalized plugin framework.
Design: ADR 0008 — keep service-specific operations behind a cohesive
responsibility shared by the privileged control path and read-only status
composition.
Runit service layout:
/etc/sv/fenris-collect/run — scheduler (sleep 120; loop { collect; sleep 300 })
/etc/sv/fenris-collect/log/run — logger to /var/log/fenris-collect/
/var/service/fenris-collect — symlink to enable
/etc/sv/fenris-collect/down — marker for dormant install
Runit guarantees:
- Completion-relative 5-minute cadence (sleep 300 after each collect)
- Initial 2-minute boot delay (sleep 120 before first collect)
- Bounded execution (90s timeout via timeout(1))
- No catch-up (service sleeps fixed interval, no Persistent= flag)
- Serialized scheduled runs (runsv does not restart until exit)
- Serialized on-demand (flock serializes fenris-collect execution)
Spec: §8.4, §8.5, §8.6, §8.7, §8.8, ADR 0008
"""
import os
import shutil
import subprocess
import sys
from enum import Enum
from pathlib import Path
from typing import Any, Dict, Optional
# ---------------------------------------------------------------------------
# Init system detection
# ---------------------------------------------------------------------------
class InitSystem(Enum):
SYSTEMD = "systemd"
RUNIT = "runit"
FENRIS_SV_DIR = Path("/etc/sv/fenris-collect")
FENRIS_SERVICE_LINK = Path("/var/service/fenris-collect")
FENRIS_LOG_DIR = Path("/var/log/fenris-collect")
COLLECT_TIMEOUT_S = 90
def detect_init_system() -> InitSystem:
"""Detect the active init system.
Checks for systemd first (PID 1 is systemd or /run/systemd/system exists),
then falls back to runit (PID 1 is runsv or /etc/sv exists).
"""
# systemd detection: /run/systemd/system exists when systemd is PID 1
if Path("/run/systemd/system").exists():
return InitSystem.SYSTEMD
# Check PID 1 name
try:
pid1_comm = Path("/proc/1/comm").read_text().strip()
if pid1_comm == "systemd":
return InitSystem.SYSTEMD
if pid1_comm in ("runsv", "runsvdir"):
return InitSystem.RUNIT
except OSError:
pass
# Fallback: check for /etc/sv (Void Linux default)
if Path("/etc/sv").is_dir():
return InitSystem.RUNIT
# Default to systemd (existing behavior)
return InitSystem.SYSTEMD
def get_init_system() -> InitSystem:
"""Get the detected init system (cached)."""
if not hasattr(get_init_system, "_cached"):
get_init_system._cached = detect_init_system()
return get_init_system._cached
def reset_init_system_cache() -> None:
"""Reset the cached init system detection (for testing)."""
if hasattr(get_init_system, "_cached"):
delattr(get_init_system, "_cached")
# ---------------------------------------------------------------------------
# systemd backend
# ---------------------------------------------------------------------------
def _systemd_enable(now: bool) -> None:
"""Enable and optionally start the systemd timer."""
cmd = ["systemctl", "enable"]
if now:
cmd.append("--now")
cmd.append("fenris-collect.timer")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print("Error enabling timer:", result.stderr, file=sys.stderr)
sys.exit(1)
print("Timer enabled" + (" and started" if now else ""))
def _systemd_disable(now: bool) -> None:
"""Disable and optionally stop the systemd timer."""
cmd = ["systemctl", "disable"]
if now:
cmd.append("--now")
cmd.append("fenris-collect.timer")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print("Error disabling timer:", result.stderr, file=sys.stderr)
sys.exit(1)
print("Timer disabled" + (" and stopped" if now else ""))
def _systemd_collect() -> None:
"""Trigger on-demand collection via systemd (blocking)."""
result = subprocess.run(
["systemctl", "start", "fenris-collect.service"],
capture_output=True,
text=True,
)
if result.returncode == 0:
print("Collection completed successfully")
else:
print("Collection failed:", result.stderr, file=sys.stderr)
sys.exit(1)
def _systemctl_show(unit: str, *properties: str) -> Dict[str, str]:
"""Query systemctl show for specific properties."""
try:
result = subprocess.run(
["systemctl", "show", unit, "--property=" + ",".join(properties)],
capture_output=True, text=True, timeout=5,
)
if result.returncode != 0:
return {}
out = {}
for line in result.stdout.splitlines():
if "=" in line:
key, _, value = line.partition("=")
out[key.strip()] = value.strip()
return out
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
return {}
def _systemd_query_state() -> Dict[str, Any]:
"""Query systemd for the four separate service facts."""
from datetime import datetime, timezone
timer_props = _systemctl_show(
"fenris-collect.timer",
"UnitFileState", "ActiveState", "LastTriggerUSec",
)
service_props = _systemctl_show(
"fenris-collect.service",
"ActiveState", "ExecMainStatus", "ExecMainExitTimestamp",
)
boot_enabled_str = timer_props.get("UnitFileState", "")
boot_enabled = boot_enabled_str == "enabled"
active_state = timer_props.get("ActiveState", "inactive")
timer_active = active_state == "active"
last_collect_ok = None
last_collect_age_s = None
last_collect_reason = None
last_trigger = timer_props.get("LastTriggerUSec", "")
if last_trigger and last_trigger != "n/a":
try:
trigger_dt = datetime.fromisoformat(last_trigger.replace("Z", "+00:00"))
now = datetime.now(timezone.utc)
last_collect_age_s = int((now - trigger_dt).total_seconds())
except (ValueError, TypeError):
pass
exec_status = service_props.get("ExecMainStatus", "")
if exec_status:
try:
exit_code = int(exec_status)
last_collect_ok = exit_code == 0
if exit_code != 0:
last_collect_reason = "exit code %d" % exit_code
except (ValueError, TypeError):
pass
return {
"boot_enabled": boot_enabled,
"timer_active": timer_active,
"last_collect_ok": last_collect_ok,
"last_collect_age_s": last_collect_age_s,
"last_collect_reason": last_collect_reason,
}
def _systemd_journal_hint(lines: int = 5) -> Optional[str]:
"""Get the last N journal lines for fenris-collect.service."""
try:
result = subprocess.run(
["journalctl", "-u", "fenris-collect.service",
"--no-pager", "-n", str(lines), "--output=short-iso"],
capture_output=True, text=True, timeout=5,
)
if result.returncode != 0 or not result.stdout.strip():
return None
return result.stdout.strip()
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
return None
# ---------------------------------------------------------------------------
# runit backend
# ---------------------------------------------------------------------------
def _runit_enable(_now: bool) -> None:
"""Enable the runit service by creating a symlink.
runit activates the service immediately when the symlink appears.
If the service is already enabled but stopped (e.g., via `sv stop`),
restart it when `now=True`.
"""
if FENRIS_SERVICE_LINK.exists():
# Already enabled — check if we need to restart
if _now and not _runit_is_running():
# Service is stopped but enabled — restart via sv
try:
subprocess.run(
["sv", "restart", "fenris-collect"],
capture_output=True, text=True, timeout=5,
)
except (FileNotFoundError, subprocess.TimeoutExpired):
pass
print("Service already enabled (idempotent)")
return
# Remove the 'down' file if present (dormant install marker)
down_file = FENRIS_SV_DIR / "down"
if down_file.exists():
down_file.unlink()
FENRIS_SERVICE_LINK.symlink_to(FENRIS_SV_DIR)
print("Service enabled")
def _runit_disable(_now: bool) -> None:
"""Disable the runit service by removing the symlink.
runit stops the service immediately when the symlink is removed.
"""
if not FENRIS_SERVICE_LINK.exists():
print("Service already disabled (idempotent)")
return
FENRIS_SERVICE_LINK.unlink()
# Place 'down' file to mark as intentionally disabled
(FENRIS_SV_DIR / "down").touch()
print("Service disabled")
def _runit_collect() -> None:
"""Trigger on-demand collection via direct execution with flock.
Serializes against the scheduler using the same lock file.
The timeout(1) command enforces bounded execution.
"""
lock_path = Path("/var/lib/fenris/fenris-collect.lock")
collect_script = Path("/usr/libexec/fenris/fenris-collect")
fallback_script = Path(__file__).parent.parent.parent / "src" / "fenris" / "collect.py"
if collect_script.exists():
script = str(collect_script)
elif fallback_script.exists():
script = str(fallback_script)
else:
print("Error: fenris-collect script not found", file=sys.stderr)
sys.exit(1)
try:
result = subprocess.run(
["flock", "--nonblock", str(lock_path),
"timeout", str(COLLECT_TIMEOUT_S), "nice", "ionice", "-c3",
sys.executable, script],
capture_output=True,
text=True,
)
if result.returncode == 0:
print("Collection completed successfully")
elif result.returncode == 124:
print("Collection timed out after %ds" % COLLECT_TIMEOUT_S, file=sys.stderr)
sys.exit(1)
else:
# exit code 1 from flock means lock is held (scheduled run in progress)
if result.returncode == 1 and "Resource temporarily unavailable" in result.stderr:
print("Collection already in progress (serialized)", file=sys.stderr)
sys.exit(1)
print("Collection failed:", result.stderr, file=sys.stderr)
sys.exit(1)
except FileNotFoundError:
print("Error: flock/timeout not found", file=sys.stderr)
sys.exit(1)
def _runit_is_enabled() -> bool:
"""Check if the runit service is enabled (symlink exists)."""
return FENRIS_SERVICE_LINK.exists()
def _runit_is_running() -> bool:
"""Check if the runit service is currently running.
Looks for a 'supervise/pid' file in the service directory.
"""
pid_file = FENRIS_SV_DIR / "supervise" / "pid"
if not pid_file.exists():
return False
try:
pid = int(pid_file.read_text().strip())
# Check if the process is alive
os.kill(pid, 0)
return True
except (ValueError, OSError):
return False
def _runit_query_state() -> Dict[str, Any]:
"""Query runit for the four separate service facts.
Checks: boot_enabled (symlink), timer_active (running), last_collect
(store-based), freshness (store-based).
"""
from datetime import datetime, timezone
from pathlib import Path
boot_enabled = _runit_is_enabled()
timer_active = _runit_is_running()
last_collect_ok = None
last_collect_age_s = None
last_collect_reason = None
# Try to get last collection info from the store
store_path = Path("/var/lib/fenris/observations.db")
if store_path.exists():
try:
import sqlite3
conn = sqlite3.connect("file:%s?mode=ro" % store_path, uri=True)
conn.row_factory = sqlite3.Row
# Get newest sample
cursor = conn.execute(
"SELECT ts FROM samples ORDER BY id DESC LIMIT 1"
)
row = cursor.fetchone()
if row and row[0]:
try:
ts = datetime.fromisoformat(row[0])
if ts.tzinfo is None:
ts = ts.replace(tzinfo=timezone.utc)
now = datetime.now(timezone.utc)
last_collect_age_s = int((now - ts).total_seconds())
last_collect_ok = True
except (ValueError, TypeError):
pass
# Check for failed collections via monitoring_periods
cursor = conn.execute(
"SELECT end_cause FROM monitoring_periods "
"WHERE ended_at IS NOT NULL ORDER BY ended_at DESC LIMIT 1"
)
row = cursor.fetchone()
if row and row[0] and row[0] not in ("user_disabled", None):
last_collect_reason = row[0]
conn.close()
except Exception:
pass
# Check for timeout/exit failures via supervise exit status
if timer_active:
exit_file = FENRIS_SV_DIR / "supervise" / "exit"
if exit_file.exists():
try:
exit_code = int(exit_file.read_text().strip())
if exit_code != 0:
last_collect_ok = False
last_collect_reason = "exit code %d" % exit_code
except (ValueError, OSError):
pass
return {
"boot_enabled": boot_enabled,
"timer_active": timer_active,
"last_collect_ok": last_collect_ok,
"last_collect_age_s": last_collect_age_s,
"last_collect_reason": last_collect_reason,
}
def _runit_journal_hint(lines: int = 5) -> Optional[str]:
"""Get the last N log lines for fenris-collect from runit logging."""
log_current = FENRIS_LOG_DIR / "current"
if not log_current.exists():
return None
try:
# Use tail to get the last N lines
result = subprocess.run(
["tail", "-n", str(lines), str(log_current)],
capture_output=True, text=True, timeout=5,
)
if result.returncode != 0 or not result.stdout.strip():
return None
return result.stdout.strip()
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
return None
# ---------------------------------------------------------------------------
# Public API — unified interface
# ---------------------------------------------------------------------------
def enable_timer(now: bool) -> None:
"""Enable the collection timer/service."""
init = get_init_system()
if init == InitSystem.SYSTEMD:
_systemd_enable(now)
else:
_runit_enable(now)
def disable_timer(now: bool) -> None:
"""Disable the collection timer/service."""
init = get_init_system()
if init == InitSystem.SYSTEMD:
_systemd_disable(now)
else:
_runit_disable(now)
def collect_now() -> None:
"""Trigger on-demand collection (blocking)."""
init = get_init_system()
if init == InitSystem.SYSTEMD:
_systemd_collect()
else:
_runit_collect()
def query_service_state() -> Dict[str, Any]:
"""Query the four separate service facts."""
init = get_init_system()
if init == InitSystem.SYSTEMD:
return _systemd_query_state()
else:
return _runit_query_state()
def journal_hint(lines: int = 5, unit: str = "fenris-collect.service") -> Optional[str]:
"""Get journal/log hints for diagnostics.
The unit parameter is accepted for backward compatibility with callers
that pass 'fenris-collect.service'. For runit, the unit parameter is
ignored since the log directory is always /var/log/fenris-collect/.
"""
init = get_init_system()
if init == InitSystem.SYSTEMD:
return _systemd_journal_hint(lines)
else:
return _runit_journal_hint(lines)
src/fenris/monitor.py
+13 -54
View File
@@ -15,9 +15,7 @@ When run as a script, uses the fenris package from the installed wheel.
import argparse
import json
import os
import subprocess
import sys
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
@@ -39,6 +37,11 @@ from fenris.monitoring_periods import (
close_period,
get_open_period,
)
from fenris.init_system import (
enable_timer,
disable_timer,
collect_now,
)
DEFAULT_STORE_PATH = Path("/var/lib/fenris/observations.db")
@@ -70,25 +73,8 @@ def cmd_enable(args: argparse.Namespace) -> None:
else:
print("Monitoring period already open (id=%d)" % open_period["id"])
# Enable and start the timer
if args.now:
result = subprocess.run(
["systemctl", "enable", "--now", "fenris-collect.timer"],
capture_output=True,
text=True,
)
else:
result = subprocess.run(
["systemctl", "enable", "fenris-collect.timer"],
capture_output=True,
text=True,
)
if result.returncode != 0:
print("Error enabling timer:", result.stderr, file=sys.stderr)
sys.exit(1)
print("Timer enabled" + (" and started" if args.now else ""))
# Enable the timer (init-system aware)
enable_timer(args.now)
finally:
conn.close()
@@ -117,25 +103,8 @@ def cmd_disable(args: argparse.Namespace) -> None:
else:
print("No open monitoring period (no-op)")
# Disable and stop the timer
if args.now:
result = subprocess.run(
["systemctl", "disable", "--now", "fenris-collect.timer"],
capture_output=True,
text=True,
)
else:
result = subprocess.run(
["systemctl", "disable", "fenris-collect.timer"],
capture_output=True,
text=True,
)
if result.returncode != 0:
print("Error disabling timer:", result.stderr, file=sys.stderr)
sys.exit(1)
print("Timer disabled" + (" and stopped" if args.now else ""))
# Disable the timer (init-system aware)
disable_timer(args.now)
finally:
conn.close()
@@ -143,22 +112,12 @@ def cmd_disable(args: argparse.Namespace) -> None:
def cmd_collect(args: argparse.Namespace) -> None:
"""Trigger on-demand collection.
Starts fenris-collect.service, blocks until exit, reports outcome.
Starts fenris-collect, blocks until exit, reports outcome.
Spec §8.7: fenris sample routes through fenris-monitor → systemctl start,
which blocks until the oneshot exits; outcome reported synchronously.
Spec §8.7: fenris sample routes through fenris-monitor → collect,
which blocks until the collection exits; outcome reported synchronously.
"""
result = subprocess.run(
["systemctl", "start", "fenris-collect.service"],
capture_output=True,
text=True,
)
if result.returncode == 0:
print("Collection completed successfully")
else:
print("Collection failed:", result.stderr, file=sys.stderr)
sys.exit(1)
collect_now()
def cmd_baseline_set(args: argparse.Namespace) -> None:
src/fenris/status.py
+12 -94
View File
@@ -1,8 +1,8 @@
"""Read-only CLI status command: the CLI twin of the TUI (spec §8.8, LC-9, CI-2).
Composes from the observation store (read-only) and allow-listed systemctl
Composes from the observation store (read-only) and allow-listed service
properties: projection facts, four separate service facts (boot enablement,
runtime activity, last collect outcome, freshness), and a journalctl hint on
runtime activity, last collect outcome, freshness), and a journal/log hint on
failure or staleness. Never auto-samples, never prompts.
Freshness constants are defined once here and shared with the TUI (§8.9):
@@ -14,7 +14,6 @@ Freshness constants are defined once here and shared with the TUI (§8.9):
Criteria: LC-9, CI-2, CI-4, FL-4, FL-5, FL-7.
"""
import sqlite3
import subprocess
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
@@ -22,6 +21,10 @@ from typing import Any, Dict, List, Optional, Tuple
from .projection import compute_projection, ConfidenceState, DISCLOSURES
from .store import SCHEMA_VERSION
from .init_system import (
query_service_state as _init_query_service_state,
journal_hint as _init_journal_hint,
)
# ---------------------------------------------------------------------------
@@ -130,97 +133,12 @@ class NewerSchema(Exception):
# ---------------------------------------------------------------------------
# Service state queries (§8.8 — allow-listed systemctl properties)
# Service state queries (§8.8 — init-system agnostic)
# ---------------------------------------------------------------------------
def _systemctl_show(unit: str, *properties: str) -> Dict[str, str]:
"""Query systemctl show for specific properties. Returns empty dict on failure."""
try:
result = subprocess.run(
["systemctl", "show", unit, "--property=" + ",".join(properties)],
capture_output=True, text=True, timeout=5,
)
if result.returncode != 0:
return {}
out = {}
for line in result.stdout.splitlines():
if "=" in line:
key, _, value = line.partition("=")
out[key.strip()] = value.strip()
return out
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
return {}
def _journalctl_hint(unit: str, lines: int = 5) -> Optional[str]:
"""Get the last N journal lines for a unit. Returns None on failure."""
try:
result = subprocess.run(
["journalctl", "-u", unit, "--no-pager", "-n", str(lines), "--output=short-iso"],
capture_output=True, text=True, timeout=5,
)
if result.returncode != 0 or not result.stdout.strip():
return None
return result.stdout.strip()
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
return None
def query_service_state() -> Dict[str, Any]:
"""Query systemctl for the four separate service facts (§7.3, LC-9).
Returns dict with keys:
boot_enabled: bool
timer_active: bool
last_collect_ok: Optional[bool]
last_collect_age_s: Optional[int]
last_collect_reason: Optional[str]
"""
timer_props = _systemctl_show(
"fenris-collect.timer",
"UnitFileState", "ActiveState", "LastTriggerUSec",
)
service_props = _systemctl_show(
"fenris-collect.service",
"ActiveState", "ExecMainStatus", "ExecMainExitTimestamp",
)
boot_enabled_str = timer_props.get("UnitFileState", "")
boot_enabled = boot_enabled_str == "enabled"
active_state = timer_props.get("ActiveState", "inactive")
timer_active = active_state == "active"
last_collect_ok = None
last_collect_age_s = None
last_collect_reason = None
last_trigger = timer_props.get("LastTriggerUSec", "")
if last_trigger and last_trigger != "n/a":
try:
trigger_dt = datetime.fromisoformat(last_trigger.replace("Z", "+00:00"))
now = datetime.now(timezone.utc)
last_collect_age_s = int((now - trigger_dt).total_seconds())
except (ValueError, TypeError):
pass
exec_status = service_props.get("ExecMainStatus", "")
if exec_status:
try:
exit_code = int(exec_status)
last_collect_ok = exit_code == 0
if exit_code != 0:
last_collect_reason = "exit code %d" % exit_code
except (ValueError, TypeError):
pass
return {
"boot_enabled": boot_enabled,
"timer_active": timer_active,
"last_collect_ok": last_collect_ok,
"last_collect_age_s": last_collect_age_s,
"last_collect_reason": last_collect_reason,
}
# Re-export for backward compatibility with tests that import directly
query_service_state = _init_query_service_state
_journalctl_hint = _init_journal_hint
# ---------------------------------------------------------------------------
@@ -601,7 +519,7 @@ def get_status(store_path: Optional[Path] = None, clock_now: Optional[datetime]
if store_fault or newer_schema:
journal_hint = None
if query_journal:
journal_hint = _journalctl_hint("fenris-collect.service")
journal_hint = _journalctl_hint()
service["freshness"] = "unknown"
service["freshness_age_s"] = None
return _format_projection(
@@ -663,7 +581,7 @@ def get_status(store_path: Optional[Path] = None, clock_now: Optional[datetime]
# --- Journal hint on failure or staleness (§8.8) ---
journal_hint = None
if query_journal and freshness in ("missed", "stale"):
journal_hint = _journalctl_hint("fenris-collect.service")
journal_hint = _journalctl_hint()
# --- Compose output ---
result = _format_projection(