Merge pull request 'feat: LLM reasoning, severity overrides, dashboard freshness' (#14) from feat/llm-reasoning into main

This commit is contained in:
pyr0ball 2026-05-11 13:00:52 -07:00
commit 3c758d3626
9 changed files with 465 additions and 41 deletions

View file

@ -62,13 +62,29 @@ def _startup() -> None:
ensure_schema(DB_PATH)
_PREFS_DEFAULTS: dict = {
"entry_point_style": "topbar",
"llm_url": "http://localhost:11434",
"llm_model": "llama3.1:8b",
"severity_overrides": [
{
"name": "PAM auth noise",
"pattern": r"pam_unix.*auth(?:entication)?\s+fail|auth could not identify",
"override_severity": "WARN",
"enabled": True,
}
],
}
def _load_prefs() -> dict[str, str]:
if PREFS_PATH.exists():
try:
return json.loads(PREFS_PATH.read_text())
saved = json.loads(PREFS_PATH.read_text())
return {**_PREFS_DEFAULTS, **saved}
except (json.JSONDecodeError, OSError):
pass
return {"entry_point_style": "topbar"}
return dict(_PREFS_DEFAULTS)
def _save_prefs(data: dict[str, str]) -> None:
@ -81,8 +97,18 @@ class DiagnoseRequest(BaseModel):
until: str | None = None
class SeverityOverride(BaseModel):
name: str
pattern: str
override_severity: str
enabled: bool = True
class SettingsBody(BaseModel):
entry_point_style: str
entry_point_style: str | None = None
llm_url: str | None = None
llm_model: str | None = None
severity_overrides: list[SeverityOverride] | None = None
class IncidentCreate(BaseModel):
@ -202,9 +228,18 @@ def diagnose_post(body: DiagnoseRequest) -> dict:
},
"entries": [],
}
result = _diagnose(DB_PATH, query=body.query, since=body.since, until=body.until)
prefs = _load_prefs()
result = _diagnose(
DB_PATH,
query=body.query,
since=body.since,
until=body.until,
llm_url=prefs.get("llm_url") or None,
llm_model=prefs.get("llm_model") or None,
)
return {
"summary": result["summary"],
"reasoning": result.get("reasoning"),
"entries": [dataclasses.asdict(r) for r in result["entries"]],
}
@ -216,10 +251,17 @@ def get_settings() -> dict:
@router.patch("/api/settings")
def patch_settings(body: SettingsBody) -> dict:
prefs = _load_prefs()
if body.entry_point_style is not None:
if body.entry_point_style not in ("topbar", "fab"):
raise HTTPException(status_code=422, detail="entry_point_style must be 'topbar' or 'fab'")
prefs = _load_prefs()
prefs["entry_point_style"] = body.entry_point_style
if body.llm_url is not None:
prefs["llm_url"] = body.llm_url
if body.llm_model is not None:
prefs["llm_model"] = body.llm_model
if body.severity_overrides is not None:
prefs["severity_overrides"] = [o.model_dump() for o in body.severity_overrides]
_save_prefs(prefs)
return prefs
@ -233,7 +275,8 @@ def list_sources() -> dict:
def get_stats(
window: Annotated[int, Query(ge=1, le=168, description="Hours to look back")] = 24,
) -> dict:
return _stats(DB_PATH, window_hours=window)
prefs = _load_prefs()
return _stats(DB_PATH, window_hours=window, severity_overrides=prefs.get("severity_overrides", []))
@router.post("/api/incidents")

View file

@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from app.services.llm import summarize
from app.services.search import SearchResult, entries_in_window, search
logger = logging.getLogger(__name__)
@ -48,6 +49,8 @@ def diagnose(
query: str,
since: str | None = None,
until: str | None = None,
llm_url: str | None = None,
llm_model: str | None = None,
) -> dict[str, Any]:
"""Run layered log search with NL time extraction. Returns summary + entries."""
time_detected = since is not None and until is not None
@ -79,6 +82,10 @@ def diagnose(
by_severity[sev] += 1
by_source[r.source_id] = by_source.get(r.source_id, 0) + 1
reasoning: str | None = None
if llm_url and llm_model:
reasoning = summarize(query, combined, llm_url=llm_url, llm_model=llm_model)
return {
"summary": {
"total": len(combined),
@ -88,6 +95,7 @@ def diagnose(
"by_severity": by_severity,
"by_source": by_source,
},
"reasoning": reasoning,
"entries": combined,
}

56
app/services/llm.py Normal file
View file

@ -0,0 +1,56 @@
import logging
import httpx
from app.services.search import SearchResult
logger = logging.getLogger(__name__)
_SEVERITY_RANK = {"CRITICAL": 0, "ERROR": 1, "WARN": 2, "WARNING": 2}
_PROMPT_TEMPLATE = """\
You are a homelab diagnostic assistant. A user described a symptom and the system retrieved relevant log entries.
Analyze the log entries below and write a 2-4 sentence plain-language diagnosis. Focus on errors and their likely root cause. Be specific and concise name the services involved, not generic platitudes.
User query: {query}
Log entries ({n} shown, highest severity first):
{log_block}
Diagnosis:"""
def _build_context(entries: list[SearchResult], max_entries: int = 25) -> str:
ranked = sorted(
entries,
key=lambda e: (_SEVERITY_RANK.get(e.severity or "", 3), e.timestamp_iso or ""),
)[:max_entries]
return "\n".join(
f"[{e.timestamp_iso or '?'}] [{e.severity or 'INFO'}] {e.text[:200]}"
for e in ranked
)
def summarize(
query: str,
entries: list[SearchResult],
llm_url: str,
llm_model: str,
timeout: float = 20.0,
) -> str | None:
if not entries:
return None
log_block = _build_context(entries)
prompt = _PROMPT_TEMPLATE.format(query=query, n=min(len(entries), 25), log_block=log_block)
try:
resp = httpx.post(
f"{llm_url.rstrip('/')}/api/generate",
json={"model": llm_model, "prompt": prompt, "stream": False},
timeout=timeout,
)
resp.raise_for_status()
return resp.json().get("response", "").strip() or None
except Exception as exc:
logger.warning("LLM summarization failed (%s): %s", type(exc).__name__, exc)
return None

View file

@ -317,11 +317,33 @@ def list_sources(db_path: Path) -> list[dict]:
]
def stats_summary(db_path: Path, window_hours: int = 24) -> dict:
def _compile_overrides(overrides: list[dict]) -> list[tuple[re.Pattern[str], str]]:
"""Return (compiled_pattern, override_severity) pairs for enabled rules."""
compiled = []
for rule in overrides:
if not rule.get("enabled", True):
continue
try:
compiled.append((re.compile(rule["pattern"], re.IGNORECASE), rule["override_severity"]))
except re.error:
pass
return compiled
def _apply_overrides(text: str, original_severity: str, rules: list[tuple[re.Pattern[str], str]]) -> str:
for pattern, new_sev in rules:
if pattern.search(text):
return new_sev
return original_severity
def stats_summary(db_path: Path, window_hours: int = 24, severity_overrides: list[dict] | None = None) -> dict:
"""Return aggregate health stats for the dashboard.
Queries plain log_entries (not FTS) so it works even before the index is built.
"""
rules = _compile_overrides(severity_overrides or [])
conn = sqlite3.connect(str(db_path))
conn.execute("PRAGMA journal_mode=WAL")
conn.row_factory = sqlite3.Row
@ -365,25 +387,36 @@ def stats_summary(db_path: Path, window_hours: int = 24) -> dict:
for r in source_rows
]
# 5 most recent critical entries
# Fetch candidate criticals (fetch more so filtering doesn't leave us with too few)
crit_rows = conn.execute("""
SELECT id as entry_id, source_id, sequence, timestamp_iso, severity,
repeat_count, out_of_order, matched_patterns, text, 0.0 as rank
SELECT id as entry_id, source_id, timestamp_iso, severity, text
FROM log_entries
WHERE severity = 'CRITICAL' AND repeat_count = 1
ORDER BY timestamp_iso DESC
LIMIT 5
LIMIT 25
""").fetchall()
recent_criticals = [
{
# Apply overrides: skip entries whose effective severity is no longer CRITICAL
suppressed = 0
recent_criticals = []
for r in crit_rows:
effective = _apply_overrides(r["text"], r["severity"], rules)
if effective == "CRITICAL":
recent_criticals.append({
"entry_id": r["entry_id"],
"source_id": r["source_id"],
"timestamp_iso": r["timestamp_iso"],
"severity": r["severity"],
"text": r["text"],
}
for r in crit_rows
]
})
if len(recent_criticals) == 5:
break
else:
suppressed += 1
# When did we last ingest anything?
last_row = conn.execute("SELECT MAX(ingest_time) AS t FROM log_entries").fetchone()
last_ingested: str | None = last_row["t"] if last_row else None
conn.close()
@ -394,6 +427,8 @@ def stats_summary(db_path: Path, window_hours: int = 24) -> dict:
"errors_24h": errors_24h,
"source_health": source_health,
"recent_criticals": recent_criticals,
"suppressed_criticals": suppressed,
"last_ingested": last_ingested,
}

View file

@ -5,3 +5,4 @@ pyyaml>=6.0
aiofiles>=23.0.0
python-multipart>=0.0.9
dateparser>=1.2.0
httpx>=0.27.0

View file

@ -0,0 +1,71 @@
"""Tests for app/services/llm.py — graceful failure and context building."""
from __future__ import annotations
from unittest.mock import MagicMock, patch
from app.services.llm import summarize, _build_context
from app.services.search import SearchResult
def _entry(text: str, severity: str = "INFO", ts: str = "2026-05-06T21:00:00+00:00") -> SearchResult:
return SearchResult(
entry_id="x",
source_id="svc",
sequence=0,
timestamp_iso=ts,
severity=severity,
text=text,
matched_patterns=[],
repeat_count=1,
out_of_order=False,
rank=0.0,
)
def test_summarize_returns_none_on_connection_error():
with patch("app.services.llm.httpx.post", side_effect=ConnectionError("refused")):
result = summarize("ollama crashed", [_entry("failed")], "http://bad", "llama3")
assert result is None
def test_summarize_returns_none_on_http_error():
mock_resp = MagicMock()
mock_resp.raise_for_status.side_effect = Exception("404")
with patch("app.services.llm.httpx.post", return_value=mock_resp):
result = summarize("ollama crashed", [_entry("failed")], "http://host", "llama3")
assert result is None
def test_summarize_returns_none_on_empty_response():
mock_resp = MagicMock()
mock_resp.raise_for_status.return_value = None
mock_resp.json.return_value = {"response": ""}
with patch("app.services.llm.httpx.post", return_value=mock_resp):
result = summarize("query", [_entry("x")], "http://host", "llama3")
assert result is None
def test_summarize_returns_text_on_success():
mock_resp = MagicMock()
mock_resp.raise_for_status.return_value = None
mock_resp.json.return_value = {"response": "Ollama exited with code 1."}
with patch("app.services.llm.httpx.post", return_value=mock_resp):
result = summarize("ollama crashed", [_entry("Failed")], "http://host", "llama3")
assert result == "Ollama exited with code 1."
def test_build_context_sorts_errors_first():
entries = [
_entry("info message", severity="INFO"),
_entry("critical crash", severity="CRITICAL"),
_entry("warn spike", severity="WARN"),
]
ctx = _build_context(entries)
lines = ctx.splitlines()
assert "CRITICAL" in lines[0]
assert "WARN" in lines[1]
def test_summarize_empty_entries_returns_none():
result = summarize("query", [], "http://host", "model")
assert result is None

View file

@ -45,6 +45,18 @@
</div>
</div>
<!-- LLM reasoning card -->
<div
v-if="reasoning"
class="mb-4 rounded border border-accent/30 bg-accent/5 p-4"
>
<div class="flex items-center gap-2 mb-2 text-xs text-text-dim font-medium uppercase tracking-wide">
<span></span>
<span>Diagnosis</span>
</div>
<p class="text-sm text-text-primary leading-relaxed whitespace-pre-wrap">{{ reasoning }}</p>
</div>
<!-- Log stream -->
<div v-if="entries.length" class="rounded border border-surface-border overflow-hidden mb-4">
<LogEntryRow v-for="entry in entries" :key="entry.entry_id" :entry="entry" />
@ -140,6 +152,7 @@ interface Summary {
const query = ref('')
const entries = ref<LogEntry[]>([])
const summary = ref<Summary | null>(null)
const reasoning = ref<string | null>(null)
const loading = ref(false)
const error = ref<string | null>(null)
const ranOnce = ref(false)
@ -185,6 +198,7 @@ async function run() {
const data = await res.json()
entries.value = data.entries
summary.value = data.summary
reasoning.value = data.reasoning ?? null
capturedSince = data.summary.window_start
capturedUntil = data.summary.window_end
} catch (e) {

View file

@ -1,6 +1,15 @@
<template>
<div class="p-6 max-w-5xl mx-auto space-y-8">
<!-- Data freshness banner -->
<div
v-if="!loading && stats && isStale"
class="flex items-center gap-2 rounded border border-surface-border bg-surface-raised px-4 py-2.5 text-xs text-text-dim"
>
<span class="text-sev-warn"></span>
<span>Last ingested: <span class="text-text-muted">{{ shortTs(stats.last_ingested) }}</span> 24h counts reflect this window, not today.</span>
</div>
<!-- Stat cards -->
<div class="grid grid-cols-1 sm:grid-cols-3 gap-4">
<div class="rounded border border-surface-border bg-surface-raised p-5">
@ -8,6 +17,9 @@
<p class="text-3xl font-semibold tabular-nums" :class="stats?.criticals_24h ? 'text-sev-critical' : 'text-text-muted'">
{{ loading ? '…' : (stats?.criticals_24h ?? 0) }}
</p>
<p v-if="stats?.suppressed_criticals" class="text-xs text-text-dim mt-1">
{{ stats.suppressed_criticals }} suppressed by overrides
</p>
</div>
<div class="rounded border border-surface-border bg-surface-raised p-5">
<p class="text-text-dim text-xs uppercase tracking-widest mb-2">Errors (24h)</p>
@ -99,6 +111,10 @@
<p class="text-text-primary text-sm font-mono leading-relaxed line-clamp-2">{{ entry.text }}</p>
</div>
</div>
<p v-if="stats.suppressed_criticals" class="text-xs text-text-dim mt-2">
{{ stats.suppressed_criticals }} additional critical{{ stats.suppressed_criticals !== 1 ? 's' : '' }} hidden by
<RouterLink to="/settings" class="text-accent hover:underline">severity overrides</RouterLink>.
</p>
</div>
<!-- Zero state: everything clean -->
@ -115,7 +131,7 @@
<script setup lang="ts">
import { ref, computed, onMounted } from 'vue'
import { useRouter } from 'vue-router'
import { useRouter, RouterLink } from 'vue-router'
const router = useRouter()
const BASE = import.meta.env.BASE_URL.replace(/\/$/, '')
@ -132,6 +148,8 @@ interface StatsResponse {
total_24h: number
criticals_24h: number
errors_24h: number
suppressed_criticals: number
last_ingested: string | null
source_health: SourceHealth[]
recent_criticals: Array<{
entry_id: string
@ -156,6 +174,12 @@ const activeIncidents = computed(() =>
incidents.value.filter(i => !i.ended_at).length
)
const isStale = computed(() => {
if (!stats.value?.last_ingested) return false
const age = Date.now() - new Date(stats.value.last_ingested).getTime()
return age > 25 * 60 * 60 * 1000 // older than 25h
})
onMounted(async () => {
await Promise.all([loadStats(), loadIncidents()])
})

View file

@ -8,6 +8,7 @@
</div>
<div class="rounded border border-surface-border bg-surface-raised p-5 space-y-6">
<!-- Entry point -->
<div>
<h2 class="text-text-primary text-sm font-semibold mb-1">Quick Capture Entry Point</h2>
<p class="text-text-dim text-xs mb-3">
@ -29,16 +30,129 @@
<div class="text-xs text-text-dim mt-0.5">{{ opt.desc }}</div>
</button>
</div>
</div>
<!-- LLM config -->
<div>
<h2 class="text-text-primary text-sm font-semibold mb-1">LLM Reasoning</h2>
<p class="text-text-dim text-xs mb-3">
Ollama endpoint used to generate plain-language diagnoses. Leave blank to disable.
</p>
<div class="space-y-3">
<div>
<label class="block text-xs text-text-dim mb-1">Ollama URL</label>
<input
v-model="prefs.llm_url"
type="text"
placeholder="http://localhost:11434"
class="w-full bg-surface border border-surface-border rounded px-3 py-2 text-sm text-text-primary placeholder-text-dim focus:outline-none focus:border-accent transition-colors"
/>
</div>
<div>
<label class="block text-xs text-text-dim mb-1">Model</label>
<input
v-model="prefs.llm_model"
type="text"
placeholder="llama3.1:8b"
class="w-full bg-surface border border-surface-border rounded px-3 py-2 text-sm text-text-primary placeholder-text-dim focus:outline-none focus:border-accent transition-colors"
/>
</div>
<button
@click="saveLlm"
class="px-4 py-2 bg-accent text-surface text-sm rounded font-medium hover:opacity-90 transition-opacity"
>
Save LLM settings
</button>
</div>
</div>
<!-- Severity overrides -->
<div>
<h2 class="text-text-primary text-sm font-semibold mb-1">Severity Overrides</h2>
<p class="text-text-dim text-xs mb-3">
Regex rules applied at query time entries that match are shown at the overridden severity on the dashboard. DB values are unchanged.
</p>
<div class="space-y-2 mb-3">
<div
v-for="(rule, i) in prefs.severity_overrides"
:key="i"
class="flex items-start gap-3 rounded border border-surface-border bg-surface p-3"
>
<button
@click="toggleOverride(i)"
:class="[
'mt-0.5 w-9 h-5 rounded-full flex-shrink-0 transition-colors relative',
rule.enabled ? 'bg-accent' : 'bg-surface-border'
]"
:title="rule.enabled ? 'Enabled — click to disable' : 'Disabled — click to enable'"
>
<span :class="['absolute top-0.5 w-4 h-4 rounded-full bg-white shadow transition-transform', rule.enabled ? 'translate-x-4' : 'translate-x-0.5']"></span>
</button>
<div class="flex-1 min-w-0">
<div class="flex items-center gap-2 flex-wrap">
<span class="text-sm text-text-primary font-medium">{{ rule.name }}</span>
<span class="text-xs px-1.5 py-0.5 rounded bg-surface-border text-text-dim"> {{ rule.override_severity }}</span>
</div>
<p class="text-xs text-text-dim font-mono mt-0.5 truncate">{{ rule.pattern }}</p>
</div>
<button
@click="deleteOverride(i)"
class="text-text-dim hover:text-sev-error text-xs flex-shrink-0 mt-0.5"
title="Delete rule"
></button>
</div>
<p v-if="!prefs.severity_overrides?.length" class="text-text-dim text-xs">No rules configured.</p>
</div>
<!-- Add rule form -->
<div v-if="!showAddOverride">
<button
@click="showAddOverride = true"
class="text-accent text-xs hover:underline"
>+ Add override rule</button>
</div>
<div v-else class="rounded border border-surface-border bg-surface p-3 space-y-2">
<div class="grid grid-cols-1 sm:grid-cols-2 gap-2">
<div>
<label class="block text-xs text-text-dim mb-1">Name</label>
<input v-model="newRule.name" type="text" placeholder="e.g. PAM auth noise"
class="w-full bg-surface-raised border border-surface-border rounded px-2 py-1.5 text-sm text-text-primary placeholder-text-dim focus:outline-none focus:border-accent" />
</div>
<div>
<label class="block text-xs text-text-dim mb-1">Override severity</label>
<select v-model="newRule.override_severity"
class="w-full bg-surface-raised border border-surface-border rounded px-2 py-1.5 text-sm text-text-primary focus:outline-none focus:border-accent">
<option>WARN</option>
<option>INFO</option>
<option>DEBUG</option>
</select>
</div>
</div>
<div>
<label class="block text-xs text-text-dim mb-1">Pattern (regex)</label>
<input v-model="newRule.pattern" type="text" placeholder="e.g. pam_unix.*auth"
class="w-full bg-surface-raised border border-surface-border rounded px-2 py-1.5 text-sm font-mono text-text-primary placeholder-text-dim focus:outline-none focus:border-accent" />
</div>
<div class="flex gap-2">
<button @click="addOverride"
class="px-3 py-1.5 bg-accent text-surface text-xs rounded font-medium hover:opacity-90 transition-opacity">
Add
</button>
<button @click="showAddOverride = false" class="text-text-dim hover:text-text-primary text-xs">Cancel</button>
</div>
</div>
</div>
<p
v-if="saveStatus"
class="text-xs mt-2"
class="text-xs"
:class="saveStatus.ok ? 'text-green-400' : 'text-sev-error'"
>
{{ saveStatus.msg }}
</p>
</div>
</div>
</div>
</template>
<script setup lang="ts">
@ -46,10 +160,24 @@ import { ref, onMounted } from 'vue'
const BASE = import.meta.env.BASE_URL.replace(/\/$/, '')
interface Prefs { entry_point_style: 'topbar' | 'fab' }
interface SeverityOverride {
name: string
pattern: string
override_severity: string
enabled: boolean
}
const prefs = ref<Prefs>({ entry_point_style: 'topbar' })
interface Prefs {
entry_point_style: 'topbar' | 'fab'
llm_url: string
llm_model: string
severity_overrides: SeverityOverride[]
}
const prefs = ref<Prefs>({ entry_point_style: 'topbar', llm_url: '', llm_model: '', severity_overrides: [] })
const saveStatus = ref<{ ok: boolean; msg: string } | null>(null)
const showAddOverride = ref(false)
const newRule = ref<SeverityOverride>({ name: '', pattern: '', override_severity: 'WARN', enabled: true })
const entryPointOptions = [
{ value: 'topbar', label: 'Top bar', desc: 'Persistent input bar below the nav on every page' },
@ -60,23 +188,67 @@ onMounted(async () => {
try {
const res = await fetch(`${BASE}/api/settings`)
if (res.ok) prefs.value = await res.json()
} catch { /* non-critical — default stays topbar */ }
} catch { /* non-critical — defaults stay */ }
})
async function setEntryPoint(style: 'topbar' | 'fab') {
prefs.value = { entry_point_style: style }
saveStatus.value = null
try {
async function patch(body: Partial<Prefs>) {
const res = await fetch(`${BASE}/api/settings`, {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ entry_point_style: style }),
body: JSON.stringify(body),
})
if (!res.ok) throw new Error(await res.text())
prefs.value = await res.json()
}
async function setEntryPoint(style: 'topbar' | 'fab') {
saveStatus.value = null
try {
await patch({ entry_point_style: style })
saveStatus.value = { ok: true, msg: 'Saved' }
setTimeout(() => { saveStatus.value = null }, 2000)
} catch {
saveStatus.value = { ok: false, msg: 'Save failed — check server connection' }
}
}
async function saveLlm() {
saveStatus.value = null
try {
await patch({ llm_url: prefs.value.llm_url, llm_model: prefs.value.llm_model })
saveStatus.value = { ok: true, msg: 'LLM settings saved' }
setTimeout(() => { saveStatus.value = null }, 2000)
} catch {
saveStatus.value = { ok: false, msg: 'Save failed — check server connection' }
}
}
async function saveOverrides() {
try {
await patch({ severity_overrides: prefs.value.severity_overrides })
saveStatus.value = { ok: true, msg: 'Overrides saved' }
setTimeout(() => { saveStatus.value = null }, 2000)
} catch {
saveStatus.value = { ok: false, msg: 'Save failed — check server connection' }
}
}
async function toggleOverride(i: number) {
const rule = prefs.value.severity_overrides[i]
if (rule) rule.enabled = !rule.enabled
await saveOverrides()
}
async function deleteOverride(i: number) {
prefs.value.severity_overrides.splice(i, 1)
await saveOverrides()
}
async function addOverride() {
if (!newRule.value.name.trim() || !newRule.value.pattern.trim()) return
prefs.value.severity_overrides.push({ ...newRule.value })
newRule.value = { name: '', pattern: '', override_severity: 'WARN', enabled: true }
showAddOverride.value = false
await saveOverrides()
}
</script>