From 0882083755d176891ad222c13dbc83039fdd11c6 Mon Sep 17 00:00:00 2001 From: pyr0ball Date: Mon, 11 May 2026 11:35:07 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20LLM=20reasoning=20layer=20=E2=80=94?= =?UTF-8?q?=20Ollama=20summarization=20on=20diagnose=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/rest.py | 38 +++++++++--- app/services/diagnose.py | 8 +++ app/services/llm.py | 56 ++++++++++++++++++ requirements.txt | 1 + tests/test_services_llm.py | 71 +++++++++++++++++++++++ web/src/components/QuickCapture.vue | 14 +++++ web/src/views/SettingsView.vue | 89 +++++++++++++++++++++++------ 7 files changed, 253 insertions(+), 24 deletions(-) create mode 100644 app/services/llm.py create mode 100644 tests/test_services_llm.py diff --git a/app/rest.py b/app/rest.py index 98d8461..156ea36 100644 --- a/app/rest.py +++ b/app/rest.py @@ -62,13 +62,21 @@ def _startup() -> None: ensure_schema(DB_PATH) +_PREFS_DEFAULTS: dict[str, str] = { + "entry_point_style": "topbar", + "llm_url": "http://localhost:11434", + "llm_model": "llama3.1:8b", +} + + def _load_prefs() -> dict[str, str]: if PREFS_PATH.exists(): try: - return json.loads(PREFS_PATH.read_text()) + saved = json.loads(PREFS_PATH.read_text()) + return {**_PREFS_DEFAULTS, **saved} except (json.JSONDecodeError, OSError): pass - return {"entry_point_style": "topbar"} + return dict(_PREFS_DEFAULTS) def _save_prefs(data: dict[str, str]) -> None: @@ -82,7 +90,9 @@ class DiagnoseRequest(BaseModel): class SettingsBody(BaseModel): - entry_point_style: str + entry_point_style: str | None = None + llm_url: str | None = None + llm_model: str | None = None class IncidentCreate(BaseModel): @@ -202,9 +212,18 @@ def diagnose_post(body: DiagnoseRequest) -> dict: }, "entries": [], } - result = _diagnose(DB_PATH, query=body.query, since=body.since, until=body.until) + prefs = _load_prefs() + result = _diagnose( + DB_PATH, + query=body.query, + since=body.since, + until=body.until, + llm_url=prefs.get("llm_url") or None, + llm_model=prefs.get("llm_model") or None, + ) return { "summary": result["summary"], + "reasoning": result.get("reasoning"), "entries": [dataclasses.asdict(r) for r in result["entries"]], } @@ -216,10 +235,15 @@ def get_settings() -> dict: @router.patch("/api/settings") def patch_settings(body: SettingsBody) -> dict: - if body.entry_point_style not in ("topbar", "fab"): - raise HTTPException(status_code=422, detail="entry_point_style must be 'topbar' or 'fab'") prefs = _load_prefs() - prefs["entry_point_style"] = body.entry_point_style + if body.entry_point_style is not None: + if body.entry_point_style not in ("topbar", "fab"): + raise HTTPException(status_code=422, detail="entry_point_style must be 'topbar' or 'fab'") + prefs["entry_point_style"] = body.entry_point_style + if body.llm_url is not None: + prefs["llm_url"] = body.llm_url + if body.llm_model is not None: + prefs["llm_model"] = body.llm_model _save_prefs(prefs) return prefs diff --git a/app/services/diagnose.py b/app/services/diagnose.py index 8665b15..516b7d8 100644 --- a/app/services/diagnose.py +++ b/app/services/diagnose.py @@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any +from app.services.llm import summarize from app.services.search import SearchResult, entries_in_window, search logger = logging.getLogger(__name__) @@ -48,6 +49,8 @@ def diagnose( query: str, since: str | None = None, until: str | None = None, + llm_url: str | None = None, + llm_model: str | None = None, ) -> dict[str, Any]: """Run layered log search with NL time extraction. Returns summary + entries.""" time_detected = since is not None and until is not None @@ -79,6 +82,10 @@ def diagnose( by_severity[sev] += 1 by_source[r.source_id] = by_source.get(r.source_id, 0) + 1 + reasoning: str | None = None + if llm_url and llm_model: + reasoning = summarize(query, combined, llm_url=llm_url, llm_model=llm_model) + return { "summary": { "total": len(combined), @@ -88,6 +95,7 @@ def diagnose( "by_severity": by_severity, "by_source": by_source, }, + "reasoning": reasoning, "entries": combined, } diff --git a/app/services/llm.py b/app/services/llm.py new file mode 100644 index 0000000..6bfa542 --- /dev/null +++ b/app/services/llm.py @@ -0,0 +1,56 @@ +import logging + +import httpx + +from app.services.search import SearchResult + +logger = logging.getLogger(__name__) + +_SEVERITY_RANK = {"CRITICAL": 0, "ERROR": 1, "WARN": 2, "WARNING": 2} + +_PROMPT_TEMPLATE = """\ +You are a homelab diagnostic assistant. A user described a symptom and the system retrieved relevant log entries. + +Analyze the log entries below and write a 2-4 sentence plain-language diagnosis. Focus on errors and their likely root cause. Be specific and concise — name the services involved, not generic platitudes. + +User query: {query} + +Log entries ({n} shown, highest severity first): +{log_block} + +Diagnosis:""" + + +def _build_context(entries: list[SearchResult], max_entries: int = 25) -> str: + ranked = sorted( + entries, + key=lambda e: (_SEVERITY_RANK.get(e.severity or "", 3), e.timestamp_iso or ""), + )[:max_entries] + return "\n".join( + f"[{e.timestamp_iso or '?'}] [{e.severity or 'INFO'}] {e.text[:200]}" + for e in ranked + ) + + +def summarize( + query: str, + entries: list[SearchResult], + llm_url: str, + llm_model: str, + timeout: float = 20.0, +) -> str | None: + if not entries: + return None + log_block = _build_context(entries) + prompt = _PROMPT_TEMPLATE.format(query=query, n=min(len(entries), 25), log_block=log_block) + try: + resp = httpx.post( + f"{llm_url.rstrip('/')}/api/generate", + json={"model": llm_model, "prompt": prompt, "stream": False}, + timeout=timeout, + ) + resp.raise_for_status() + return resp.json().get("response", "").strip() or None + except Exception as exc: + logger.warning("LLM summarization failed (%s): %s", type(exc).__name__, exc) + return None diff --git a/requirements.txt b/requirements.txt index 14a443b..66b35f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ pyyaml>=6.0 aiofiles>=23.0.0 python-multipart>=0.0.9 dateparser>=1.2.0 +httpx>=0.27.0 diff --git a/tests/test_services_llm.py b/tests/test_services_llm.py new file mode 100644 index 0000000..447091f --- /dev/null +++ b/tests/test_services_llm.py @@ -0,0 +1,71 @@ +"""Tests for app/services/llm.py — graceful failure and context building.""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from app.services.llm import summarize, _build_context +from app.services.search import SearchResult + + +def _entry(text: str, severity: str = "INFO", ts: str = "2026-05-06T21:00:00+00:00") -> SearchResult: + return SearchResult( + entry_id="x", + source_id="svc", + sequence=0, + timestamp_iso=ts, + severity=severity, + text=text, + matched_patterns=[], + repeat_count=1, + out_of_order=False, + rank=0.0, + ) + + +def test_summarize_returns_none_on_connection_error(): + with patch("app.services.llm.httpx.post", side_effect=ConnectionError("refused")): + result = summarize("ollama crashed", [_entry("failed")], "http://bad", "llama3") + assert result is None + + +def test_summarize_returns_none_on_http_error(): + mock_resp = MagicMock() + mock_resp.raise_for_status.side_effect = Exception("404") + with patch("app.services.llm.httpx.post", return_value=mock_resp): + result = summarize("ollama crashed", [_entry("failed")], "http://host", "llama3") + assert result is None + + +def test_summarize_returns_none_on_empty_response(): + mock_resp = MagicMock() + mock_resp.raise_for_status.return_value = None + mock_resp.json.return_value = {"response": ""} + with patch("app.services.llm.httpx.post", return_value=mock_resp): + result = summarize("query", [_entry("x")], "http://host", "llama3") + assert result is None + + +def test_summarize_returns_text_on_success(): + mock_resp = MagicMock() + mock_resp.raise_for_status.return_value = None + mock_resp.json.return_value = {"response": "Ollama exited with code 1."} + with patch("app.services.llm.httpx.post", return_value=mock_resp): + result = summarize("ollama crashed", [_entry("Failed")], "http://host", "llama3") + assert result == "Ollama exited with code 1." + + +def test_build_context_sorts_errors_first(): + entries = [ + _entry("info message", severity="INFO"), + _entry("critical crash", severity="CRITICAL"), + _entry("warn spike", severity="WARN"), + ] + ctx = _build_context(entries) + lines = ctx.splitlines() + assert "CRITICAL" in lines[0] + assert "WARN" in lines[1] + + +def test_summarize_empty_entries_returns_none(): + result = summarize("query", [], "http://host", "model") + assert result is None diff --git a/web/src/components/QuickCapture.vue b/web/src/components/QuickCapture.vue index 8f41c3e..5061d17 100644 --- a/web/src/components/QuickCapture.vue +++ b/web/src/components/QuickCapture.vue @@ -45,6 +45,18 @@ + +
+
+ ⚡ + Diagnosis +
+

{{ reasoning }}

+
+
@@ -140,6 +152,7 @@ interface Summary { const query = ref('') const entries = ref([]) const summary = ref(null) +const reasoning = ref(null) const loading = ref(false) const error = ref(null) const ranOnce = ref(false) @@ -185,6 +198,7 @@ async function run() { const data = await res.json() entries.value = data.entries summary.value = data.summary + reasoning.value = data.reasoning ?? null capturedSince = data.summary.window_start capturedUntil = data.summary.window_end } catch (e) { diff --git a/web/src/views/SettingsView.vue b/web/src/views/SettingsView.vue index 3c52fdb..ef8f069 100644 --- a/web/src/views/SettingsView.vue +++ b/web/src/views/SettingsView.vue @@ -8,6 +8,7 @@
+

Quick Capture Entry Point

@@ -29,14 +30,49 @@

{{ opt.desc }}
-

- {{ saveStatus.msg }} -

+ + +
+

LLM Reasoning

+

+ Ollama endpoint used to generate plain-language diagnoses. Leave blank to disable. +

+
+
+ + +
+
+ + +
+ +
+
+ +

+ {{ saveStatus.msg }} +

@@ -46,9 +82,13 @@ import { ref, onMounted } from 'vue' const BASE = import.meta.env.BASE_URL.replace(/\/$/, '') -interface Prefs { entry_point_style: 'topbar' | 'fab' } +interface Prefs { + entry_point_style: 'topbar' | 'fab' + llm_url: string + llm_model: string +} -const prefs = ref({ entry_point_style: 'topbar' }) +const prefs = ref({ entry_point_style: 'topbar', llm_url: '', llm_model: '' }) const saveStatus = ref<{ ok: boolean; msg: string } | null>(null) const entryPointOptions = [ @@ -60,23 +100,38 @@ onMounted(async () => { try { const res = await fetch(`${BASE}/api/settings`) if (res.ok) prefs.value = await res.json() - } catch { /* non-critical — default stays topbar */ } + } catch { /* non-critical — defaults stay */ } }) +async function patch(body: Partial) { + const res = await fetch(`${BASE}/api/settings`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!res.ok) throw new Error(await res.text()) + prefs.value = await res.json() +} + async function setEntryPoint(style: 'topbar' | 'fab') { - prefs.value = { entry_point_style: style } saveStatus.value = null try { - const res = await fetch(`${BASE}/api/settings`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ entry_point_style: style }), - }) - if (!res.ok) throw new Error(await res.text()) + await patch({ entry_point_style: style }) saveStatus.value = { ok: true, msg: 'Saved' } setTimeout(() => { saveStatus.value = null }, 2000) } catch { saveStatus.value = { ok: false, msg: 'Save failed — check server connection' } } } + +async function saveLlm() { + saveStatus.value = null + try { + await patch({ llm_url: prefs.value.llm_url, llm_model: prefs.value.llm_model }) + saveStatus.value = { ok: true, msg: 'LLM settings saved' } + setTimeout(() => { saveStatus.value = null }, 2000) + } catch { + saveStatus.value = { ok: false, msg: 'Save failed — check server connection' } + } +} From c12cc6d68ae67e5ec749c5b748dff43b7a16e81c Mon Sep 17 00:00:00 2001 From: pyr0ball Date: Mon, 11 May 2026 13:00:11 -0700 Subject: [PATCH 2/2] feat: severity overrides + last_ingested timestamp on dashboard --- app/rest.py | 23 +++++- app/services/search.py | 65 +++++++++++++---- web/src/views/DashboardView.vue | 26 ++++++- web/src/views/SettingsView.vue | 119 +++++++++++++++++++++++++++++++- 4 files changed, 214 insertions(+), 19 deletions(-) diff --git a/app/rest.py b/app/rest.py index 156ea36..09c3903 100644 --- a/app/rest.py +++ b/app/rest.py @@ -62,10 +62,18 @@ def _startup() -> None: ensure_schema(DB_PATH) -_PREFS_DEFAULTS: dict[str, str] = { +_PREFS_DEFAULTS: dict = { "entry_point_style": "topbar", "llm_url": "http://localhost:11434", "llm_model": "llama3.1:8b", + "severity_overrides": [ + { + "name": "PAM auth noise", + "pattern": r"pam_unix.*auth(?:entication)?\s+fail|auth could not identify", + "override_severity": "WARN", + "enabled": True, + } + ], } @@ -89,10 +97,18 @@ class DiagnoseRequest(BaseModel): until: str | None = None +class SeverityOverride(BaseModel): + name: str + pattern: str + override_severity: str + enabled: bool = True + + class SettingsBody(BaseModel): entry_point_style: str | None = None llm_url: str | None = None llm_model: str | None = None + severity_overrides: list[SeverityOverride] | None = None class IncidentCreate(BaseModel): @@ -244,6 +260,8 @@ def patch_settings(body: SettingsBody) -> dict: prefs["llm_url"] = body.llm_url if body.llm_model is not None: prefs["llm_model"] = body.llm_model + if body.severity_overrides is not None: + prefs["severity_overrides"] = [o.model_dump() for o in body.severity_overrides] _save_prefs(prefs) return prefs @@ -257,7 +275,8 @@ def list_sources() -> dict: def get_stats( window: Annotated[int, Query(ge=1, le=168, description="Hours to look back")] = 24, ) -> dict: - return _stats(DB_PATH, window_hours=window) + prefs = _load_prefs() + return _stats(DB_PATH, window_hours=window, severity_overrides=prefs.get("severity_overrides", [])) @router.post("/api/incidents") diff --git a/app/services/search.py b/app/services/search.py index 9334605..1665eb2 100644 --- a/app/services/search.py +++ b/app/services/search.py @@ -317,11 +317,33 @@ def list_sources(db_path: Path) -> list[dict]: ] -def stats_summary(db_path: Path, window_hours: int = 24) -> dict: +def _compile_overrides(overrides: list[dict]) -> list[tuple[re.Pattern[str], str]]: + """Return (compiled_pattern, override_severity) pairs for enabled rules.""" + compiled = [] + for rule in overrides: + if not rule.get("enabled", True): + continue + try: + compiled.append((re.compile(rule["pattern"], re.IGNORECASE), rule["override_severity"])) + except re.error: + pass + return compiled + + +def _apply_overrides(text: str, original_severity: str, rules: list[tuple[re.Pattern[str], str]]) -> str: + for pattern, new_sev in rules: + if pattern.search(text): + return new_sev + return original_severity + + +def stats_summary(db_path: Path, window_hours: int = 24, severity_overrides: list[dict] | None = None) -> dict: """Return aggregate health stats for the dashboard. Queries plain log_entries (not FTS) so it works even before the index is built. """ + rules = _compile_overrides(severity_overrides or []) + conn = sqlite3.connect(str(db_path)) conn.execute("PRAGMA journal_mode=WAL") conn.row_factory = sqlite3.Row @@ -365,25 +387,36 @@ def stats_summary(db_path: Path, window_hours: int = 24) -> dict: for r in source_rows ] - # 5 most recent critical entries + # Fetch candidate criticals (fetch more so filtering doesn't leave us with too few) crit_rows = conn.execute(""" - SELECT id as entry_id, source_id, sequence, timestamp_iso, severity, - repeat_count, out_of_order, matched_patterns, text, 0.0 as rank + SELECT id as entry_id, source_id, timestamp_iso, severity, text FROM log_entries WHERE severity = 'CRITICAL' AND repeat_count = 1 ORDER BY timestamp_iso DESC - LIMIT 5 + LIMIT 25 """).fetchall() - recent_criticals = [ - { - "entry_id": r["entry_id"], - "source_id": r["source_id"], - "timestamp_iso": r["timestamp_iso"], - "severity": r["severity"], - "text": r["text"], - } - for r in crit_rows - ] + + # Apply overrides: skip entries whose effective severity is no longer CRITICAL + suppressed = 0 + recent_criticals = [] + for r in crit_rows: + effective = _apply_overrides(r["text"], r["severity"], rules) + if effective == "CRITICAL": + recent_criticals.append({ + "entry_id": r["entry_id"], + "source_id": r["source_id"], + "timestamp_iso": r["timestamp_iso"], + "severity": r["severity"], + "text": r["text"], + }) + if len(recent_criticals) == 5: + break + else: + suppressed += 1 + + # When did we last ingest anything? + last_row = conn.execute("SELECT MAX(ingest_time) AS t FROM log_entries").fetchone() + last_ingested: str | None = last_row["t"] if last_row else None conn.close() @@ -394,6 +427,8 @@ def stats_summary(db_path: Path, window_hours: int = 24) -> dict: "errors_24h": errors_24h, "source_health": source_health, "recent_criticals": recent_criticals, + "suppressed_criticals": suppressed, + "last_ingested": last_ingested, } diff --git a/web/src/views/DashboardView.vue b/web/src/views/DashboardView.vue index 9b88ff4..21805bc 100644 --- a/web/src/views/DashboardView.vue +++ b/web/src/views/DashboardView.vue @@ -1,6 +1,15 @@