From f3d807d991ed48789aaec5c859413670ebf6ff79 Mon Sep 17 00:00:00 2001
From: pyr0ball <pyroballpcs@gmail.com>
Date: Thu, 11 Jun 2026 22:04:53 -0700
Subject: [PATCH] feat(diagnose): conversational chat mode + NL source
 discovery
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- New ChatDiagnose.vue: multi-turn chat UI in the Diagnose tab
  - Textarea input (auto-grows) for long free-form problem descriptions
  - Source suggestion pre-flight: debounced POST /api/sources/suggest
    identifies relevant log sources from the query text and shows them
    as interactive chips (deselect to exclude before searching)
  - Conversation history preserved across turns with LLM reasoning,
    collapsible log entries, and "Save as incident" per turn
  - Reuses existing /api/diagnose/stream — no new pipeline
- DiagnoseView.vue: Chat is now default tab; viewport-height layout
- POST /api/sources/suggest: token-overlap source ranking, no LLM
- Fix: add missing 'import re' causing 500 on suggest route
---
 app/rest.py                         |  54 ++++
 web/src/components/ChatDiagnose.vue | 370 ++++++++++++++++++++++++++++
 web/src/views/DiagnoseView.vue      |  60 +++--
 3 files changed, 465 insertions(+), 19 deletions(-)
 create mode 100644 web/src/components/ChatDiagnose.vue

diff --git a/app/rest.py b/app/rest.py
index a62070f..246b5cc 100644
--- a/app/rest.py
+++ b/app/rest.py
@@ -12,6 +12,7 @@ import hmac
 import json
 import logging
 import os
+import re
 import time
 
 # Offline mode: must be set before any HuggingFace library is imported.
@@ -277,6 +278,10 @@ class DiagnoseRequest(BaseModel):
     source: str | None = None
 
 
+class SourceSuggestRequest(BaseModel):
+    query: str
+
+
 class SeverityOverride(BaseModel):
     name: str
     pattern: str
@@ -523,6 +528,55 @@ async def diagnose_post_stream(body: DiagnoseRequest) -> StreamingResponse:
     )
 
 
+_SUGGEST_STOPWORDS = frozenset({
+    "the", "and", "that", "this", "with", "have", "from", "they",
+    "been", "their", "what", "when", "there", "some", "would", "make",
+    "like", "into", "time", "look", "just", "know", "take", "year",
+    "your", "good", "some", "could", "them", "then", "very", "also",
+    "back", "after", "work", "need", "even", "much", "most", "tell",
+    "does", "more", "once", "help", "seem", "here", "about", "issue",
+    "thing", "logs", "error", "again", "still", "these", "those",
+    "getting", "having", "trying", "going", "where", "which", "cant",
+    "now", "set", "kind", "weird", "stable", "huge", "real", "nice",
+})
+
+
+@router.post("/api/sources/suggest")
+def suggest_sources(body: SourceSuggestRequest) -> dict:
+    """Return source IDs ranked by relevance to a natural-language problem description."""
+    all_sources = _list_sources(DB_PATH)
+    query_tokens = {
+        t.lower()
+        for t in re.findall(r"[a-zA-Z]+", body.query)
+        if len(t) > 2 and t.lower() not in _SUGGEST_STOPWORDS
+    }
+
+    suggestions = []
+    for src in all_sources:
+        src_id: str = src["source_id"]
+        # Tokenise source ID: split on colon, dash, underscore, digits
+        parts = {
+            p.lower()
+            for seg in re.split(r"[:\-_\d]+", src_id)
+            for p in [seg.strip()]
+            if len(p) > 2
+        }
+        matched = query_tokens & parts
+        if matched:
+            score = round(len(matched) / max(len(parts), 1), 3)
+            suggestions.append({
+                "source_id": src_id,
+                "score": score,
+                "matched_tokens": sorted(matched),
+            })
+
+    suggestions.sort(key=lambda x: x["score"], reverse=True)
+    return {
+        "suggested": suggestions,
+        "all_source_ids": [s["source_id"] for s in all_sources],
+    }
+
+
 @router.get("/api/settings")
 def get_settings() -> dict:
     return _load_prefs()
diff --git a/web/src/components/ChatDiagnose.vue b/web/src/components/ChatDiagnose.vue
new file mode 100644
index 0000000..eb87110
--- /dev/null
+++ b/web/src/components/ChatDiagnose.vue
@@ -0,0 +1,370 @@
+<template>
+  <div class="flex flex-col h-full min-h-0">
+
+    <!-- Conversation history -->
+    <div
+      ref="scrollEl"
+      class="flex-1 overflow-y-auto space-y-6 pb-4 pr-1"
+      aria-live="polite"
+      aria-label="Conversation history"
+    >
+      <!-- Empty state -->
+      <div v-if="!turns.length" class="flex flex-col items-center justify-center py-16 text-center px-4">
+        <div class="text-4xl mb-3" aria-hidden="true">🪵</div>
+        <p class="text-text-primary text-base font-medium mb-1">Describe your issue</p>
+        <p class="text-text-dim text-sm max-w-md">
+          Write what you're seeing — however you'd say it. Turnstone will search
+          your logs and explain what it finds. Mention a service name to focus
+          the search ("meshtasticd keeps disconnecting after 4.10 update").
+        </p>
+      </div>
+
+      <!-- Turn history -->
+      <template v-for="(turn, idx) in turns" :key="idx">
+        <!-- User bubble -->
+        <div class="flex justify-end">
+          <div class="max-w-[80%] rounded-2xl rounded-tr-sm bg-accent text-white px-4 py-2.5 text-sm whitespace-pre-wrap leading-relaxed">
+            {{ turn.query }}
+          </div>
+        </div>
+
+        <!-- Assistant response -->
+        <div class="flex flex-col gap-3">
+          <!-- Source chips -->
+          <div v-if="turn.sources?.length" class="flex flex-wrap gap-1.5 items-center">
+            <span class="text-xs text-text-dim">Searched:</span>
+            <span
+              v-for="s in turn.sources"
+              :key="s"
+              class="font-mono text-xs bg-surface-raised border border-surface-border rounded px-2 py-0.5 text-text-muted"
+            >{{ s }}</span>
+          </div>
+
+          <!-- Loading spinner -->
+          <div v-if="turn.loading" class="flex items-center gap-2 text-xs text-text-dim py-2">
+            <span class="inline-block w-3 h-3 rounded-full border-2 border-accent border-t-transparent animate-spin motion-reduce:animate-none" aria-hidden="true" />
+            <span>{{ turn.status ?? 'Searching logs…' }}</span>
+          </div>
+
+          <!-- LLM reasoning -->
+          <div
+            v-if="turn.reasoning"
+            class="rounded-r border-l-4 border-accent bg-surface-raised px-4 py-3"
+          >
+            <div class="flex items-center gap-2 mb-2 text-xs text-accent font-semibold uppercase tracking-wide">
+              <span aria-hidden="true">⚡</span>
+              <span>Diagnosis</span>
+            </div>
+            <p class="text-sm text-text-primary leading-relaxed whitespace-pre-wrap">{{ turn.reasoning }}</p>
+          </div>
+
+          <!-- Summary bar -->
+          <div v-if="turn.summary" class="flex flex-wrap gap-x-5 gap-y-1 text-xs text-text-dim px-1">
+            <span class="font-medium text-text-muted">{{ turn.summary.total }} entr{{ turn.summary.total !== 1 ? 'ies' : 'y' }}</span>
+            <span v-if="turn.summary.window_start">
+              {{ fmtTs(turn.summary.window_start) }} → {{ fmtTs(turn.summary.window_end) }}
+            </span>
+            <button
+              v-if="turn.entries?.length && !turn.showEntries"
+              @click="turn.showEntries = true"
+              class="text-accent hover:underline"
+            >show {{ turn.entries.length }} log lines</button>
+            <button
+              v-if="turn.showEntries"
+              @click="turn.showEntries = false"
+              class="text-text-dim hover:text-text-primary"
+            >hide entries</button>
+          </div>
+
+          <!-- Log entries (collapsible) -->
+          <div
+            v-if="turn.showEntries && turn.entries?.length"
+            class="rounded border border-surface-border overflow-hidden"
+          >
+            <LogEntryRow
+              v-for="entry in turn.entries"
+              :key="entry.entry_id"
+              :entry="entry"
+            />
+          </div>
+
+          <!-- No results -->
+          <p
+            v-if="!turn.loading && turn.summary?.total === 0"
+            class="text-sm text-text-dim px-1"
+          >
+            No log evidence found for that query. Check Sources to confirm data is
+            gleaned, or try different wording.
+          </p>
+
+          <!-- Save as incident -->
+          <div v-if="!turn.loading && (turn.entries?.length ?? 0) > 0 && !turn.saved" class="flex gap-3 mt-1">
+            <button
+              @click="saveIncident(turn)"
+              :disabled="turn.saving"
+              class="px-3 py-1.5 bg-surface-raised border border-surface-border rounded text-xs text-text-muted hover:text-text-primary hover:border-accent transition-colors disabled:opacity-40"
+            >
+              {{ turn.saving ? 'Saving…' : 'Save as incident' }}
+            </button>
+          </div>
+          <p v-if="turn.saved" class="text-xs text-green-400 px-1">
+            Saved —
+            <RouterLink to="/incidents" class="underline underline-offset-2 hover:text-green-300">view in Incidents</RouterLink>
+          </p>
+        </div>
+      </template>
+    </div>
+
+    <!-- Source suggestion pre-flight -->
+    <div
+      v-if="suggestedSources.length && !activeTurn"
+      class="mb-3 p-3 rounded border border-surface-border bg-surface-raised"
+    >
+      <p class="text-xs text-text-dim mb-2">Detected sources — deselect to exclude:</p>
+      <div class="flex flex-wrap gap-2">
+        <button
+          v-for="s in suggestedSources"
+          :key="s.source_id"
+          @click="toggleSource(s.source_id)"
+          :aria-pressed="!excludedSources.has(s.source_id)"
+          :class="[
+            'font-mono text-xs rounded px-2 py-1 border transition-colors',
+            excludedSources.has(s.source_id)
+              ? 'bg-surface border-surface-border text-text-dim line-through'
+              : 'bg-accent/10 border-accent/40 text-accent'
+          ]"
+        >{{ s.source_id }}</button>
+      </div>
+    </div>
+
+    <!-- Input row -->
+    <div class="border-t border-surface-border pt-3">
+      <div class="flex gap-2 items-end">
+        <div class="flex-1">
+          <label :for="inputId" class="sr-only">Describe your issue</label>
+          <textarea
+            :id="inputId"
+            ref="textareaEl"
+            v-model="draft"
+            :disabled="!!activeTurn"
+            :placeholder="turns.length
+              ? 'Follow up, or ask about something else…'
+              : 'Paste or type your issue — as much detail as you want…'"
+            rows="3"
+            class="w-full bg-surface-raised border border-surface-border rounded-xl px-4 py-2.5 text-sm text-text-primary placeholder-text-dim focus:outline-none focus:border-accent transition-colors resize-none leading-relaxed disabled:opacity-50"
+            @input="onInput"
+            @keydown.enter.exact.prevent="submit"
+            @keydown.enter.shift.exact.stop
+          />
+          <p class="text-right text-xs text-text-dim mt-1">Enter to search · Shift+Enter for new line</p>
+        </div>
+        <button
+          :disabled="!draft.trim() || !!activeTurn"
+          @click="submit"
+          class="shrink-0 px-4 py-2.5 rounded-xl bg-accent text-white text-sm font-semibold hover:bg-blue-400 transition-colors disabled:opacity-40 self-end mb-6"
+          aria-label="Search logs"
+        >
+          <span v-if="activeTurn">…</span>
+          <span v-else>Search</span>
+        </button>
+      </div>
+    </div>
+
+  </div>
+</template>
+
+<script setup lang="ts">
+import { ref, nextTick, onMounted } from 'vue'
+import { RouterLink } from 'vue-router'
+import LogEntryRow from '@/components/LogEntryRow.vue'
+import type { LogEntry } from '@/stores/search'
+
+const BASE    = import.meta.env.BASE_URL.replace(/\/$/, '')
+const inputId = `chat-input-${Math.random().toString(36).slice(2, 7)}`
+
+interface Summary {
+  total: number
+  window_start: string | null
+  window_end: string | null
+  time_detected: boolean
+  by_severity: Record<string, number>
+  by_source: Record<string, number>
+}
+
+interface SuggestedSource {
+  source_id: string
+  score: number
+  matched_tokens: string[]
+}
+
+interface Turn {
+  query:       string
+  loading:     boolean
+  status:      string | null
+  reasoning:   string | null
+  summary:     Summary | null
+  entries:     LogEntry[]
+  sources:     string[]
+  showEntries: boolean
+  saved:       boolean
+  saving:      boolean
+  since:       string | null
+  until:       string | null
+}
+
+const turns            = ref<Turn[]>([])
+const draft            = ref('')
+const suggestedSources = ref<SuggestedSource[]>([])
+const excludedSources  = ref(new Set<string>())
+const activeTurn       = ref<Turn | null>(null)
+const scrollEl         = ref<HTMLElement | null>(null)
+const textareaEl       = ref<HTMLTextAreaElement | null>(null)
+
+let suggestTimer: ReturnType<typeof setTimeout> | null = null
+
+onMounted(() => textareaEl.value?.focus())
+
+function onInput() {
+  // Auto-grow textarea
+  const el = textareaEl.value
+  if (el) {
+    el.style.height = 'auto'
+    el.style.height = `${Math.min(el.scrollHeight, 240)}px`
+  }
+  // Debounce source suggestion
+  if (suggestTimer) clearTimeout(suggestTimer)
+  if (draft.value.trim().length > 8) {
+    suggestTimer = setTimeout(fetchSuggestions, 400)
+  } else {
+    suggestedSources.value = []
+  }
+}
+
+async function fetchSuggestions() {
+  try {
+    const res = await fetch(`${BASE}/api/sources/suggest`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ query: draft.value }),
+    })
+    if (!res.ok) return
+    const data = await res.json()
+    suggestedSources.value = (data.suggested ?? []).slice(0, 6)
+    // Reset exclusions when suggestions change
+    excludedSources.value = new Set()
+  } catch { /* non-critical */ }
+}
+
+function toggleSource(id: string) {
+  const next = new Set(excludedSources.value)
+  if (next.has(id)) next.delete(id)
+  else next.add(id)
+  excludedSources.value = next
+}
+
+async function submit() {
+  const text = draft.value.trim()
+  if (!text || activeTurn.value) return
+
+  draft.value = ''
+  suggestedSources.value = []
+  if (textareaEl.value) textareaEl.value.style.height = 'auto'
+
+  // Determine source scope from non-excluded suggestions
+  const sources = suggestedSources.value
+    .filter(s => !excludedSources.value.has(s.source_id))
+    .map(s => s.source_id)
+  excludedSources.value = new Set()
+
+  const turn: Turn = {
+    query:       text,
+    loading:     true,
+    status:      'Searching…',
+    reasoning:   null,
+    summary:     null,
+    entries:     [],
+    sources,
+    showEntries: false,
+    saved:       false,
+    saving:      false,
+    since:       null,
+    until:       null,
+  }
+  turns.value.push(turn)
+  activeTurn.value = turn
+  await nextTick()
+  scrollEl.value?.scrollTo({ top: scrollEl.value.scrollHeight, behavior: 'smooth' })
+
+  try {
+    const res = await fetch(`${BASE}/api/diagnose/stream`, {
+      method:  'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body:    JSON.stringify({
+        query:  text,
+        source: sources.length === 1 ? sources[0] : null,
+      }),
+    })
+    if (!res.ok || !res.body) throw new Error(`API ${res.status}`)
+
+    const reader  = res.body.getReader()
+    const decoder = new TextDecoder()
+    let   buf     = ''
+
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      buf += decoder.decode(value, { stream: true })
+      const parts = buf.split('\n\n')
+      buf = parts.pop() ?? ''
+      for (const part of parts) {
+        const line = part.trim()
+        if (!line.startsWith('data: ')) continue
+        const evt = JSON.parse(line.slice(6))
+        if      (evt.type === 'status')    { turn.status    = evt.message }
+        else if (evt.type === 'summary')   { turn.summary   = evt.data; turn.since = evt.data.window_start; turn.until = evt.data.window_end }
+        else if (evt.type === 'entries')   { turn.entries   = evt.data; turn.showEntries = evt.data.length > 0 && evt.data.length <= 10 }
+        else if (evt.type === 'reasoning') { turn.reasoning = evt.text; await nextTick(); scrollEl.value?.scrollTo({ top: scrollEl.value.scrollHeight, behavior: 'smooth' }) }
+        else if (evt.type === 'done')      { turn.status    = null }
+      }
+    }
+  } catch (e) {
+    turn.reasoning = `Error: ${e instanceof Error ? e.message : String(e)}`
+  } finally {
+    turn.loading     = false
+    turn.status      = null
+    activeTurn.value = null
+    await nextTick()
+    scrollEl.value?.scrollTo({ top: scrollEl.value.scrollHeight, behavior: 'smooth' })
+    textareaEl.value?.focus()
+  }
+}
+
+async function saveIncident(turn: Turn) {
+  turn.saving = true
+  try {
+    const res = await fetch(`${BASE}/api/incidents`, {
+      method:  'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body:    JSON.stringify({
+        label:      turn.query.slice(0, 120),
+        started_at: turn.since,
+        ended_at:   turn.until,
+        severity:   'medium',
+        notes:      turn.reasoning ?? '',
+      }),
+    })
+    if (!res.ok) throw new Error(await res.text())
+    turn.saved = true
+  } catch { /* surface silently — not worth crashing the chat */ }
+  finally   { turn.saving = false }
+}
+
+function fmtTs(iso: string | null): string {
+  if (!iso) return '—'
+  try {
+    return new Date(iso).toLocaleString(undefined, {
+      month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit',
+    })
+  } catch { return iso }
+}
+</script>
diff --git a/web/src/views/DiagnoseView.vue b/web/src/views/DiagnoseView.vue
index d434b31..75e182a 100644
--- a/web/src/views/DiagnoseView.vue
+++ b/web/src/views/DiagnoseView.vue
@@ -1,15 +1,20 @@
 <template>
-  <div class="p-4 sm:p-6 max-w-4xl mx-auto">
-    <div class="mb-5">
+  <div
+    class="p-4 sm:p-6 mx-auto"
+    :class="activeTab === 'chat' ? 'max-w-3xl flex flex-col' : 'max-w-4xl'"
+    :style="activeTab === 'chat' ? 'height: calc(100vh - 5rem)' : ''"
+  >
+    <div class="mb-5 shrink-0">
       <h1 class="text-text-primary text-xl font-semibold mb-1">Diagnose</h1>
       <p class="text-text-dim text-sm">
-        Quick: describe a symptom to surface log evidence.
-        Structured: tag a timestamped incident record.
+        <template v-if="activeTab === 'chat'">Describe your issue in plain language — Turnstone searches your logs and explains what it finds.</template>
+        <template v-else-if="activeTab === 'quick'">Single-shot: describe a symptom to surface log evidence and LLM reasoning.</template>
+        <template v-else>Tag and timestamp a known issue to build an incident record.</template>
       </p>
     </div>
 
-    <!-- Tab toggle -->
-    <div role="tablist" aria-label="Diagnose mode" class="flex gap-1 mb-6 border-b border-surface-border">
+    <!-- Tab strip -->
+    <div role="tablist" aria-label="Diagnose mode" class="flex gap-1 mb-6 border-b border-surface-border shrink-0">
       <button
         v-for="(t, idx) in tabs"
         :key="t.key"
@@ -18,7 +23,7 @@
         :id="`tab-${t.key}`"
         :aria-controls="`tabpanel-${t.key}`"
         :tabindex="activeTab === t.key ? 0 : -1"
-        @click="activeTab = t.key as 'quick' | 'structured'"
+        @click="activeTab = t.key as TabKey"
         @keydown="handleTabKey($event, t.key)"
         :ref="(el) => { if (el) tabRefs[idx] = el as HTMLButtonElement }"
         :class="[
@@ -30,7 +35,18 @@
       >{{ t.label }}</button>
     </div>
 
-    <!-- Quick tab panel -->
+    <!-- Chat tab — full-height flex layout -->
+    <div
+      v-show="activeTab === 'chat'"
+      role="tabpanel"
+      id="tabpanel-chat"
+      aria-labelledby="tab-chat"
+      class="flex-1 min-h-0"
+    >
+      <ChatDiagnose />
+    </div>
+
+    <!-- Quick tab -->
     <div
       v-show="activeTab === 'quick'"
       role="tabpanel"
@@ -41,7 +57,7 @@
       <QuickCapture />
     </div>
 
-    <!-- Structured tab panel -->
+    <!-- Structured tab -->
     <div
       v-show="activeTab === 'structured'"
       role="tabpanel"
@@ -64,36 +80,42 @@
 <script setup lang="ts">
 import { ref, onMounted, watch, nextTick } from 'vue'
 import { useRoute, RouterLink } from 'vue-router'
-import QuickCapture from '@/components/QuickCapture.vue'
-import IncidentForm from '@/components/IncidentForm.vue'
+import QuickCapture   from '@/components/QuickCapture.vue'
+import IncidentForm   from '@/components/IncidentForm.vue'
+import ChatDiagnose   from '@/components/ChatDiagnose.vue'
 
 const route = useRoute()
-const tabs: { key: 'quick' | 'structured'; label: string }[] = [
+
+type TabKey = 'chat' | 'quick' | 'structured'
+
+const tabs: { key: TabKey; label: string }[] = [
+  { key: 'chat',       label: 'Chat' },
   { key: 'quick',      label: 'Quick' },
   { key: 'structured', label: 'Structured' },
 ]
-const activeTab    = ref<'quick' | 'structured'>('quick')
+const activeTab    = ref<TabKey>('chat')
 const createdLabel = ref('')
 const tabRefs      = ref<HTMLButtonElement[]>([])
 
-function handleTabKey(e: KeyboardEvent, currentKey: 'quick' | 'structured') {
+function handleTabKey(e: KeyboardEvent, currentKey: TabKey) {
   const keys = tabs.map(t => t.key)
-  const idx = keys.indexOf(currentKey)
-  let next = idx
+  const idx  = keys.indexOf(currentKey)
+  let next   = idx
   if (e.key === 'ArrowRight') next = (idx + 1) % keys.length
   else if (e.key === 'ArrowLeft') next = (idx - 1 + keys.length) % keys.length
   else return
   e.preventDefault()
-  activeTab.value = keys[next] as 'quick' | 'structured'
+  activeTab.value = keys[next] as TabKey
   nextTick(() => tabRefs.value[next]?.focus())
 }
 
 onMounted(() => {
-  if (route.query.tab === 'structured') activeTab.value = 'structured'
+  const tab = route.query.tab as string | undefined
+  if (tab === 'structured' || tab === 'quick' || tab === 'chat') activeTab.value = tab
 })
 
 watch(() => route.query.tab, (tab) => {
-  if (tab === 'structured' || tab === 'quick') activeTab.value = tab
+  if (tab === 'structured' || tab === 'quick' || tab === 'chat') activeTab.value = tab as TabKey
 })
 
 function onCreated(label: string) {