pyr0ball
4c1940d12e
- app/services/diagnose/_llm_client.py: strip <think>…</think> blocks (case-insensitive, multiline) from LLM response content before it reaches the UI or any JSON parser — affects DeepSeek-R1, Qwen QwQ, and any other model that emits chain-of-thought in content - app/rest.py: suggest_sources now also returns untracked_names — query tokens that look like hostnames/service names but don't appear in any monitored source, so the UI can prompt the user to add them - web/src/components/ChatDiagnose.vue: show amber "Not monitoring: X" banner with "Add as a log source →" link when untracked_names present - tests/test_llm_client.py: 13 tests covering think-strip edge cases (single/multi-line, multiple blocks, case-insensitive, only-thinking) plus existing extract_content and JSON-fence helpers
87 lines
3.5 KiB
Python
87 lines
3.5 KiB
Python
"""Tests for diagnose/_llm_client.py — thinking-tag stripping and content extraction."""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
|
|
def _resp(content: str | None) -> dict:
|
|
if content is None:
|
|
return {"choices": []}
|
|
return {"choices": [{"message": {"content": content}}]}
|
|
|
|
|
|
class TestExtractContent:
|
|
def test_returns_plain_content(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
assert extract_content(_resp("hello world")) == "hello world"
|
|
|
|
def test_returns_none_on_empty_choices(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
assert extract_content({"choices": []}) is None
|
|
|
|
def test_returns_none_on_empty_content(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
assert extract_content(_resp("")) is None
|
|
|
|
def test_strips_single_think_block(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
raw = "<think>Let me reason about this…</think>\nThe answer is 42."
|
|
assert extract_content(_resp(raw)) == "The answer is 42."
|
|
|
|
def test_strips_multi_line_think_block(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
raw = "<think>\nStep 1: consider X\nStep 2: consider Y\n</think>\n\nFinal answer here."
|
|
result = extract_content(_resp(raw))
|
|
assert result == "Final answer here."
|
|
assert "<think>" not in result
|
|
|
|
def test_strips_multiple_think_blocks(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
raw = "<think>first</think> actual <think>second</think> content"
|
|
result = extract_content(_resp(raw))
|
|
assert "<think>" not in result
|
|
assert "actual" in result
|
|
assert "content" in result
|
|
|
|
def test_strips_case_insensitive(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
raw = "<THINK>hidden</THINK> visible"
|
|
result = extract_content(_resp(raw))
|
|
assert result == "visible"
|
|
|
|
def test_returns_none_when_only_thinking_remains(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
raw = "<think>only thinking, no output</think>"
|
|
assert extract_content(_resp(raw)) is None
|
|
|
|
def test_content_without_thinking_unchanged(self):
|
|
from app.services.diagnose._llm_client import extract_content
|
|
raw = "Redis OOM at 03:00 — key eviction triggered by batch job."
|
|
assert extract_content(_resp(raw)) == raw
|
|
|
|
|
|
class TestStripJsonFences:
|
|
def test_strips_json_fence(self):
|
|
from app.services.diagnose._llm_client import strip_json_fences
|
|
raw = "```json\n[{\"a\": 1}]\n```"
|
|
assert strip_json_fences(raw) == '[{"a": 1}]'
|
|
|
|
def test_strips_plain_fence(self):
|
|
from app.services.diagnose._llm_client import strip_json_fences
|
|
raw = "```\nhello\n```"
|
|
assert "```" not in strip_json_fences(raw)
|
|
|
|
|
|
class TestExtractFirstJsonArray:
|
|
def test_extracts_array_from_mixed_text(self):
|
|
from app.services.diagnose._llm_client import extract_first_json_array
|
|
raw = 'Here is the result:\n[{"id": 1}, {"id": 2}]\nThat is all.'
|
|
result = extract_first_json_array(raw)
|
|
import json
|
|
parsed = json.loads(result)
|
|
assert len(parsed) == 2
|
|
|
|
def test_returns_original_when_no_array(self):
|
|
from app.services.diagnose._llm_client import extract_first_json_array
|
|
raw = "no array here"
|
|
assert extract_first_json_array(raw) == raw
|