diff --git a/app/api/chat.py b/app/api/chat.py index 0fe3815..9226578 100644 --- a/app/api/chat.py +++ b/app/api/chat.py @@ -47,7 +47,7 @@ class ChatFeedbackRequest(BaseModel): def _get_llm_router(): - """Return LLMRouter if Ollama configured, else None.""" + """Return LLMRouter if Ollama/cf-orch configured, else None.""" from app.config import get_llm_config cfg = get_llm_config() @@ -58,6 +58,65 @@ def _get_llm_router(): return LLMRouter(cfg) +def _build_llm_for_alloc(alloc) -> "LLMRouter": + """Wrap a cf-orch task allocation in a minimal LLMRouter for completion calls.""" + from circuitforge_core.llm import LLMRouter + + base_url = alloc.url.rstrip("/") + if not base_url.endswith("/v1"): + base_url += "/v1" + cfg = { + "fallback_order": ["orch_task"], + "backends": { + "orch_task": { + "type": "openai_compat", + "base_url": base_url, + "model": alloc.model or "default", + "supports_images": False, + } + }, + } + return LLMRouter(cfg) + + +def _run_chat(req: "ChatRequest", ctx: "UserCtx", llm) -> "ChatResponse": + retriever = Retriever(ctx.bm25) + chunks = retriever.hybrid_search( + query=req.message, + top_k=req.top_k, + doc_ids=req.doc_ids, + db_path=ctx.db_path, + vec_db_path=ctx.vec_db_path, + llm=llm, + ) + if not chunks: + return ChatResponse( + answer=( + "I couldn't find any relevant passages. " + "Try a different query or check which documents are indexed." + ), + citations=[], + ) + synth = Synthesizer(llm) + result = synth.synthesize( + message=req.message, + history=[t.model_dump() for t in req.history], + chunks=chunks, + ) + return ChatResponse( + answer=result.answer, + citations=[ + { + "doc_id": c.doc_id, + "page_number": c.page_number, + "snippet": c.snippet, + "bm25_score": c.bm25_score, + } + for c in result.citations + ], + ) + + def _require_llm(): """Return LLMRouter or raise 402.""" llm = _get_llm_router() @@ -81,46 +140,24 @@ def chat( ctx: UserCtx = Depends(get_user_ctx), _tier: str = Depends(require_paid_tier), ) -> ChatResponse: + orch_url = os.environ.get("CF_ORCH_URL", "").strip() + + if orch_url: + try: + from circuitforge_orch.client import CFOrchClient, TaskNotFound # type: ignore[import] + api_key = os.environ.get("CF_LICENSE_KEY", "") + client = CFOrchClient(orch_url, api_key=api_key) + with client.task_allocate("pagepiper", "rag_query") as alloc: + llm = _build_llm_for_alloc(alloc) + return _run_chat(req, ctx, llm) + except Exception as exc: + logger.warning( + "cf-orch task allocation for pagepiper.rag_query failed, falling back to LLMRouter: %s", + exc, + ) + llm = _require_llm() - - retriever = Retriever(ctx.bm25) - chunks = retriever.hybrid_search( - query=req.message, - top_k=req.top_k, - doc_ids=req.doc_ids, - db_path=ctx.db_path, - vec_db_path=ctx.vec_db_path, - llm=llm, - ) - - if not chunks: - return ChatResponse( - answer=( - "I couldn't find any relevant passages. " - "Try a different query or check which documents are indexed." - ), - citations=[], - ) - - synth = Synthesizer(llm) - result = synth.synthesize( - message=req.message, - history=[t.model_dump() for t in req.history], - chunks=chunks, - ) - - return ChatResponse( - answer=result.answer, - citations=[ - { - "doc_id": c.doc_id, - "page_number": c.page_number, - "snippet": c.snippet, - "bm25_score": c.bm25_score, - } - for c in result.citations - ], - ) + return _run_chat(req, ctx, llm) @router.get("/feedback/status")