app/api/chat.py

@@ -47,7 +47,7 @@ class ChatFeedbackRequest(BaseModel):
 
 
 def _get_llm_router():
-    """Return LLMRouter if Ollama configured, else None."""
+    """Return LLMRouter if Ollama/cf-orch configured, else None."""
     from app.config import get_llm_config
 
     cfg = get_llm_config()
@@ -58,6 +58,65 @@ def _get_llm_router():
     return LLMRouter(cfg)
 
 
+def _build_llm_for_alloc(alloc) -> "LLMRouter":
+    """Wrap a cf-orch task allocation in a minimal LLMRouter for completion calls."""
+    from circuitforge_core.llm import LLMRouter
+
+    base_url = alloc.url.rstrip("/")
+    if not base_url.endswith("/v1"):
+        base_url += "/v1"
+    cfg = {
+        "fallback_order": ["orch_task"],
+        "backends": {
+            "orch_task": {
+                "type": "openai_compat",
+                "base_url": base_url,
+                "model": alloc.model or "default",
+                "supports_images": False,
+            }
+        },
+    }
+    return LLMRouter(cfg)
+
+
+def _run_chat(req: "ChatRequest", ctx: "UserCtx", llm) -> "ChatResponse":
+    retriever = Retriever(ctx.bm25)
+    chunks = retriever.hybrid_search(
+        query=req.message,
+        top_k=req.top_k,
+        doc_ids=req.doc_ids,
+        db_path=ctx.db_path,
+        vec_db_path=ctx.vec_db_path,
+        llm=llm,
+    )
+    if not chunks:
+        return ChatResponse(
+            answer=(
+                "I couldn't find any relevant passages. "
+                "Try a different query or check which documents are indexed."
+            ),
+            citations=[],
+        )
+    synth = Synthesizer(llm)
+    result = synth.synthesize(
+        message=req.message,
+        history=[t.model_dump() for t in req.history],
+        chunks=chunks,
+    )
+    return ChatResponse(
+        answer=result.answer,
+        citations=[
+            {
+                "doc_id": c.doc_id,
+                "page_number": c.page_number,
+                "snippet": c.snippet,
+                "bm25_score": c.bm25_score,
+            }
+            for c in result.citations
+        ],
+    )
+
+
 def _require_llm():
     """Return LLMRouter or raise 402."""
     llm = _get_llm_router()
@@ -81,46 +140,24 @@ def chat(
     ctx: UserCtx = Depends(get_user_ctx),
     _tier: str = Depends(require_paid_tier),
 ) -> ChatResponse:
+    orch_url = os.environ.get("CF_ORCH_URL", "").strip()
+
+    if orch_url:
+        try:
+            from circuitforge_orch.client import CFOrchClient, TaskNotFound  # type: ignore[import]
+            api_key = os.environ.get("CF_LICENSE_KEY", "")
+            client = CFOrchClient(orch_url, api_key=api_key)
+            with client.task_allocate("pagepiper", "rag_query") as alloc:
+                llm = _build_llm_for_alloc(alloc)
+                return _run_chat(req, ctx, llm)
+        except Exception as exc:
+            logger.warning(
+                "cf-orch task allocation for pagepiper.rag_query failed, falling back to LLMRouter: %s",
+                exc,
+            )
+
     llm = _require_llm()
-
-    retriever = Retriever(ctx.bm25)
-    chunks = retriever.hybrid_search(
-        query=req.message,
-        top_k=req.top_k,
-        doc_ids=req.doc_ids,
-        db_path=ctx.db_path,
-        vec_db_path=ctx.vec_db_path,
-        llm=llm,
-    )
-
-    if not chunks:
-        return ChatResponse(
-            answer=(
-                "I couldn't find any relevant passages. "
-                "Try a different query or check which documents are indexed."
-            ),
-            citations=[],
-        )
-
-    synth = Synthesizer(llm)
-    result = synth.synthesize(
-        message=req.message,
-        history=[t.model_dump() for t in req.history],
-        chunks=chunks,
-    )
-
-    return ChatResponse(
-        answer=result.answer,
-        citations=[
-            {
-                "doc_id": c.doc_id,
-                "page_number": c.page_number,
-                "snippet": c.snippet,
-                "bm25_score": c.bm25_score,
-            }
-            for c in result.citations
-        ],
-    )
+    return _run_chat(req, ctx, llm)
 
 
 @router.get("/feedback/status")