pyr0ball
e52bdb5128
Retrieval:
- Add _fetch_adjacent() to retriever: fetches page ± 1 chunks from DB
after ranking so mid-sentence EPUB chunk boundaries don't lose context
- Fix vec DB doc-filter: oversample to top_k*20 before Python filter
instead of post-filtering an already-small global pool (fixes wrong-book
results when searching within a single document)
- top_k default 5 → 10; context per chunk 500 → 1500 chars; citation
snippet 200 → 400 chars
Artifact cleaning:
- Add scripts/text_clean.py: strips ABC Amber LIT Converter watermarks,
processtext.com URLs, bare page numbers, piracy stamps from extracted text
- Wire clean_paragraph() into ingest_pdf.py and new ingest_epub.py
Startup validation:
- _check_vec_schema() at boot: detects embedding dimension mismatch,
deletes stale vec DB, and queues sequential re-embed in background thread
- Sequential _reembed_docs() prevents SQLite lock races on startup re-embed
cf-orch integration:
- Wire CF_ORCH_URL / CF_LICENSE_KEY into LLMRouter backend config so
allocate() fires and keeps the Ollama model warm between requests
Ingestion progress UI:
- GET /api/library/{doc_id}/status now returns vec_count from page_vecs_meta
- DocumentCard.vue polls status every 3 s while processing and shows
two-phase progress: indeterminate animation during extraction,
determinate "Embedding N/M pages" bar once vectors start landing
Other:
- Chat feedback endpoint + thumbs up/down UI (FeedbackButton.vue)
- EPUB ingest script (ingest_epub.py) with heading-based chunking
- migration 002: chat_feedback table
- README.md with setup and feature overview
46 lines
1.6 KiB
Python
46 lines
1.6 KiB
Python
"""Configuration from environment variables — no file parsing required for basic use."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
DATA_DIR = Path(os.environ.get("PAGEPIPER_DATA_DIR", "data"))
|
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
DB_PATH = str(DATA_DIR / "pagepiper.db")
|
|
VEC_DB_PATH = str(DATA_DIR / "pagepiper_vecs.db")
|
|
WATCH_DIR = Path(os.environ.get("PAGEPIPER_WATCH_DIR", "books"))
|
|
VEC_DIMENSIONS = int(os.environ.get("PAGEPIPER_EMBED_DIMS", "1024"))
|
|
|
|
|
|
def get_llm_config() -> dict | None:
|
|
"""Build LLMRouter config from env vars. Returns None if PAGEPIPER_OLLAMA_URL is unset."""
|
|
url = os.environ.get("PAGEPIPER_OLLAMA_URL", "").strip()
|
|
if not url:
|
|
return None
|
|
_clean = url.rstrip("/")
|
|
_base_url = _clean if _clean.endswith("/v1") else _clean + "/v1"
|
|
chat_model = os.environ.get("PAGEPIPER_CHAT_MODEL", "mistral:7b")
|
|
|
|
backend: dict = {
|
|
"type": "openai_compat",
|
|
"base_url": _base_url,
|
|
"model": chat_model,
|
|
"embedding_model": os.environ.get("PAGEPIPER_EMBED_MODEL", "nomic-embed-text"),
|
|
"supports_images": False,
|
|
}
|
|
|
|
# Wire cf-orch allocation when coordinator is configured so the model stays warm
|
|
# and cold-start latency doesn't cause chat timeouts.
|
|
orch_url = os.environ.get("CF_ORCH_URL", "").strip()
|
|
if orch_url:
|
|
backend["cf_orch"] = {
|
|
"service": "ollama",
|
|
"model_candidates": [chat_model],
|
|
"ttl_s": 3600,
|
|
}
|
|
|
|
return {
|
|
"fallback_order": ["ollama"],
|
|
"backends": {"ollama": backend},
|
|
}
|