pyr0ball
f941ebdeeb
Extends Pagepiper's document shelving pipeline (renamed from "ingest" — see below) to cover the formats most likely to appear in a real-world engineering document corpus, prompted by scoping a STERIS licensing pitch that needs DOCX/ODT coverage. - Rename the ingest pipeline to "shelve" throughout (scripts/, app/api, tests, docs, frontend). "Glean" (Turnstone's term) was considered and rejected — that's a harvest metaphor for log/knowledge extraction, not a fit for documents entering a library. Documented as a general CF naming principle in the org-level CLAUDE.md. - Wire DOCX into the upload/scan UI, README, and docs — the extraction logic (heading-based chunking, table serialization) already existed but wasn't exposed to users or covered by tests. - Add ODT support via odfpy, mirroring DOCX's chunking strategy. - Add Apple Pages support via headless LibreOffice conversion to ODT. No maintained Python library parses the IWA format directly; libreoffice bundles libetonyek, the only real open-source Pages parser. Adds libreoffice-writer to the Docker image (~300-400MB) for this. - 24 new/updated tests across shelve_docx, shelve_odt, and shelve_pages; full suite (72 tests) passing. Known gaps not addressed here: no Windchill/DocPortal connector exists yet (metadata-only PowerShell recon only), Excel/.xlsx is unsupported, and circuitforge_core.tasks.dispatch_task does not currently exist in circuitforge-core — cf-orch dispatch is dead code, always falling through to local BackgroundTasks. See circuitforge-plans/pagepiper/superpowers/plans/2026-07-10-steris-licensing-pitch.md for the full writeup.
63 lines
2.1 KiB
Python
63 lines
2.1 KiB
Python
# app/main.py
|
|
"""FastAPI application factory for pagepiper."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from contextlib import asynccontextmanager
|
|
|
|
from fastapi import FastAPI
|
|
|
|
from app.config import DB_PATH, VEC_DB_PATH, VEC_DIMENSIONS
|
|
from app.services.bm25_index import BM25Index
|
|
|
|
logger = logging.getLogger("pagepiper")
|
|
|
|
# Per-user BM25 registry — keyed by user_id; "__local__" for single-user mode
|
|
_bm25_map: dict[str, BM25Index] = {}
|
|
|
|
|
|
def _get_bm25_for(user_id: str) -> BM25Index:
|
|
if user_id not in _bm25_map:
|
|
_bm25_map[user_id] = BM25Index()
|
|
return _bm25_map[user_id]
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
from app.cloud_session import CLOUD_MODE
|
|
from app.config import LOCAL_USER_ID
|
|
from app.startup import apply_migrations, check_and_rebuild_vec_schema
|
|
|
|
embed_model = os.environ.get("PAGEPIPER_EMBED_MODEL", "nomic-embed-text")
|
|
logger.info("Pagepiper starting — embed model: %s, dims: %d", embed_model, VEC_DIMENSIONS)
|
|
|
|
if CLOUD_MODE:
|
|
from app.startup import warn_if_unencrypted
|
|
from app.config import DATA_DIR
|
|
warn_if_unencrypted(str(DATA_DIR))
|
|
else:
|
|
# In cloud mode, per-user migration and vec schema check run on first request (deps.py).
|
|
apply_migrations(DB_PATH)
|
|
check_and_rebuild_vec_schema(VEC_DB_PATH, VEC_DIMENSIONS, DB_PATH)
|
|
_get_bm25_for(LOCAL_USER_ID).mark_dirty()
|
|
|
|
yield
|
|
|
|
|
|
app = FastAPI(title="Pagepiper", lifespan=lifespan)
|
|
|
|
# Register routers
|
|
from app.api.library import router as library_router # noqa: E402
|
|
from app.api.shelve import router as shelve_router # noqa: E402
|
|
from app.api.search import router as search_router # noqa: E402
|
|
from app.api.chat import router as chat_router # noqa: E402
|
|
from app.api.feedback import router as feedback_router # noqa: E402
|
|
from app.api.feedback_attach import router as feedback_attach_router # noqa: E402
|
|
|
|
app.include_router(library_router)
|
|
app.include_router(shelve_router)
|
|
app.include_router(search_router)
|
|
app.include_router(chat_router)
|
|
app.include_router(feedback_router, prefix="/api/v1/feedback")
|
|
app.include_router(feedback_attach_router, prefix="/api/v1/feedback")
|