pyr0ball
d39cfbd87a
Extends the shelve pipeline to cover spreadsheets, closing the Excel gap called out in the PR's original "known gaps" list — Windchill/DocPortal corpora commonly include parts lists and spec sheets as spreadsheets, not just prose documents. - scripts/shelve_xlsx.py — openpyxl, chunked by sheet with row-window splitting for large sheets (header row repeated in every window so each chunk stays self-describing for retrieval). - scripts/shelve_ods.py — same chunking strategy via odfpy (already a dependency from ODT support), OpenDocumentSpreadsheet's Table/TableRow/ TableCell. - scripts/shelve_numbers.py — converts via headless LibreOffice to XLSX and delegates to shelve_xlsx, mirroring shelve_pages.py's pattern for .pages. Adds libreoffice-calc to the Docker image alongside the existing libreoffice-writer. - Upload button text changed from an ever-growing format list to "Upload Document or Spreadsheet" — the Supported Formats table in README/docs is now the source of truth for the full list. - 13 new tests (XLSX, ODS, Numbers); full suite (85 tests) passing. Manually verified via Playwright against an isolated test instance: XLSX and ODS both upload, shelve to "ready", and store correctly row-serialized, header-repeated chunks (confirmed via sample-chunks). BM25 search against a 2-chunk toy corpus returned no hits for terms split 1-vs-1 across the two chunks — traced to Okapi BM25's IDF formula giving an exact 0 for terms in exactly half a tiny corpus (log((N-n+0.5)/(n+0.5)) = log(1.0) = 0, filtered by `score <= 0`), not a defect in the new shelvers. The earlier DOCX/ODT/PDF Playwright pass (5 chunks total) diluted this enough to return real results.
30 lines
685 B
TOML
30 lines
685 B
TOML
[build-system]
|
|
requires = ["setuptools>=68", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "pagepiper"
|
|
version = "0.1.0"
|
|
description = "Self-hosted PDF library manager with RAG chat and page-level citations"
|
|
readme = "README.md"
|
|
requires-python = ">=3.11"
|
|
dependencies = [
|
|
"fastapi>=0.110",
|
|
"uvicorn[standard]>=0.29",
|
|
"python-multipart>=0.0.9",
|
|
"rank-bm25>=0.2",
|
|
"PyYAML>=6.0",
|
|
"httpx>=0.27",
|
|
"circuitforge-core[pdf,vector]>=0.19.0",
|
|
"python-docx>=1.0",
|
|
"odfpy>=1.4",
|
|
"openpyxl>=3.1",
|
|
]
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["."]
|
|
include = ["app*"]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|
|
asyncio_mode = "auto"
|