pyr0ball
d39cfbd87a
Extends the shelve pipeline to cover spreadsheets, closing the Excel gap called out in the PR's original "known gaps" list — Windchill/DocPortal corpora commonly include parts lists and spec sheets as spreadsheets, not just prose documents. - scripts/shelve_xlsx.py — openpyxl, chunked by sheet with row-window splitting for large sheets (header row repeated in every window so each chunk stays self-describing for retrieval). - scripts/shelve_ods.py — same chunking strategy via odfpy (already a dependency from ODT support), OpenDocumentSpreadsheet's Table/TableRow/ TableCell. - scripts/shelve_numbers.py — converts via headless LibreOffice to XLSX and delegates to shelve_xlsx, mirroring shelve_pages.py's pattern for .pages. Adds libreoffice-calc to the Docker image alongside the existing libreoffice-writer. - Upload button text changed from an ever-growing format list to "Upload Document or Spreadsheet" — the Supported Formats table in README/docs is now the source of truth for the full list. - 13 new tests (XLSX, ODS, Numbers); full suite (85 tests) passing. Manually verified via Playwright against an isolated test instance: XLSX and ODS both upload, shelve to "ready", and store correctly row-serialized, header-repeated chunks (confirmed via sample-chunks). BM25 search against a 2-chunk toy corpus returned no hits for terms split 1-vs-1 across the two chunks — traced to Okapi BM25's IDF formula giving an exact 0 for terms in exactly half a tiny corpus (log((N-n+0.5)/(n+0.5)) = log(1.0) = 0, filtered by `score <= 0`), not a defect in the new shelvers. The earlier DOCX/ODT/PDF Playwright pass (5 chunks total) diluted this enough to return real results.
101 lines
3.5 KiB
Python
101 lines
3.5 KiB
Python
# scripts/shelve_numbers.py
|
|
"""
|
|
cf-orch task: pagepiper/shelve_numbers
|
|
|
|
Converts an Apple Numbers (.numbers) file to XLSX via headless LibreOffice
|
|
(soffice), then delegates extraction/chunking/embedding to shelve_xlsx's
|
|
pipeline.
|
|
|
|
Like .pages, .numbers is IWA internally (Snappy-compressed Protocol
|
|
Buffers) — no maintained Python library parses it directly. LibreOffice
|
|
bundles libetonyek, the only mature open-source parser for Apple's iWork
|
|
formats, so shelling out to headless LibreOffice is the only realistic
|
|
full-fidelity extraction path.
|
|
|
|
Entry point:
|
|
python scripts/shelve_numbers.py --doc-id X --file-path Y --db-path Z --vec-db-path W
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger("pagepiper.shelve_numbers")
|
|
|
|
_CONVERT_TIMEOUT_SECONDS = 120
|
|
|
|
|
|
def _update_status_error(db_path: str, doc_id: str, error_msg: str) -> None:
|
|
conn = sqlite3.connect(db_path, timeout=30)
|
|
try:
|
|
conn.execute(
|
|
"UPDATE documents SET status='error', error_msg=?, updated_at=datetime('now') WHERE id=?",
|
|
[error_msg, doc_id],
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _convert_to_xlsx(numbers_path: str, out_dir: str) -> str:
|
|
"""Shell out to headless LibreOffice to convert .numbers -> .xlsx. Returns the output path."""
|
|
result = subprocess.run(
|
|
["soffice", "--headless", "--convert-to", "xlsx", "--outdir", out_dir, numbers_path],
|
|
capture_output=True, text=True, timeout=_CONVERT_TIMEOUT_SECONDS,
|
|
)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(
|
|
f"LibreOffice conversion failed: {result.stderr.strip() or result.stdout.strip()}"
|
|
)
|
|
|
|
stem = Path(numbers_path).stem
|
|
xlsx_path = Path(out_dir) / f"{stem}.xlsx"
|
|
if not xlsx_path.exists():
|
|
raise RuntimeError(f"LibreOffice reported success but produced no output for {numbers_path}")
|
|
return str(xlsx_path)
|
|
|
|
|
|
def run(doc_id: str, file_path: str, db_path: str, vec_db_path: str) -> None:
|
|
"""Convert a .numbers file to XLSX, then run the XLSX shelve pipeline against it."""
|
|
from scripts.shelve_xlsx import run as run_xlsx
|
|
|
|
tmp_dir = tempfile.mkdtemp(prefix="pagepiper_numbers_")
|
|
try:
|
|
logger.info("Converting %s to XLSX via headless LibreOffice", file_path)
|
|
xlsx_path = _convert_to_xlsx(file_path, tmp_dir)
|
|
logger.info("Converted to %s — handing off to XLSX shelver", xlsx_path)
|
|
run_xlsx(doc_id=doc_id, file_path=xlsx_path, db_path=db_path, vec_db_path=vec_db_path)
|
|
except Exception as exc:
|
|
logger.error("Shelve failed for doc %s: %s", doc_id, exc, exc_info=True)
|
|
try:
|
|
_update_status_error(db_path, doc_id, str(exc))
|
|
except Exception:
|
|
logger.warning("Could not write error status for doc %s", doc_id)
|
|
raise
|
|
finally:
|
|
shutil.rmtree(tmp_dir, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Shelve an Apple Numbers .numbers file (cf-orch task entry point)"
|
|
)
|
|
parser.add_argument("--doc-id", required=True)
|
|
parser.add_argument("--file-path", required=True)
|
|
parser.add_argument("--db-path", required=True)
|
|
parser.add_argument("--vec-db-path", required=True)
|
|
a = parser.parse_args()
|
|
run(
|
|
doc_id=a.doc_id,
|
|
file_path=a.file_path,
|
|
db_path=a.db_path,
|
|
vec_db_path=a.vec_db_path,
|
|
)
|