pyr0ball
f941ebdeeb
Extends Pagepiper's document shelving pipeline (renamed from "ingest" — see below) to cover the formats most likely to appear in a real-world engineering document corpus, prompted by scoping a STERIS licensing pitch that needs DOCX/ODT coverage. - Rename the ingest pipeline to "shelve" throughout (scripts/, app/api, tests, docs, frontend). "Glean" (Turnstone's term) was considered and rejected — that's a harvest metaphor for log/knowledge extraction, not a fit for documents entering a library. Documented as a general CF naming principle in the org-level CLAUDE.md. - Wire DOCX into the upload/scan UI, README, and docs — the extraction logic (heading-based chunking, table serialization) already existed but wasn't exposed to users or covered by tests. - Add ODT support via odfpy, mirroring DOCX's chunking strategy. - Add Apple Pages support via headless LibreOffice conversion to ODT. No maintained Python library parses the IWA format directly; libreoffice bundles libetonyek, the only real open-source Pages parser. Adds libreoffice-writer to the Docker image (~300-400MB) for this. - 24 new/updated tests across shelve_docx, shelve_odt, and shelve_pages; full suite (72 tests) passing. Known gaps not addressed here: no Windchill/DocPortal connector exists yet (metadata-only PowerShell recon only), Excel/.xlsx is unsupported, and circuitforge_core.tasks.dispatch_task does not currently exist in circuitforge-core — cf-orch dispatch is dead code, always falling through to local BackgroundTasks. See circuitforge-plans/pagepiper/superpowers/plans/2026-07-10-steris-licensing-pitch.md for the full writeup.
100 lines
3.4 KiB
Python
100 lines
3.4 KiB
Python
# scripts/shelve_pages.py
|
|
"""
|
|
cf-orch task: pagepiper/shelve_pages
|
|
|
|
Converts an Apple Pages (.pages) file to ODT via headless LibreOffice
|
|
(soffice), then delegates extraction/chunking/embedding to shelve_odt's
|
|
pipeline.
|
|
|
|
.pages is IWA internally (Snappy-compressed Protocol Buffers) — no
|
|
maintained Python library parses it directly. LibreOffice bundles
|
|
libetonyek, the only mature open-source Pages parser, so shelling out to
|
|
headless LibreOffice is the only realistic full-fidelity extraction path.
|
|
|
|
Entry point:
|
|
python scripts/shelve_pages.py --doc-id X --file-path Y --db-path Z --vec-db-path W
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger("pagepiper.shelve_pages")
|
|
|
|
_CONVERT_TIMEOUT_SECONDS = 120
|
|
|
|
|
|
def _update_status_error(db_path: str, doc_id: str, error_msg: str) -> None:
|
|
conn = sqlite3.connect(db_path, timeout=30)
|
|
try:
|
|
conn.execute(
|
|
"UPDATE documents SET status='error', error_msg=?, updated_at=datetime('now') WHERE id=?",
|
|
[error_msg, doc_id],
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _convert_to_odt(pages_path: str, out_dir: str) -> str:
|
|
"""Shell out to headless LibreOffice to convert .pages -> .odt. Returns the output path."""
|
|
result = subprocess.run(
|
|
["soffice", "--headless", "--convert-to", "odt:writer8", "--outdir", out_dir, pages_path],
|
|
capture_output=True, text=True, timeout=_CONVERT_TIMEOUT_SECONDS,
|
|
)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(
|
|
f"LibreOffice conversion failed: {result.stderr.strip() or result.stdout.strip()}"
|
|
)
|
|
|
|
stem = Path(pages_path).stem
|
|
odt_path = Path(out_dir) / f"{stem}.odt"
|
|
if not odt_path.exists():
|
|
raise RuntimeError(f"LibreOffice reported success but produced no output for {pages_path}")
|
|
return str(odt_path)
|
|
|
|
|
|
def run(doc_id: str, file_path: str, db_path: str, vec_db_path: str) -> None:
|
|
"""Convert a .pages file to ODT, then run the ODT shelve pipeline against it."""
|
|
from scripts.shelve_odt import run as run_odt
|
|
|
|
tmp_dir = tempfile.mkdtemp(prefix="pagepiper_pages_")
|
|
try:
|
|
logger.info("Converting %s to ODT via headless LibreOffice", file_path)
|
|
odt_path = _convert_to_odt(file_path, tmp_dir)
|
|
logger.info("Converted to %s — handing off to ODT shelver", odt_path)
|
|
run_odt(doc_id=doc_id, file_path=odt_path, db_path=db_path, vec_db_path=vec_db_path)
|
|
except Exception as exc:
|
|
logger.error("Shelve failed for doc %s: %s", doc_id, exc, exc_info=True)
|
|
try:
|
|
_update_status_error(db_path, doc_id, str(exc))
|
|
except Exception:
|
|
logger.warning("Could not write error status for doc %s", doc_id)
|
|
raise
|
|
finally:
|
|
shutil.rmtree(tmp_dir, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Shelve an Apple Pages .pages file (cf-orch task entry point)"
|
|
)
|
|
parser.add_argument("--doc-id", required=True)
|
|
parser.add_argument("--file-path", required=True)
|
|
parser.add_argument("--db-path", required=True)
|
|
parser.add_argument("--vec-db-path", required=True)
|
|
a = parser.parse_args()
|
|
run(
|
|
doc_id=a.doc_id,
|
|
file_path=a.file_path,
|
|
db_path=a.db_path,
|
|
vec_db_path=a.vec_db_path,
|
|
)
|