diff --git a/Dockerfile b/Dockerfile index a3dc1ad..a66ed9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,13 +2,15 @@ FROM continuumio/miniconda3:latest WORKDIR /app -# System deps for pytesseract (OCR), pdfplumber, and Apple Pages conversion -# (libreoffice-writer bundles libetonyek, the only maintained open-source -# .pages parser — shelve_pages.py shells out to headless soffice) +# System deps for pytesseract (OCR), pdfplumber, and Apple Pages/Numbers +# conversion (libreoffice bundles libetonyek, the only maintained open-source +# parser for Apple's iWork formats — shelve_pages.py / shelve_numbers.py +# shell out to headless soffice for .pages / .numbers respectively) RUN apt-get update && apt-get install -y --no-install-recommends \ tesseract-ocr \ libgl1 \ libreoffice-writer \ + libreoffice-calc \ && rm -rf /var/lib/apt/lists/* # Install circuitforge-core from sibling directory (compose sets context: ..) diff --git a/README.md b/README.md index e0f0b3e..7bd77f9 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![License: MIT / BSL 1.1](https://img.shields.io/badge/license-MIT%20%2F%20BSL%201.1-blue)](LICENSE) [![Version](https://img.shields.io/badge/version-v0.1.0-orange)](https://git.opensourcesolarpunk.com/Circuit-Forge/pagepiper/releases) -Self-hosted PDF, EPUB, DOCX, ODT, and Apple Pages search with BM25 (Best Match 25) full-text indexing and LLM (large language model) synthesis. Drop your documents in, ask a question, get an answer that tells you exactly which page to turn to. +Self-hosted document and spreadsheet search — PDF, EPUB, DOCX, ODT, Apple Pages, XLSX, ODS, and Apple Numbers — with BM25 (Best Match 25) full-text indexing and LLM (large language model) synthesis. Drop your documents in, ask a question, get an answer that tells you exactly which page to turn to. Built for TTRPG (tabletop roleplaying game) players who are tired of ctrl-F'ing through six-hundred-page rulebooks. Works equally well for legal research, technical manuals, academic papers, or any personal document library you want to query in plain language. @@ -86,6 +86,9 @@ PAGEPIPER_EMBED_MODEL=nomic-embed-text | DOCX | Yes | Yes (section/heading) | | ODT | Yes | Yes (section/heading) | | Pages | Yes | Yes (section/heading, via LibreOffice) | +| XLSX | Yes | Yes (sheet/row-window) | +| ODS | Yes | Yes (sheet/row-window) | +| Numbers | Yes | Yes (sheet/row-window, via LibreOffice) | --- @@ -123,7 +126,7 @@ Default ports: Web UI `8521`, API `8540`. | Feature | Free | Paid (BYOK) | |---------|------|-------------| -| PDF, EPUB, DOCX, ODT, and Pages upload | Yes | Yes | +| All supported document/spreadsheet formats upload | Yes | Yes | | Directory scan | Yes | Yes | | BM25 full-text search | Yes | Yes | | Unlimited local shelving | Yes | Yes | @@ -144,7 +147,7 @@ Pagepiper is developed and hosted at [git.opensourcesolarpunk.com/Circuit-Forge/ Pagepiper uses a split license: -- **MIT:** Document shelve pipeline, BM25 full-text index, library management, EPUB/DOCX/ODT/Pages support — the core discovery and retrieval layer. +- **MIT:** Document shelve pipeline, BM25 full-text index, library management, all format support (EPUB/DOCX/ODT/Pages/XLSX/ODS/Numbers) — the core discovery and retrieval layer. - **BSL 1.1 (Business Source License):** Hybrid vector search, LLM synthesis, RAG (retrieval-augmented generation) chat interface — free for personal non-commercial self-hosting; commercial use or SaaS re-hosting requires a license. Converts to MIT after four years. --- diff --git a/app/api/library.py b/app/api/library.py index 3349f22..5a6afb9 100644 --- a/app/api/library.py +++ b/app/api/library.py @@ -29,6 +29,9 @@ _SHELVE_TASKS = { ".docx": "pagepiper/shelve_docx", ".odt": "pagepiper/shelve_odt", ".pages": "pagepiper/shelve_pages", + ".xlsx": "pagepiper/shelve_xlsx", + ".ods": "pagepiper/shelve_ods", + ".numbers": "pagepiper/shelve_numbers", } _SHELVE_RUNNERS = { @@ -37,6 +40,9 @@ _SHELVE_RUNNERS = { ".docx": "scripts.shelve_docx", ".odt": "scripts.shelve_odt", ".pages": "scripts.shelve_pages", + ".xlsx": "scripts.shelve_xlsx", + ".ods": "scripts.shelve_ods", + ".numbers": "scripts.shelve_numbers", } @@ -143,6 +149,9 @@ def scan_library( + list(watch.glob("**/*.docx")) + list(watch.glob("**/*.odt")) + list(watch.glob("**/*.pages")) + + list(watch.glob("**/*.xlsx")) + + list(watch.glob("**/*.ods")) + + list(watch.glob("**/*.numbers")) ) queued = [] @@ -266,7 +275,10 @@ def upload_document( name = Path(file.filename or "").name suffix = Path(name).suffix.lower() if suffix not in _SHELVE_TASKS: - raise HTTPException(status_code=400, detail="Supported formats: PDF, EPUB, DOCX, ODT, Pages") + raise HTTPException( + status_code=400, + detail="Supported formats: PDF, EPUB, DOCX, ODT, Pages, XLSX, ODS, Numbers", + ) content = file.file.read() if len(content) > _MAX_UPLOAD_BYTES: diff --git a/app/startup.py b/app/startup.py index b982ba7..38e0d8f 100644 --- a/app/startup.py +++ b/app/startup.py @@ -70,6 +70,12 @@ def reembed_docs(docs: list[tuple[str, str]], db_path: str, vec_db_path: str) -> from scripts.shelve_odt import run elif suffix == ".pages": from scripts.shelve_pages import run + elif suffix == ".xlsx": + from scripts.shelve_xlsx import run + elif suffix == ".ods": + from scripts.shelve_ods import run + elif suffix == ".numbers": + from scripts.shelve_numbers import run else: from scripts.shelve_pdf import run logger.info("Auto re-embed: starting %s", os.path.basename(file_path)) diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index 889150d..4803d99 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -14,7 +14,7 @@ Open `http://localhost:8521` in your browser. You have two options: -**Upload directly** — click **Upload PDF / EPUB / DOCX / ODT / Pages** in the library header and pick a file from your computer. +**Upload directly** — click **Upload Document or Spreadsheet** in the library header and pick a file from your computer. **Scan a directory** — set `PAGEPIPER_WATCH_DIR` in your `.env` to a folder of PDFs or EPUBs, then click **Scan for PDFs**. Pagepiper indexes every file it finds. @@ -22,7 +22,7 @@ You have two options: The document card shows progress while text is being extracted and embedded: -- **Extracting text...** (animated bar) — PDF/EPUB/DOCX/ODT/Pages is being parsed into page chunks +- **Extracting text...** (animated bar) — the file is being parsed into page chunks - **Embedding N / M pages (X%)** (filling bar) — vectors are being written to the vector store (only when Ollama is configured) Once the badge shows **READY**, the document is searchable. diff --git a/docs/index.md b/docs/index.md index b1aed29..b7470cc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # Pagepiper -Self-hosted document search with BM25 full-text indexing and (with local Ollama) hybrid vector search and LLM-powered chat. Supports PDF, EPUB, DOCX, ODT, and Apple Pages files. +Self-hosted document search with BM25 full-text indexing and (with local Ollama) hybrid vector search and LLM-powered chat. Supports PDF, EPUB, DOCX, ODT, Apple Pages, XLSX, ODS, and Apple Numbers files. ## Demo @@ -25,7 +25,7 @@ Ask questions across your indexed documents. Results cite the source document an | Feature | Free | Paid (BYOK) | |---------|------|-------------| | BM25 full-text search | Yes | Yes | -| PDF, EPUB, DOCX, ODT, and Pages upload via browser | Yes | Yes | +| All supported formats upload via browser | Yes | Yes | | Unlimited local shelving | Yes | Yes | | Hybrid vector search | No | Yes (local Ollama) | | LLM chat over documents | No | Yes (local Ollama) | diff --git a/docs/reference/architecture.md b/docs/reference/architecture.md index f39ad8e..581780e 100644 --- a/docs/reference/architecture.md +++ b/docs/reference/architecture.md @@ -19,7 +19,7 @@ Browser (Vue 3 SPA) ## Shelve pipeline ``` -PDF / EPUB / DOCX / ODT / Pages file +any supported document or spreadsheet file │ ├─ PDFExtractor (pdfminer + OCR fallback) ← circuitforge_core │ or diff --git a/docs/user-guide/library.md b/docs/user-guide/library.md index e8af94d..77131e6 100644 --- a/docs/user-guide/library.md +++ b/docs/user-guide/library.md @@ -4,9 +4,9 @@ The library is the home screen. It shows all indexed documents and lets you add ## Adding documents -**Upload** — click **Upload PDF / EPUB / DOCX / ODT / Pages** and select a file. Files up to 200 MB are accepted. The document is saved to `data/uploads/` and queued for indexing immediately. +**Upload** — click **Upload Document or Spreadsheet** and select a file. Files up to 200 MB are accepted. The document is saved to `data/uploads/` and queued for indexing immediately. -**Scan** — set `PAGEPIPER_WATCH_DIR` to a directory in your `.env`, then click **Scan for PDFs**. Any PDF, EPUB, DOCX, ODT, or Pages file not already in the library is queued. Re-scanning is safe; already-indexed documents are skipped. +**Scan** — set `PAGEPIPER_WATCH_DIR` to a directory in your `.env`, then click **Scan for PDFs**. Any supported document or spreadsheet file not already in the library is queued. Re-scanning is safe; already-indexed documents are skipped. ## Document states diff --git a/mkdocs.yml b/mkdocs.yml index ae077e1..66e7443 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: Pagepiper -site_description: Self-hosted PDF, EPUB, DOCX, ODT, and Apple Pages library with BM25 full-text search, hybrid vector retrieval, and LLM-powered RAG chat. +site_description: Self-hosted document and spreadsheet library (PDF, EPUB, DOCX, ODT, Apple Pages, XLSX, ODS, Apple Numbers) with BM25 full-text search, hybrid vector retrieval, and LLM-powered RAG chat. site_author: Circuit Forge LLC site_url: https://docs.circuitforge.tech/pagepiper repo_url: https://git.opensourcesolarpunk.com/Circuit-Forge/pagepiper diff --git a/pyproject.toml b/pyproject.toml index ce6a2de..56362a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "circuitforge-core[pdf,vector]>=0.19.0", "python-docx>=1.0", "odfpy>=1.4", + "openpyxl>=3.1", ] [tool.setuptools.packages.find] diff --git a/scripts/shelve_numbers.py b/scripts/shelve_numbers.py new file mode 100644 index 0000000..9accb30 --- /dev/null +++ b/scripts/shelve_numbers.py @@ -0,0 +1,101 @@ +# scripts/shelve_numbers.py +""" +cf-orch task: pagepiper/shelve_numbers + +Converts an Apple Numbers (.numbers) file to XLSX via headless LibreOffice +(soffice), then delegates extraction/chunking/embedding to shelve_xlsx's +pipeline. + +Like .pages, .numbers is IWA internally (Snappy-compressed Protocol +Buffers) — no maintained Python library parses it directly. LibreOffice +bundles libetonyek, the only mature open-source parser for Apple's iWork +formats, so shelling out to headless LibreOffice is the only realistic +full-fidelity extraction path. + +Entry point: + python scripts/shelve_numbers.py --doc-id X --file-path Y --db-path Z --vec-db-path W +""" +from __future__ import annotations + +import logging +import shutil +import sqlite3 +import subprocess +import tempfile +from pathlib import Path + +logger = logging.getLogger("pagepiper.shelve_numbers") + +_CONVERT_TIMEOUT_SECONDS = 120 + + +def _update_status_error(db_path: str, doc_id: str, error_msg: str) -> None: + conn = sqlite3.connect(db_path, timeout=30) + try: + conn.execute( + "UPDATE documents SET status='error', error_msg=?, updated_at=datetime('now') WHERE id=?", + [error_msg, doc_id], + ) + conn.commit() + finally: + conn.close() + + +def _convert_to_xlsx(numbers_path: str, out_dir: str) -> str: + """Shell out to headless LibreOffice to convert .numbers -> .xlsx. Returns the output path.""" + result = subprocess.run( + ["soffice", "--headless", "--convert-to", "xlsx", "--outdir", out_dir, numbers_path], + capture_output=True, text=True, timeout=_CONVERT_TIMEOUT_SECONDS, + ) + if result.returncode != 0: + raise RuntimeError( + f"LibreOffice conversion failed: {result.stderr.strip() or result.stdout.strip()}" + ) + + stem = Path(numbers_path).stem + xlsx_path = Path(out_dir) / f"{stem}.xlsx" + if not xlsx_path.exists(): + raise RuntimeError(f"LibreOffice reported success but produced no output for {numbers_path}") + return str(xlsx_path) + + +def run(doc_id: str, file_path: str, db_path: str, vec_db_path: str) -> None: + """Convert a .numbers file to XLSX, then run the XLSX shelve pipeline against it.""" + from scripts.shelve_xlsx import run as run_xlsx + + tmp_dir = tempfile.mkdtemp(prefix="pagepiper_numbers_") + try: + logger.info("Converting %s to XLSX via headless LibreOffice", file_path) + xlsx_path = _convert_to_xlsx(file_path, tmp_dir) + logger.info("Converted to %s — handing off to XLSX shelver", xlsx_path) + run_xlsx(doc_id=doc_id, file_path=xlsx_path, db_path=db_path, vec_db_path=vec_db_path) + except Exception as exc: + logger.error("Shelve failed for doc %s: %s", doc_id, exc, exc_info=True) + try: + _update_status_error(db_path, doc_id, str(exc)) + except Exception: + logger.warning("Could not write error status for doc %s", doc_id) + raise + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +if __name__ == "__main__": + import argparse + + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description="Shelve an Apple Numbers .numbers file (cf-orch task entry point)" + ) + parser.add_argument("--doc-id", required=True) + parser.add_argument("--file-path", required=True) + parser.add_argument("--db-path", required=True) + parser.add_argument("--vec-db-path", required=True) + a = parser.parse_args() + run( + doc_id=a.doc_id, + file_path=a.file_path, + db_path=a.db_path, + vec_db_path=a.vec_db_path, + ) diff --git a/scripts/shelve_ods.py b/scripts/shelve_ods.py new file mode 100644 index 0000000..9a4560c --- /dev/null +++ b/scripts/shelve_ods.py @@ -0,0 +1,211 @@ +# scripts/shelve_ods.py +""" +cf-orch task: pagepiper/shelve_ods + +Extracts rows from an OpenDocument Spreadsheet (.ods), stores chunks in +SQLite, and (if Ollama is configured) generates embeddings in the +sqlite-vec store. + +Chunking strategy identical to shelve_xlsx.py: one chunk per sheet (a + element), row-windowed for large sheets with the header row +repeated in every window. + +Entry point: + python scripts/shelve_ods.py --doc-id X --file-path Y --db-path Z --vec-db-path W +""" +from __future__ import annotations + +import logging +import os +import sqlite3 +from dataclasses import dataclass + +logger = logging.getLogger("pagepiper.shelve_ods") + +EMBED_BATCH_SIZE = 64 +ROWS_PER_CHUNK = 200 + + +@dataclass +class _Chunk: + page_number: int + text: str + source: str + word_count: int + + +def _row_to_text(cells: list[str]) -> str: + cells = [c.strip() for c in cells] + return " | ".join(cells) if any(cells) else "" + + +def _sheet_to_chunks(sheet_name: str, rows: list[list[str]], start_page: int) -> list[_Chunk]: + """Chunk one sheet's rows into one-or-more page-numbered chunks.""" + text_rows = [_row_to_text(r) for r in rows] + text_rows = [r for r in text_rows if r] + if not text_rows: + return [] + + header = text_rows[0] + body = text_rows[1:] + + chunks: list[_Chunk] = [] + if len(body) == 0: + lines = [f"Sheet: {sheet_name}", header] + text = "\n".join(lines) + chunks.append(_Chunk(start_page, text, "sheet", len(text.split()))) + return chunks + + for i in range(0, len(body), ROWS_PER_CHUNK): + window = body[i : i + ROWS_PER_CHUNK] + lines = [f"Sheet: {sheet_name}", header] + window + text = "\n".join(lines) + chunks.append(_Chunk(start_page + len(chunks), text, "sheet", len(text.split()))) + + return chunks + + +def _extract_chunks(file_path: str) -> list[_Chunk]: + from odf.opendocument import load + from odf.table import Table, TableRow, TableCell + from odf import teletype + + doc = load(file_path) + chunks: list[_Chunk] = [] + for element in doc.spreadsheet.childNodes: + if element.qname[1] != "table": + continue + sheet_name = element.getAttribute("name") or "Sheet" + rows: list[list[str]] = [] + for row in element.getElementsByType(TableRow): + cells = [ + teletype.extractText(c).replace("\n", " ") + for c in row.getElementsByType(TableCell) + ] + rows.append(cells) + next_page = len(chunks) + 1 + chunks.extend(_sheet_to_chunks(sheet_name, rows, next_page)) + return chunks + + +def _update_status( + conn: sqlite3.Connection, + doc_id: str, + status: str, + page_count: int | None = None, + error_msg: str | None = None, +) -> None: + if page_count is not None: + conn.execute( + "UPDATE documents SET status=?, page_count=?, updated_at=datetime('now') WHERE id=?", + [status, page_count, doc_id], + ) + elif error_msg is not None: + conn.execute( + "UPDATE documents SET status=?, error_msg=?, updated_at=datetime('now') WHERE id=?", + [status, error_msg, doc_id], + ) + else: + conn.execute( + "UPDATE documents SET status=?, updated_at=datetime('now') WHERE id=?", + [status, doc_id], + ) + conn.commit() + + +def run(doc_id: str, file_path: str, db_path: str, vec_db_path: str) -> None: + """Run the full shelve pipeline for one ODS. Called by cf-orch or BackgroundTasks.""" + conn: sqlite3.Connection | None = None + try: + conn = sqlite3.connect(db_path, timeout=30) + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA foreign_keys = ON") + _update_status(conn, doc_id, "processing") + + logger.info("Extracting sheets from %s", file_path) + chunks = _extract_chunks(file_path) + logger.info("Extracted %d chunks", len(chunks)) + + from scripts.text_clean import clean_paragraph + conn.execute("DELETE FROM page_chunks WHERE doc_id=?", [doc_id]) + chunk_rows: list[tuple[str, int, str]] = [] + for chunk in chunks: + cleaned = clean_paragraph(chunk.text) + if not cleaned: + continue + row = conn.execute( + """INSERT INTO page_chunks(doc_id, page_number, text, source, word_count) + VALUES (?,?,?,?,?) RETURNING id""", + [doc_id, chunk.page_number, cleaned, chunk.source, len(cleaned.split())], + ).fetchone() + chunk_rows.append((row[0], chunk.page_number, cleaned)) + conn.commit() + + from app.config import get_llm_config + llm_cfg = get_llm_config() + if llm_cfg and chunks: + try: + logger.info("Embedding %d chunks", len(chunks)) + from circuitforge_core.llm import LLMRouter + from circuitforge_core.vector.sqlite_vec import LocalSQLiteVecStore + + router = LLMRouter(llm_cfg) + embed_dims = int(os.environ.get("PAGEPIPER_EMBED_DIMS", "1024")) + vec_store = LocalSQLiteVecStore( + db_path=vec_db_path, table="page_vecs", dimensions=embed_dims + ) + vec_store.delete_where({"doc_id": doc_id}) + + texts = [text for _, _, text in chunk_rows] + vectors: list[list[float]] = [] + for i in range(0, len(texts), EMBED_BATCH_SIZE): + vectors.extend(router.embed(texts[i : i + EMBED_BATCH_SIZE])) + + for (chunk_id, page_number, _), vector in zip(chunk_rows, vectors): + vec_store.upsert( + entry_id=chunk_id, + vector=vector, + metadata={"doc_id": doc_id, "page_number": page_number}, + ) + logger.info("Stored %d embeddings", len(vectors)) + except Exception as embed_exc: + logger.warning( + "Embedding skipped for doc %s — BM25 only (reason: %s)", + doc_id, embed_exc, + ) + + _update_status(conn, doc_id, "ready", page_count=len(chunks)) + logger.info("Shelve complete for doc %s (%d chunks)", doc_id, len(chunks)) + + except Exception as exc: + logger.error("Shelve failed for doc %s: %s", doc_id, exc, exc_info=True) + if conn is not None: + try: + _update_status(conn, doc_id, "error", error_msg=str(exc)) + except Exception: + logger.warning("Could not write error status for doc %s", doc_id) + raise + finally: + if conn is not None: + conn.close() + + +if __name__ == "__main__": + import argparse + + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description="Shelve an OpenDocument Spreadsheet .ods (cf-orch task entry point)" + ) + parser.add_argument("--doc-id", required=True) + parser.add_argument("--file-path", required=True) + parser.add_argument("--db-path", required=True) + parser.add_argument("--vec-db-path", required=True) + a = parser.parse_args() + run( + doc_id=a.doc_id, + file_path=a.file_path, + db_path=a.db_path, + vec_db_path=a.vec_db_path, + ) diff --git a/scripts/shelve_xlsx.py b/scripts/shelve_xlsx.py new file mode 100644 index 0000000..4389917 --- /dev/null +++ b/scripts/shelve_xlsx.py @@ -0,0 +1,208 @@ +# scripts/shelve_xlsx.py +""" +cf-orch task: pagepiper/shelve_xlsx + +Extracts rows from an Excel .xlsx workbook, stores chunks in SQLite, and +(if Ollama is configured) generates embeddings in the sqlite-vec store. + +Chunking strategy: + - One chunk per sheet if the sheet has <= ROWS_PER_CHUNK rows. + - Larger sheets split into row-window chunks, with the header row (the + first row) repeated at the top of every window so each chunk stays + self-describing for BM25/embedding retrieval. + +Rows are serialised as pipe-delimited cells, matching the table-serialization +style used by the DOCX/ODT shelvers. + +Entry point: + python scripts/shelve_xlsx.py --doc-id X --file-path Y --db-path Z --vec-db-path W +""" +from __future__ import annotations + +import logging +import os +import sqlite3 +from dataclasses import dataclass + +logger = logging.getLogger("pagepiper.shelve_xlsx") + +EMBED_BATCH_SIZE = 64 +ROWS_PER_CHUNK = 200 + + +@dataclass +class _Chunk: + page_number: int + text: str + source: str + word_count: int + + +def _row_to_text(row: tuple) -> str: + cells = [str(c).strip() if c is not None else "" for c in row] + return " | ".join(cells) if any(cells) else "" + + +def _sheet_to_chunks(sheet_name: str, rows: list[tuple], start_page: int) -> list[_Chunk]: + """Chunk one sheet's rows into one-or-more page-numbered chunks.""" + text_rows = [_row_to_text(r) for r in rows] + text_rows = [r for r in text_rows if r] + if not text_rows: + return [] + + header = text_rows[0] + body = text_rows[1:] + + chunks: list[_Chunk] = [] + if len(body) == 0: + lines = [f"Sheet: {sheet_name}", header] + text = "\n".join(lines) + chunks.append(_Chunk(start_page, text, "sheet", len(text.split()))) + return chunks + + for i in range(0, len(body), ROWS_PER_CHUNK): + window = body[i : i + ROWS_PER_CHUNK] + lines = [f"Sheet: {sheet_name}", header] + window + text = "\n".join(lines) + chunks.append(_Chunk(start_page + len(chunks), text, "sheet", len(text.split()))) + + return chunks + + +def _extract_chunks(file_path: str) -> list[_Chunk]: + import openpyxl + + wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True) + try: + chunks: list[_Chunk] = [] + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + rows = list(ws.iter_rows(values_only=True)) + next_page = len(chunks) + 1 + chunks.extend(_sheet_to_chunks(sheet_name, rows, next_page)) + return chunks + finally: + wb.close() + + +def _update_status( + conn: sqlite3.Connection, + doc_id: str, + status: str, + page_count: int | None = None, + error_msg: str | None = None, +) -> None: + if page_count is not None: + conn.execute( + "UPDATE documents SET status=?, page_count=?, updated_at=datetime('now') WHERE id=?", + [status, page_count, doc_id], + ) + elif error_msg is not None: + conn.execute( + "UPDATE documents SET status=?, error_msg=?, updated_at=datetime('now') WHERE id=?", + [status, error_msg, doc_id], + ) + else: + conn.execute( + "UPDATE documents SET status=?, updated_at=datetime('now') WHERE id=?", + [status, doc_id], + ) + conn.commit() + + +def run(doc_id: str, file_path: str, db_path: str, vec_db_path: str) -> None: + """Run the full shelve pipeline for one XLSX. Called by cf-orch or BackgroundTasks.""" + conn: sqlite3.Connection | None = None + try: + conn = sqlite3.connect(db_path, timeout=30) + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA foreign_keys = ON") + _update_status(conn, doc_id, "processing") + + logger.info("Extracting sheets from %s", file_path) + chunks = _extract_chunks(file_path) + logger.info("Extracted %d chunks", len(chunks)) + + from scripts.text_clean import clean_paragraph + conn.execute("DELETE FROM page_chunks WHERE doc_id=?", [doc_id]) + chunk_rows: list[tuple[str, int, str]] = [] + for chunk in chunks: + cleaned = clean_paragraph(chunk.text) + if not cleaned: + continue + row = conn.execute( + """INSERT INTO page_chunks(doc_id, page_number, text, source, word_count) + VALUES (?,?,?,?,?) RETURNING id""", + [doc_id, chunk.page_number, cleaned, chunk.source, len(cleaned.split())], + ).fetchone() + chunk_rows.append((row[0], chunk.page_number, cleaned)) + conn.commit() + + from app.config import get_llm_config + llm_cfg = get_llm_config() + if llm_cfg and chunks: + try: + logger.info("Embedding %d chunks", len(chunks)) + from circuitforge_core.llm import LLMRouter + from circuitforge_core.vector.sqlite_vec import LocalSQLiteVecStore + + router = LLMRouter(llm_cfg) + embed_dims = int(os.environ.get("PAGEPIPER_EMBED_DIMS", "1024")) + vec_store = LocalSQLiteVecStore( + db_path=vec_db_path, table="page_vecs", dimensions=embed_dims + ) + vec_store.delete_where({"doc_id": doc_id}) + + texts = [text for _, _, text in chunk_rows] + vectors: list[list[float]] = [] + for i in range(0, len(texts), EMBED_BATCH_SIZE): + vectors.extend(router.embed(texts[i : i + EMBED_BATCH_SIZE])) + + for (chunk_id, page_number, _), vector in zip(chunk_rows, vectors): + vec_store.upsert( + entry_id=chunk_id, + vector=vector, + metadata={"doc_id": doc_id, "page_number": page_number}, + ) + logger.info("Stored %d embeddings", len(vectors)) + except Exception as embed_exc: + logger.warning( + "Embedding skipped for doc %s — BM25 only (reason: %s)", + doc_id, embed_exc, + ) + + _update_status(conn, doc_id, "ready", page_count=len(chunks)) + logger.info("Shelve complete for doc %s (%d chunks)", doc_id, len(chunks)) + + except Exception as exc: + logger.error("Shelve failed for doc %s: %s", doc_id, exc, exc_info=True) + if conn is not None: + try: + _update_status(conn, doc_id, "error", error_msg=str(exc)) + except Exception: + logger.warning("Could not write error status for doc %s", doc_id) + raise + finally: + if conn is not None: + conn.close() + + +if __name__ == "__main__": + import argparse + + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + description="Shelve an Excel .xlsx workbook (cf-orch task entry point)" + ) + parser.add_argument("--doc-id", required=True) + parser.add_argument("--file-path", required=True) + parser.add_argument("--db-path", required=True) + parser.add_argument("--vec-db-path", required=True) + a = parser.parse_args() + run( + doc_id=a.doc_id, + file_path=a.file_path, + db_path=a.db_path, + vec_db_path=a.vec_db_path, + ) diff --git a/tests/test_shelve_numbers.py b/tests/test_shelve_numbers.py new file mode 100644 index 0000000..b193da8 --- /dev/null +++ b/tests/test_shelve_numbers.py @@ -0,0 +1,118 @@ +# tests/test_shelve_numbers.py +"""Unit tests for scripts/shelve_numbers.py. + +soffice isn't available in the test environment, so _convert_to_xlsx is +mocked to copy a pre-built XLSX fixture into the expected output path — +everything downstream (extraction/chunking/storage) runs for real via +scripts.shelve_xlsx.run. +""" +from __future__ import annotations + +import shutil +import sqlite3 +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +from scripts.shelve_numbers import run + + +@pytest.fixture +def shelve_db(tmp_path) -> tuple[str, str]: + db_path = str(tmp_path / "test.db") + schema = Path("migrations/001_initial_schema.sql").read_text() + conn = sqlite3.connect(db_path) + conn.executescript(schema) + conn.execute( + "INSERT INTO documents(id, title, file_path, status) VALUES ('d1','Test','test.numbers','pending')" + ) + conn.commit() + conn.close() + vec_db_path = str(tmp_path / "vecs.db") + return db_path, vec_db_path + + +def _make_fixture_xlsx(path: Path) -> None: + import openpyxl + + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Config" + ws.append(["Field", "Value"]) + ws.append(["Default IP", "192.168.1.50"]) + wb.save(str(path)) + + +def _fake_soffice_convert(fixture_xlsx: Path): + def _fake_run(cmd, **kwargs): + out_dir = Path(cmd[cmd.index("--outdir") + 1]) + numbers_path = Path(cmd[-1]) + shutil.copy(fixture_xlsx, out_dir / f"{numbers_path.stem}.xlsx") + return subprocess.CompletedProcess(cmd, returncode=0, stdout="", stderr="") + + return _fake_run + + +def test_shelve_numbers_converts_and_shelves_successfully(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + numbers_path = tmp_path / "test.numbers" + numbers_path.write_bytes(b"not a real numbers bundle, just needs to exist") + + fixture_xlsx = tmp_path / "fixture.xlsx" + _make_fixture_xlsx(fixture_xlsx) + + with patch("scripts.shelve_numbers.subprocess.run", side_effect=_fake_soffice_convert(fixture_xlsx)): + run(doc_id="d1", file_path=str(numbers_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + row = conn.execute("SELECT status, page_count FROM documents WHERE id='d1'").fetchone() + rows = conn.execute( + "SELECT text FROM page_chunks WHERE doc_id='d1' ORDER BY page_number" + ).fetchall() + conn.close() + assert row[0] == "ready" + assert row[1] == 1 + assert "Default IP | 192.168.1.50" in rows[0][0] + + +def test_shelve_numbers_sets_error_status_on_conversion_failure(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + numbers_path = tmp_path / "test.numbers" + numbers_path.write_bytes(b"not a real numbers bundle") + + fake_result = subprocess.CompletedProcess( + args=[], returncode=1, stdout="", stderr="soffice: unsupported document format" + ) + with patch("scripts.shelve_numbers.subprocess.run", return_value=fake_result): + with pytest.raises(RuntimeError, match="LibreOffice conversion failed"): + run(doc_id="d1", file_path=str(numbers_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + row = conn.execute("SELECT status, error_msg FROM documents WHERE id='d1'").fetchone() + conn.close() + assert row[0] == "error" + assert "unsupported document format" in row[1] + + +def test_shelve_numbers_cleans_up_temp_dir(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + numbers_path = tmp_path / "test.numbers" + numbers_path.write_bytes(b"not a real numbers bundle") + + fixture_xlsx = tmp_path / "fixture.xlsx" + _make_fixture_xlsx(fixture_xlsx) + + captured_tmp_dirs: list[str] = [] + real_convert = _fake_soffice_convert(fixture_xlsx) + + def _tracking_run(cmd, **kwargs): + captured_tmp_dirs.append(cmd[cmd.index("--outdir") + 1]) + return real_convert(cmd, **kwargs) + + with patch("scripts.shelve_numbers.subprocess.run", side_effect=_tracking_run): + run(doc_id="d1", file_path=str(numbers_path), db_path=db_path, vec_db_path=vec_db_path) + + assert captured_tmp_dirs + assert not Path(captured_tmp_dirs[0]).exists(), "temp conversion dir should be cleaned up" diff --git a/tests/test_shelve_ods.py b/tests/test_shelve_ods.py new file mode 100644 index 0000000..788465e --- /dev/null +++ b/tests/test_shelve_ods.py @@ -0,0 +1,135 @@ +# tests/test_shelve_ods.py +"""Unit tests for scripts/shelve_ods.py.""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from scripts.shelve_ods import run, ROWS_PER_CHUNK + + +@pytest.fixture +def shelve_db(tmp_path) -> tuple[str, str]: + db_path = str(tmp_path / "test.db") + schema = Path("migrations/001_initial_schema.sql").read_text() + conn = sqlite3.connect(db_path) + conn.executescript(schema) + conn.execute( + "INSERT INTO documents(id, title, file_path, status) VALUES ('d1','Test','test.ods','pending')" + ) + conn.commit() + conn.close() + vec_db_path = str(tmp_path / "vecs.db") + return db_path, vec_db_path + + +def _make_ods(path: Path, big_sheet: bool = False) -> None: + from odf.opendocument import OpenDocumentSpreadsheet + from odf.table import Table, TableRow, TableCell + from odf.text import P + + def _row(vals): + row = TableRow() + for v in vals: + cell = TableCell() + cell.addElement(P(text=v)) + row.addElement(cell) + return row + + doc = OpenDocumentSpreadsheet() + + t1 = Table(name="Config") + t1.addElement(_row(["Field", "Value"])) + t1.addElement(_row(["Default IP", "192.168.1.50"])) + t1.addElement(_row(["AVC-X Model", "AVC-X200"])) + doc.spreadsheet.addElement(t1) + + t2 = Table(name="Parts List") + t2.addElement(_row(["Part Number", "Description"])) + if big_sheet: + for i in range(ROWS_PER_CHUNK + 50): + t2.addElement(_row([f"P{i:04d}", f"Widget {i}"])) + else: + t2.addElement(_row(["P0001", "Filter cartridge"])) + doc.spreadsheet.addElement(t2) + + doc.save(str(path)) + + +def test_shelve_ods_sets_status_ready_on_success(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + ods_path = tmp_path / "test.ods" + _make_ods(ods_path) + + run(doc_id="d1", file_path=str(ods_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + row = conn.execute("SELECT status, page_count FROM documents WHERE id='d1'").fetchone() + conn.close() + assert row[0] == "ready" + assert row[1] == 2 + + +def test_shelve_ods_serializes_rows_with_header(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + ods_path = tmp_path / "test.ods" + _make_ods(ods_path) + + run(doc_id="d1", file_path=str(ods_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + rows = conn.execute( + "SELECT text FROM page_chunks WHERE doc_id='d1' ORDER BY page_number" + ).fetchall() + conn.close() + assert "Sheet: Config" in rows[0][0] + assert "Field | Value" in rows[0][0] + assert "Default IP | 192.168.1.50" in rows[0][0] + assert "Sheet: Parts List" in rows[1][0] + assert "P0001 | Filter cartridge" in rows[1][0] + + +def test_shelve_ods_splits_large_sheet_into_row_windows(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + ods_path = tmp_path / "test.ods" + _make_ods(ods_path, big_sheet=True) + + run(doc_id="d1", file_path=str(ods_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + rows = conn.execute( + "SELECT text FROM page_chunks WHERE doc_id='d1' ORDER BY page_number" + ).fetchall() + conn.close() + assert len(rows) >= 3 + parts_list_chunks = [r[0] for r in rows if "Sheet: Parts List" in r[0]] + assert len(parts_list_chunks) >= 2 + for chunk_text in parts_list_chunks: + assert "Part Number | Description" in chunk_text + + +def test_shelve_ods_sets_error_status_on_failure(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + missing_path = tmp_path / "does-not-exist.ods" + + with pytest.raises(Exception): + run(doc_id="d1", file_path=str(missing_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + row = conn.execute("SELECT status, error_msg FROM documents WHERE id='d1'").fetchone() + conn.close() + assert row[0] == "error" + assert row[1] + + +def test_shelve_ods_skips_embeddings_without_ollama_url(shelve_db, tmp_path, monkeypatch): + db_path, vec_db_path = shelve_db + monkeypatch.delenv("PAGEPIPER_OLLAMA_URL", raising=False) + ods_path = tmp_path / "test.ods" + _make_ods(ods_path) + + run(doc_id="d1", file_path=str(ods_path), db_path=db_path, vec_db_path=vec_db_path) + + assert not Path(vec_db_path).exists(), "vec DB should not be created without OLLAMA_URL" diff --git a/tests/test_shelve_xlsx.py b/tests/test_shelve_xlsx.py new file mode 100644 index 0000000..494aea4 --- /dev/null +++ b/tests/test_shelve_xlsx.py @@ -0,0 +1,125 @@ +# tests/test_shelve_xlsx.py +"""Unit tests for scripts/shelve_xlsx.py.""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from scripts.shelve_xlsx import run, ROWS_PER_CHUNK + + +@pytest.fixture +def shelve_db(tmp_path) -> tuple[str, str]: + db_path = str(tmp_path / "test.db") + schema = Path("migrations/001_initial_schema.sql").read_text() + conn = sqlite3.connect(db_path) + conn.executescript(schema) + conn.execute( + "INSERT INTO documents(id, title, file_path, status) VALUES ('d1','Test','test.xlsx','pending')" + ) + conn.commit() + conn.close() + vec_db_path = str(tmp_path / "vecs.db") + return db_path, vec_db_path + + +def _make_xlsx(path: Path, big_sheet: bool = False) -> None: + import openpyxl + + wb = openpyxl.Workbook() + ws1 = wb.active + ws1.title = "Config" + ws1.append(["Field", "Value"]) + ws1.append(["Default IP", "192.168.1.50"]) + ws1.append(["AVC-X Model", "AVC-X200"]) + + ws2 = wb.create_sheet("Parts List") + ws2.append(["Part Number", "Description"]) + if big_sheet: + for i in range(ROWS_PER_CHUNK + 50): + ws2.append([f"P{i:04d}", f"Widget {i}"]) + else: + ws2.append(["P0001", "Filter cartridge"]) + + wb.save(str(path)) + + +def test_shelve_xlsx_sets_status_ready_on_success(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + xlsx_path = tmp_path / "test.xlsx" + _make_xlsx(xlsx_path) + + run(doc_id="d1", file_path=str(xlsx_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + row = conn.execute("SELECT status, page_count FROM documents WHERE id='d1'").fetchone() + conn.close() + assert row[0] == "ready" + assert row[1] == 2 # one chunk per sheet, both small + + +def test_shelve_xlsx_serializes_rows_with_header(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + xlsx_path = tmp_path / "test.xlsx" + _make_xlsx(xlsx_path) + + run(doc_id="d1", file_path=str(xlsx_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + rows = conn.execute( + "SELECT text FROM page_chunks WHERE doc_id='d1' ORDER BY page_number" + ).fetchall() + conn.close() + assert "Sheet: Config" in rows[0][0] + assert "Field | Value" in rows[0][0] + assert "Default IP | 192.168.1.50" in rows[0][0] + assert "Sheet: Parts List" in rows[1][0] + assert "P0001 | Filter cartridge" in rows[1][0] + + +def test_shelve_xlsx_splits_large_sheet_into_row_windows(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + xlsx_path = tmp_path / "test.xlsx" + _make_xlsx(xlsx_path, big_sheet=True) + + run(doc_id="d1", file_path=str(xlsx_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + rows = conn.execute( + "SELECT text FROM page_chunks WHERE doc_id='d1' ORDER BY page_number" + ).fetchall() + conn.close() + # Config sheet (1 chunk) + Parts List sheet split across >=2 row-window chunks + assert len(rows) >= 3 + parts_list_chunks = [r[0] for r in rows if "Sheet: Parts List" in r[0]] + assert len(parts_list_chunks) >= 2 + # header row repeated in every window + for chunk_text in parts_list_chunks: + assert "Part Number | Description" in chunk_text + + +def test_shelve_xlsx_sets_error_status_on_failure(shelve_db, tmp_path): + db_path, vec_db_path = shelve_db + missing_path = tmp_path / "does-not-exist.xlsx" + + with pytest.raises(Exception): + run(doc_id="d1", file_path=str(missing_path), db_path=db_path, vec_db_path=vec_db_path) + + conn = sqlite3.connect(db_path) + row = conn.execute("SELECT status, error_msg FROM documents WHERE id='d1'").fetchone() + conn.close() + assert row[0] == "error" + assert row[1] + + +def test_shelve_xlsx_skips_embeddings_without_ollama_url(shelve_db, tmp_path, monkeypatch): + db_path, vec_db_path = shelve_db + monkeypatch.delenv("PAGEPIPER_OLLAMA_URL", raising=False) + xlsx_path = tmp_path / "test.xlsx" + _make_xlsx(xlsx_path) + + run(doc_id="d1", file_path=str(xlsx_path), db_path=db_path, vec_db_path=vec_db_path) + + assert not Path(vec_db_path).exists(), "vec DB should not be created without OLLAMA_URL" diff --git a/web/src/views/LibraryView.vue b/web/src/views/LibraryView.vue index 6d503c4..4bbf7f7 100644 --- a/web/src/views/LibraryView.vue +++ b/web/src/views/LibraryView.vue @@ -4,9 +4,9 @@

Library

- +