pyr0ball
f941ebdeeb
Extends Pagepiper's document shelving pipeline (renamed from "ingest" — see below) to cover the formats most likely to appear in a real-world engineering document corpus, prompted by scoping a STERIS licensing pitch that needs DOCX/ODT coverage. - Rename the ingest pipeline to "shelve" throughout (scripts/, app/api, tests, docs, frontend). "Glean" (Turnstone's term) was considered and rejected — that's a harvest metaphor for log/knowledge extraction, not a fit for documents entering a library. Documented as a general CF naming principle in the org-level CLAUDE.md. - Wire DOCX into the upload/scan UI, README, and docs — the extraction logic (heading-based chunking, table serialization) already existed but wasn't exposed to users or covered by tests. - Add ODT support via odfpy, mirroring DOCX's chunking strategy. - Add Apple Pages support via headless LibreOffice conversion to ODT. No maintained Python library parses the IWA format directly; libreoffice bundles libetonyek, the only real open-source Pages parser. Adds libreoffice-writer to the Docker image (~300-400MB) for this. - 24 new/updated tests across shelve_docx, shelve_odt, and shelve_pages; full suite (72 tests) passing. Known gaps not addressed here: no Windchill/DocPortal connector exists yet (metadata-only PowerShell recon only), Excel/.xlsx is unsupported, and circuitforge_core.tasks.dispatch_task does not currently exist in circuitforge-core — cf-orch dispatch is dead code, always falling through to local BackgroundTasks. See circuitforge-plans/pagepiper/superpowers/plans/2026-07-10-steris-licensing-pitch.md for the full writeup.
105 lines
3.5 KiB
Python
105 lines
3.5 KiB
Python
# tests/test_library_api.py
|
|
"""Tests for GET/POST /api/library endpoints."""
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
|
|
|
|
def _add_doc(db_path: str, title: str, path: str, status: str = "ready") -> str:
|
|
conn = sqlite3.connect(db_path)
|
|
doc_id = conn.execute(
|
|
"INSERT INTO documents(title, file_path, status) VALUES (?,?,?) RETURNING id",
|
|
[title, path, status],
|
|
).fetchone()[0]
|
|
conn.commit()
|
|
conn.close()
|
|
return doc_id
|
|
|
|
|
|
def test_list_library_empty(client):
|
|
resp = client.get("/api/library")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
def test_list_library_returns_documents(client, test_db):
|
|
_add_doc(test_db, "Player's Handbook", "/books/phb.pdf")
|
|
resp = client.get("/api/library")
|
|
assert resp.status_code == 200
|
|
docs = resp.json()
|
|
assert len(docs) == 1
|
|
assert docs[0]["title"] == "Player's Handbook"
|
|
assert "status" in docs[0]
|
|
|
|
|
|
def test_delete_document_removes_record(client, test_db):
|
|
doc_id = _add_doc(test_db, "Monster Manual", "/books/mm.pdf")
|
|
resp = client.delete(f"/api/library/{doc_id}")
|
|
assert resp.status_code == 204
|
|
resp2 = client.get("/api/library")
|
|
assert resp2.json() == []
|
|
|
|
|
|
def test_delete_nonexistent_returns_404(client):
|
|
resp = client.delete("/api/library/does-not-exist")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
def test_reshelve_returns_task_id(client, test_db, tmp_path):
|
|
pdf_path = str(tmp_path / "books" / "test.pdf")
|
|
open(pdf_path, "wb").write(b"%PDF-1.4")
|
|
doc_id = _add_doc(test_db, "Test Book", pdf_path)
|
|
resp = client.post(f"/api/library/{doc_id}/reshelve")
|
|
assert resp.status_code == 202
|
|
assert "task_id" in resp.json()
|
|
|
|
|
|
def test_reshelve_updates_status_to_processing(client, test_db, tmp_path):
|
|
from pathlib import Path
|
|
pdf_path = str(tmp_path / "books" / "dm_guide.pdf")
|
|
Path(pdf_path).write_bytes(b"%PDF-1.4 empty fixture")
|
|
doc_id = _add_doc(test_db, "DM Guide", pdf_path)
|
|
|
|
resp = client.post(f"/api/library/{doc_id}/reshelve")
|
|
assert resp.status_code == 202
|
|
|
|
# Document should be in processing state (or beyond if stub shelve ran instantly)
|
|
status_resp = client.get(f"/api/library/{doc_id}/status")
|
|
assert status_resp.json()["status"] in ("processing", "error", "ready")
|
|
|
|
|
|
def _add_chunks(db_path: str, doc_id: str, count: int) -> None:
|
|
conn = sqlite3.connect(db_path)
|
|
for i in range(count):
|
|
conn.execute(
|
|
"INSERT INTO page_chunks(doc_id, page_number, text, source, word_count) VALUES (?,?,?,?,?)",
|
|
[doc_id, i + 1, f"Page {i + 1} text content.", "text_layer", 4],
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
def test_sample_chunks_empty_returns_empty(client):
|
|
resp = client.get("/api/library/sample-chunks")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
def test_sample_chunks_returns_fields(client, test_db):
|
|
doc_id = _add_doc(test_db, "Monster Manual", "/books/mm.pdf")
|
|
_add_chunks(test_db, doc_id, 5)
|
|
resp = client.get("/api/library/sample-chunks?limit=3")
|
|
assert resp.status_code == 200
|
|
chunks = resp.json()
|
|
assert len(chunks) == 3
|
|
for c in chunks:
|
|
assert {"chunk_id", "doc_id", "page_number", "text"} == set(c.keys())
|
|
assert c["doc_id"] == doc_id
|
|
|
|
|
|
def test_sample_chunks_limit_capped_at_200(client, test_db):
|
|
doc_id = _add_doc(test_db, "Big Book", "/books/big.pdf")
|
|
_add_chunks(test_db, doc_id, 10)
|
|
resp = client.get("/api/library/sample-chunks?limit=9999")
|
|
assert resp.status_code == 200
|
|
assert len(resp.json()) <= 200
|