pyr0ball
d39cfbd87a
Extends the shelve pipeline to cover spreadsheets, closing the Excel gap called out in the PR's original "known gaps" list — Windchill/DocPortal corpora commonly include parts lists and spec sheets as spreadsheets, not just prose documents. - scripts/shelve_xlsx.py — openpyxl, chunked by sheet with row-window splitting for large sheets (header row repeated in every window so each chunk stays self-describing for retrieval). - scripts/shelve_ods.py — same chunking strategy via odfpy (already a dependency from ODT support), OpenDocumentSpreadsheet's Table/TableRow/ TableCell. - scripts/shelve_numbers.py — converts via headless LibreOffice to XLSX and delegates to shelve_xlsx, mirroring shelve_pages.py's pattern for .pages. Adds libreoffice-calc to the Docker image alongside the existing libreoffice-writer. - Upload button text changed from an ever-growing format list to "Upload Document or Spreadsheet" — the Supported Formats table in README/docs is now the source of truth for the full list. - 13 new tests (XLSX, ODS, Numbers); full suite (85 tests) passing. Manually verified via Playwright against an isolated test instance: XLSX and ODS both upload, shelve to "ready", and store correctly row-serialized, header-repeated chunks (confirmed via sample-chunks). BM25 search against a 2-chunk toy corpus returned no hits for terms split 1-vs-1 across the two chunks — traced to Okapi BM25's IDF formula giving an exact 0 for terms in exactly half a tiny corpus (log((N-n+0.5)/(n+0.5)) = log(1.0) = 0, filtered by `score <= 0`), not a defect in the new shelvers. The earlier DOCX/ODT/PDF Playwright pass (5 chunks total) diluted this enough to return real results.
118 lines
4.1 KiB
Python
118 lines
4.1 KiB
Python
# tests/test_shelve_numbers.py
|
|
"""Unit tests for scripts/shelve_numbers.py.
|
|
|
|
soffice isn't available in the test environment, so _convert_to_xlsx is
|
|
mocked to copy a pre-built XLSX fixture into the expected output path —
|
|
everything downstream (extraction/chunking/storage) runs for real via
|
|
scripts.shelve_xlsx.run.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from scripts.shelve_numbers import run
|
|
|
|
|
|
@pytest.fixture
|
|
def shelve_db(tmp_path) -> tuple[str, str]:
|
|
db_path = str(tmp_path / "test.db")
|
|
schema = Path("migrations/001_initial_schema.sql").read_text()
|
|
conn = sqlite3.connect(db_path)
|
|
conn.executescript(schema)
|
|
conn.execute(
|
|
"INSERT INTO documents(id, title, file_path, status) VALUES ('d1','Test','test.numbers','pending')"
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
vec_db_path = str(tmp_path / "vecs.db")
|
|
return db_path, vec_db_path
|
|
|
|
|
|
def _make_fixture_xlsx(path: Path) -> None:
|
|
import openpyxl
|
|
|
|
wb = openpyxl.Workbook()
|
|
ws = wb.active
|
|
ws.title = "Config"
|
|
ws.append(["Field", "Value"])
|
|
ws.append(["Default IP", "192.168.1.50"])
|
|
wb.save(str(path))
|
|
|
|
|
|
def _fake_soffice_convert(fixture_xlsx: Path):
|
|
def _fake_run(cmd, **kwargs):
|
|
out_dir = Path(cmd[cmd.index("--outdir") + 1])
|
|
numbers_path = Path(cmd[-1])
|
|
shutil.copy(fixture_xlsx, out_dir / f"{numbers_path.stem}.xlsx")
|
|
return subprocess.CompletedProcess(cmd, returncode=0, stdout="", stderr="")
|
|
|
|
return _fake_run
|
|
|
|
|
|
def test_shelve_numbers_converts_and_shelves_successfully(shelve_db, tmp_path):
|
|
db_path, vec_db_path = shelve_db
|
|
numbers_path = tmp_path / "test.numbers"
|
|
numbers_path.write_bytes(b"not a real numbers bundle, just needs to exist")
|
|
|
|
fixture_xlsx = tmp_path / "fixture.xlsx"
|
|
_make_fixture_xlsx(fixture_xlsx)
|
|
|
|
with patch("scripts.shelve_numbers.subprocess.run", side_effect=_fake_soffice_convert(fixture_xlsx)):
|
|
run(doc_id="d1", file_path=str(numbers_path), db_path=db_path, vec_db_path=vec_db_path)
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
row = conn.execute("SELECT status, page_count FROM documents WHERE id='d1'").fetchone()
|
|
rows = conn.execute(
|
|
"SELECT text FROM page_chunks WHERE doc_id='d1' ORDER BY page_number"
|
|
).fetchall()
|
|
conn.close()
|
|
assert row[0] == "ready"
|
|
assert row[1] == 1
|
|
assert "Default IP | 192.168.1.50" in rows[0][0]
|
|
|
|
|
|
def test_shelve_numbers_sets_error_status_on_conversion_failure(shelve_db, tmp_path):
|
|
db_path, vec_db_path = shelve_db
|
|
numbers_path = tmp_path / "test.numbers"
|
|
numbers_path.write_bytes(b"not a real numbers bundle")
|
|
|
|
fake_result = subprocess.CompletedProcess(
|
|
args=[], returncode=1, stdout="", stderr="soffice: unsupported document format"
|
|
)
|
|
with patch("scripts.shelve_numbers.subprocess.run", return_value=fake_result):
|
|
with pytest.raises(RuntimeError, match="LibreOffice conversion failed"):
|
|
run(doc_id="d1", file_path=str(numbers_path), db_path=db_path, vec_db_path=vec_db_path)
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
row = conn.execute("SELECT status, error_msg FROM documents WHERE id='d1'").fetchone()
|
|
conn.close()
|
|
assert row[0] == "error"
|
|
assert "unsupported document format" in row[1]
|
|
|
|
|
|
def test_shelve_numbers_cleans_up_temp_dir(shelve_db, tmp_path):
|
|
db_path, vec_db_path = shelve_db
|
|
numbers_path = tmp_path / "test.numbers"
|
|
numbers_path.write_bytes(b"not a real numbers bundle")
|
|
|
|
fixture_xlsx = tmp_path / "fixture.xlsx"
|
|
_make_fixture_xlsx(fixture_xlsx)
|
|
|
|
captured_tmp_dirs: list[str] = []
|
|
real_convert = _fake_soffice_convert(fixture_xlsx)
|
|
|
|
def _tracking_run(cmd, **kwargs):
|
|
captured_tmp_dirs.append(cmd[cmd.index("--outdir") + 1])
|
|
return real_convert(cmd, **kwargs)
|
|
|
|
with patch("scripts.shelve_numbers.subprocess.run", side_effect=_tracking_run):
|
|
run(doc_id="d1", file_path=str(numbers_path), db_path=db_path, vec_db_path=vec_db_path)
|
|
|
|
assert captured_tmp_dirs
|
|
assert not Path(captured_tmp_dirs[0]).exists(), "temp conversion dir should be cleaned up"
|