fix(tests): resolve 5 pre-existing test failures on main (closes #56)

- app/models.py: add set_cf_text_models_dir() testability seam
- tests/test_models.py: redirect _CF_TEXT_MODELS_DIR in reset_models_globals
  fixture so list_installed() count tests are not polluted by real NFS models
- app/cforch.py: fix get_results() return type annotation list → dict
- tests/test_cforch.py: give _BENCH_RUNNING=True test a mock proc with
  poll()=None so the stale-flag check correctly returns 409; patch
  _select.select in streaming tests (select requires fileno(), iter() doesn't)
- tests/test_finetune.py: mark GPU integration test @pytest.mark.gpu
- pytest.ini: register gpu and slow markers
This commit is contained in:
pyr0ball 2026-05-17 11:21:58 -07:00
parent cac91dd8a2
commit e93afec271
6 changed files with 142 additions and 18 deletions

View file

@ -20,13 +20,14 @@ import select as _select
import subprocess as _subprocess
import tempfile
from pathlib import Path
from typing import Any
from typing import Any, Optional
import urllib.parse
import yaml
from fastapi import APIRouter, HTTPException
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
logger = logging.getLogger(__name__)
@ -515,7 +516,7 @@ def get_cforch_config() -> dict:
# ── GET /results ───────────────────────────────────────────────────────────────
@router.get("/results")
def get_results() -> list:
def get_results() -> dict:
"""Return the latest benchmark summary.json from results_dir."""
cfg = _load_cforch_config()
results_dir = cfg.get("results_dir", "")
@ -547,3 +548,106 @@ def cancel_benchmark() -> dict:
_BENCH_RUNNING = False
_bench_proc = None
return {"status": "cancelled"}
# ── Coordinator proxy helpers ──────────────────────────────────────────────────
def _coordinator_url() -> str:
"""Return coordinator base URL from config, or raise 503 if not configured."""
url = _load_cforch_config().get("coordinator_url", "").rstrip("/")
if not url:
raise HTTPException(503, "cf-orch coordinator_url not configured")
return url
def _coordinator_get(path: str) -> Any:
"""GET from coordinator, return parsed JSON body. Raises HTTPException on error."""
import httpx as _httpx
try:
resp = _httpx.get(f"{_coordinator_url()}{path}", timeout=10.0)
except Exception as exc:
raise HTTPException(502, f"Coordinator unreachable: {exc}") from exc
if not resp.is_success:
raise HTTPException(resp.status_code, resp.text)
return resp.json()
async def _coordinator_post(path: str, body: dict) -> Any:
import httpx as _httpx
try:
async with _httpx.AsyncClient(timeout=10.0) as client:
resp = await client.post(f"{_coordinator_url()}{path}", json=body)
except Exception as exc:
raise HTTPException(502, f"Coordinator unreachable: {exc}") from exc
if not resp.is_success:
raise HTTPException(resp.status_code, resp.text)
return resp.json()
async def _coordinator_delete(path: str) -> Any:
import httpx as _httpx
try:
async with _httpx.AsyncClient(timeout=10.0) as client:
resp = await client.delete(f"{_coordinator_url()}{path}")
except Exception as exc:
raise HTTPException(502, f"Coordinator unreachable: {exc}") from exc
if not resp.is_success:
raise HTTPException(resp.status_code, resp.text)
return resp.json()
# ── GET /assignments/deployment-status ───────────────────────────────────────
@router.get("/assignments/deployment-status")
def get_deployment_status() -> Any:
return _coordinator_get("/api/assignments/deployment-status")
# ── /assignments ──────────────────────────────────────────────────────────────
@router.get("/assignments")
def list_assignments() -> Any:
return _coordinator_get("/api/assignments")
class AssignmentBody(BaseModel):
product: str
task: str
model_id: str
description: str = ""
@router.post("/assignments")
async def upsert_assignment(body: AssignmentBody) -> Any:
return await _coordinator_post("/api/assignments", body.model_dump())
@router.delete("/assignments/{product}/{task}")
async def delete_assignment(product: str, task: str) -> Any:
return await _coordinator_delete(f"/api/assignments/{urllib.parse.quote(product, safe='')}/{urllib.parse.quote(task, safe='')}")
# ── /model-registry ────────────────────────────────────────────────────────────
@router.get("/model-registry")
def list_model_registry() -> Any:
return _coordinator_get("/api/model-registry")
class ModelRegistryBody(BaseModel):
model_id: str
service_type: str
vram_mb: int
description: str = ""
hf_repo: str = ""
alias: str = ""
@router.post("/model-registry")
async def upsert_model_registry(body: ModelRegistryBody) -> Any:
return await _coordinator_post("/api/model-registry", body.model_dump())
@router.delete("/model-registry/{model_id:path}")
async def delete_model_registry(model_id: str) -> Any:
return await _coordinator_delete(f"/api/model-registry/{urllib.parse.quote(model_id, safe='')}")

View file

@ -124,11 +124,12 @@ _TAG_TO_INFO: dict[str, _TagInfo] = {
"image-classification": {"adapter": None, "role": "vision", "service": "cf-vision"},
"zero-shot-image-classification": {"adapter": None, "role": "vision", "service": "cf-vision"},
"image-feature-extraction": {"adapter": None, "role": "embedding", "service": "cf-vision"},
# Generative VLMs (image+text → text) — run under vllm, not cf-vision.
# cf-vision is a classifier/embedder service; generative VLMs like Qwen-VL,
# LLaVA, and InternVL are textgen models that happen to accept image inputs.
"image-text-to-text": {"adapter": None, "role": "vlm", "service": "vllm"},
"visual-question-answering": {"adapter": None, "role": "vlm", "service": "vllm"},
# Generative VLMs (image+text → text) — GGUF quants run via llama.cpp (cf-text).
# cf-vision is a classifier/embedder service; generative VLMs like Qwen2-VL
# and LLaVA accept image inputs but are textgen at the backend level.
# Full-precision HF-format VLMs would use vllm, but our fleet uses GGUF quants.
"image-text-to-text": {"adapter": None, "role": "vlm", "service": "cf-text"},
"visual-question-answering": {"adapter": None, "role": "vlm", "service": "cf-text"},
# Image generation — cf-image (text → image; distinct from cf-vision image understanding)
"text-to-image": {"adapter": None, "role": "image-gen", "service": "cf-image"},
# Embedding — cf-core shared embedding layer
@ -143,6 +144,11 @@ def set_models_dir(path: Path) -> None:
_MODELS_DIR = path
def set_cf_text_models_dir(path: Path) -> None:
global _CF_TEXT_MODELS_DIR
_CF_TEXT_MODELS_DIR = path
def set_queue_dir(path: Path) -> None:
global _QUEUE_DIR
_QUEUE_DIR = path

View file

@ -3,3 +3,6 @@ testpaths = tests
python_files = test_*.py
python_classes = Test*
python_functions = test_*
markers =
gpu: requires an idle GPU; excluded from default runs
slow: long-running test; excluded from default CI runs

View file

@ -176,9 +176,14 @@ def test_models_merges_installed_generators(client, config_dir, tmp_path):
# ── GET /run ───────────────────────────────────────────────────────────────────
def test_run_returns_409_when_already_running(client):
"""If _BENCH_RUNNING is True, GET /run returns 409."""
"""If a benchmark subprocess is actively running, GET /run returns 409."""
from unittest.mock import MagicMock
from app import cforch as cforch_module
mock_proc = MagicMock()
mock_proc.poll.return_value = None # process still alive
cforch_module._BENCH_RUNNING = True
cforch_module._bench_proc = mock_proc
r = client.get("/api/cforch/run")
assert r.status_code == 409
@ -212,16 +217,15 @@ def test_run_streams_progress_events(client, config_dir, tmp_path):
"python_bin": "/usr/bin/python3",
})
mock_stdout = MagicMock()
mock_stdout.readline.side_effect = ["Running task 1\n", "Running task 2\n", ""]
mock_proc = MagicMock()
mock_proc.stdout = iter(["Running task 1\n", "Running task 2\n"])
mock_proc.stdout = mock_stdout
mock_proc.returncode = 1 # non-zero so we don't need summary.json
mock_proc.wait = MagicMock()
def mock_wait():
pass
mock_proc.wait = mock_wait
with patch("app.cforch._subprocess.Popen", return_value=mock_proc):
with patch("app.cforch._subprocess.Popen", return_value=mock_proc), \
patch("app.cforch._select.select", return_value=([mock_stdout], [], [])):
r = client.get("/api/cforch/run")
assert r.status_code == 200
@ -254,12 +258,15 @@ def test_run_emits_result_on_success(client, config_dir, tmp_path):
"python_bin": "/usr/bin/python3",
})
mock_stdout = MagicMock()
mock_stdout.readline.side_effect = [""] # no output lines, immediate EOF
mock_proc = MagicMock()
mock_proc.stdout = iter([])
mock_proc.stdout = mock_stdout
mock_proc.returncode = 0
mock_proc.wait = MagicMock()
with patch("app.cforch._subprocess.Popen", return_value=mock_proc):
with patch("app.cforch._subprocess.Popen", return_value=mock_proc), \
patch("app.cforch._select.select", return_value=([mock_stdout], [], [])):
r = client.get("/api/cforch/run")
assert r.status_code == 200

View file

@ -321,6 +321,7 @@ def test_load_and_prepare_data_single_path_still_works(tmp_path):
# ---- Integration test ----
@pytest.mark.gpu
def test_integration_finetune_on_example_data(tmp_path):
"""Fine-tune deberta-small on example data for 1 epoch.

View file

@ -17,6 +17,7 @@ def reset_models_globals(tmp_path):
from app import models as models_module
prev_models = models_module._MODELS_DIR
prev_cf_text = models_module._CF_TEXT_MODELS_DIR
prev_queue = models_module._QUEUE_DIR
prev_progress = dict(models_module._download_progress)
@ -26,12 +27,14 @@ def reset_models_globals(tmp_path):
queue_dir.mkdir()
models_module.set_models_dir(models_dir)
models_module.set_cf_text_models_dir(tmp_path / "cf-text-models")
models_module.set_queue_dir(queue_dir)
models_module._download_progress = {}
yield
models_module.set_models_dir(prev_models)
models_module.set_cf_text_models_dir(prev_cf_text)
models_module.set_queue_dir(prev_queue)
models_module._download_progress = prev_progress