Circuit-Forge/kiwi
3
0
Fork 0
Code Issues 14 Pull requests Projects Releases 3 Packages Wiki Activity Actions 4

feat: visual label capture for unenriched barcodes (kiwi#79)

Browse source 
When a barcode scan finds no product in FDC/OFF, paid-tier users now see a
"Capture label" offer instead of a dead-end "add manually" prompt.

Backend:
- Migration 036: captured_products local cache table (keyed by barcode,
  UPSERT on conflict so re-capture refreshes rather than errors)
- store.get_captured_product / save_captured_product (with JSON decode for
  ingredient_names and allergens)
- app/services/label_capture.py: wraps cf-core VisionRouter (caption API);
  graceful fallback to zero-confidence mock when stub/error; JSON fence
  stripping; confidence clamped to [0,1]; KIWI_LABEL_CAPTURE_MOCK=1 for tests
- New schemas: LabelCaptureResponse, LabelConfirmRequest, LabelConfirmResponse
- POST /inventory/scan/label-capture — image to extraction (paid+ gate, 403)
- POST /inventory/scan/label-confirm — save confirmed product + optional
  inventory add
- Both scan endpoints now: check captured_products cache before FDC/OFF;
  set needs_visual_capture=True for gap products on paid tier; BarcodeScanResult
  gains needs_visual_capture field
- visual_label_capture feature gate added to tiers.py (paid)

Tests: 42 new tests (service, store/migration, API endpoints) — 367 total passing

Frontend:
- InventoryList.vue: capturePhase state machine (offer => uploading => reviewing)
- Offer card appears after scan gap (calm UX: no urgency, Discard always visible)
- Review form: pre-populated from extraction; amber label highlights for
  unread fields (confidence < 0.7); comma-separated ingredients/allergens
- api.ts: LabelCaptureResult + LabelConfirmRequest types; captureLabelPhoto()
  and confirmLabelCapture() API methods
This commit is contained in:
pyr0ball 2026-04-24 17:57:25 -07:00
parent 3463aa1e17
commit 17e62c451f
12 changed files with 1421 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

206
app/api/endpoints/inventory.py
View file

@ -37,6 +37,7 @@ from app.models.schemas.inventory import (
TagCreate,
TagResponse,
)
from app.models.schemas.label_capture import LabelConfirmRequest
router = APIRouter()
@ -349,6 +350,31 @@ class BarcodeScanTextRequest(BaseModel):
auto_add_to_inventory: bool = True
def _captured_to_product_info(row: dict) -> dict:
"""Convert a captured_products row to the product_info dict shape used by
the barcode scan flow (mirrors what OpenFoodFactsService returns)."""
macros: dict = {}
for field in ("calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g",
"fiber_g", "protein_g", "sodium_mg", "serving_size_g"):
if row.get(field) is not None:
macros[field] = row[field]
return {
"name": row.get("product_name") or row.get("barcode", "Unknown Product"),
"brand": row.get("brand"),
"category": None,
"nutrition_data": macros,
"ingredient_names": row.get("ingredient_names") or [],
"allergens": row.get("allergens") or [],
"source": "visual_capture",
}
def _gap_message(tier: str, has_visual_capture: bool) -> str:
if has_visual_capture:
return "We couldn't find this product. Photograph the nutrition label to add it."
return "Not found in any product database — add manually"
@router.post("/scan/text", response_model=BarcodeScanResponse)
async def scan_barcode_text(
body: BarcodeScanTextRequest,
@ -359,10 +385,21 @@ async def scan_barcode_text(
log.info("scan auth=%s tier=%s barcode=%r", _auth_label(session.user_id), session.tier, body.barcode)
from app.services.openfoodfacts import OpenFoodFactsService
from app.services.expiration_predictor import ExpirationPredictor
from app.tiers import can_use
off = OpenFoodFactsService()
predictor = ExpirationPredictor()
has_visual_capture = can_use("visual_label_capture", session.tier, session.has_byok)
# 1. Check local captured-products cache before hitting FDC/OFF
cached = await asyncio.to_thread(store.get_captured_product, body.barcode)
if cached and cached.get("confirmed_by_user"):
product_info: dict | None = _captured_to_product_info(cached)
product_source = "visual_capture"
else:
off = OpenFoodFactsService()
product_info = await off.lookup_product(body.barcode)
product_source = "openfoodfacts"
inventory_item = None
if product_info and body.auto_add_to_inventory:
@ -373,7 +410,7 @@ async def scan_barcode_text(
brand=product_info.get("brand"),
category=product_info.get("category"),
nutrition_data=product_info.get("nutrition_data", {}),
source="openfoodfacts",
source=product_source,
source_data=product_info,
)
exp = predictor.predict_expiration(
@ -399,6 +436,7 @@ async def scan_barcode_text(
result_product = None
product_found = product_info is not None
needs_capture = not product_found and has_visual_capture
return BarcodeScanResponse(
success=True,
barcodes_found=1,
@ -408,8 +446,9 @@ async def scan_barcode_text(
"product": result_product,
"inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None,
"added_to_inventory": inventory_item is not None,
"needs_manual_entry": not product_found,
"message": "Added to inventory" if inventory_item else "Not found in any product database — add manually",
"needs_manual_entry": not product_found and not needs_capture,
"needs_visual_capture": needs_capture,
"message": "Added to inventory" if inventory_item else _gap_message(session.tier, needs_capture),
}],
message="Barcode processed",
)
@ -426,6 +465,9 @@ async def scan_barcode_image(
):
"""Scan a barcode from an uploaded image. Requires Phase 2 scanner integration."""
log.info("scan_image auth=%s tier=%s", _auth_label(session.user_id), session.tier)
from app.tiers import can_use
has_visual_capture = can_use("visual_label_capture", session.tier, session.has_byok)
temp_dir = Path("/tmp/kiwi_barcode_scans")
temp_dir.mkdir(parents=True, exist_ok=True)
temp_file = temp_dir / f"{uuid.uuid4()}_{file.filename}"
@ -448,7 +490,16 @@ async def scan_barcode_image(
results = []
for bc in barcodes:
code = bc["data"]
# Check local visual-capture cache before hitting FDC/OFF
cached = await asyncio.to_thread(store.get_captured_product, code)
if cached and cached.get("confirmed_by_user"):
product_info: dict | None = _captured_to_product_info(cached)
product_source = "visual_capture"
else:
product_info = await off.lookup_product(code)
product_source = "openfoodfacts"
inventory_item = None
if product_info and auto_add_to_inventory:
product, _ = await asyncio.to_thread(
@ -458,7 +509,7 @@ async def scan_barcode_image(
brand=product_info.get("brand"),
category=product_info.get("category"),
nutrition_data=product_info.get("nutrition_data", {}),
source="openfoodfacts",
source=product_source,
source_data=product_info,
)
exp = predictor.predict_expiration(
@ -478,13 +529,17 @@ async def scan_barcode_image(
expiration_date=str(exp) if exp else None,
source="barcode_scan",
)
product_found = product_info is not None
needs_capture = not product_found and has_visual_capture
results.append({
"barcode": code,
"barcode_type": bc.get("type", "unknown"),
"product": ProductResponse.model_validate(product) if product_info else None,
"product": ProductResponse.model_validate(product_info) if product_info else None,
"inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None,
"added_to_inventory": inventory_item is not None,
"message": "Added to inventory" if inventory_item else "Barcode scanned",
"needs_manual_entry": not product_found and not needs_capture,
"needs_visual_capture": needs_capture,
"message": "Added to inventory" if inventory_item else _gap_message(session.tier, needs_capture),
})
return BarcodeScanResponse(
success=True, barcodes_found=len(barcodes), results=results,
@ -495,6 +550,143 @@ async def scan_barcode_image(
temp_file.unlink()
# ── Visual label capture (kiwi#79) ────────────────────────────────────────────
@router.post("/scan/label-capture")
async def capture_nutrition_label(
file: UploadFile = File(...),
barcode: str = Form(...),
store: Store = Depends(get_store),
session: CloudUser = Depends(get_session),
):
"""Photograph a nutrition label for an unenriched product (paid tier).
Sends the image to the vision model and returns structured nutrition data
for user review. Fields extracted with confidence < 0.7 should be
highlighted in amber in the UI.
"""
from app.tiers import can_use
from app.models.schemas.label_capture import LabelCaptureResponse
from app.services.label_capture import extract_label, needs_review as _needs_review
if not can_use("visual_label_capture", session.tier, session.has_byok):
raise HTTPException(status_code=403, detail="Visual label capture requires a Paid tier or higher.")
log.info("label_capture tier=%s barcode=%r", session.tier, barcode)
image_bytes = await file.read()
extraction = await asyncio.to_thread(extract_label, image_bytes)
return LabelCaptureResponse(
barcode=barcode,
product_name=extraction.get("product_name"),
brand=extraction.get("brand"),
serving_size_g=extraction.get("serving_size_g"),
calories=extraction.get("calories"),
fat_g=extraction.get("fat_g"),
saturated_fat_g=extraction.get("saturated_fat_g"),
carbs_g=extraction.get("carbs_g"),
sugar_g=extraction.get("sugar_g"),
fiber_g=extraction.get("fiber_g"),
protein_g=extraction.get("protein_g"),
sodium_mg=extraction.get("sodium_mg"),
ingredient_names=extraction.get("ingredient_names") or [],
allergens=extraction.get("allergens") or [],
confidence=extraction.get("confidence", 0.0),
needs_review=_needs_review(extraction),
)
@router.post("/scan/label-confirm")
async def confirm_nutrition_label(
body: LabelConfirmRequest,
store: Store = Depends(get_store),
session: CloudUser = Depends(get_session),
):
"""Confirm and save a user-reviewed label extraction.
Saves the product to the local cache so future scans of the same barcode
resolve instantly without another capture. Optionally adds the item to
the user's inventory.
"""
from app.tiers import can_use
from app.models.schemas.label_capture import LabelConfirmResponse
from app.services.expiration_predictor import ExpirationPredictor
if not can_use("visual_label_capture", session.tier, session.has_byok):
raise HTTPException(status_code=403, detail="Visual label capture requires a Paid tier or higher.")
log.info("label_confirm tier=%s barcode=%r", session.tier, body.barcode)
# Persist to local visual-capture cache
await asyncio.to_thread(
store.save_captured_product,
body.barcode,
product_name=body.product_name,
brand=body.brand,
serving_size_g=body.serving_size_g,
calories=body.calories,
fat_g=body.fat_g,
saturated_fat_g=body.saturated_fat_g,
carbs_g=body.carbs_g,
sugar_g=body.sugar_g,
fiber_g=body.fiber_g,
protein_g=body.protein_g,
sodium_mg=body.sodium_mg,
ingredient_names=body.ingredient_names,
allergens=body.allergens,
confidence=body.confidence,
confirmed_by_user=True,
)
product_id: int | None = None
inventory_item_id: int | None = None
if body.auto_add:
predictor = ExpirationPredictor()
nutrition = {}
for field in ("calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g",
"fiber_g", "protein_g", "sodium_mg", "serving_size_g"):
val = getattr(body, field, None)
if val is not None:
nutrition[field] = val
product, _ = await asyncio.to_thread(
store.get_or_create_product,
body.product_name or body.barcode,
body.barcode,
brand=body.brand,
category=None,
nutrition_data=nutrition,
source="visual_capture",
source_data={},
)
product_id = product["id"]
exp = predictor.predict_expiration(
"",
body.location,
product_name=body.product_name or body.barcode,
tier=session.tier,
has_byok=session.has_byok,
)
inv_item = await asyncio.to_thread(
store.add_inventory_item,
product_id, body.location,
quantity=body.quantity,
unit="count",
expiration_date=str(exp) if exp else None,
source="visual_capture",
)
inventory_item_id = inv_item["id"]
return LabelConfirmResponse(
ok=True,
barcode=body.barcode,
product_id=product_id,
inventory_item_id=inventory_item_id,
message="Product saved" + (" and added to inventory" if body.auto_add else ""),
)
# ── Tags ──────────────────────────────────────────────────────────────────────
@router.post("/tags", response_model=TagResponse, status_code=status.HTTP_201_CREATED)

26
app/db/migrations/036_captured_products.sql Normal file
View file

@ -0,0 +1,26 @@
-- Migration 036: captured_products local cache
-- Products captured via visual label scanning (kiwi#79).
-- Keyed by barcode; checked before FDC/OFF on future scans so each product
-- is only captured once per device.
CREATE TABLE IF NOT EXISTS captured_products (
id INTEGER PRIMARY KEY AUTOINCREMENT,
barcode TEXT UNIQUE NOT NULL,
product_name TEXT,
brand TEXT,
serving_size_g REAL,
calories REAL,
fat_g REAL,
saturated_fat_g REAL,
carbs_g REAL,
sugar_g REAL,
fiber_g REAL,
protein_g REAL,
sodium_mg REAL,
ingredient_names TEXT NOT NULL DEFAULT '[]', -- JSON array
allergens TEXT NOT NULL DEFAULT '[]', -- JSON array
confidence REAL,
source TEXT NOT NULL DEFAULT 'visual_capture',
captured_at TEXT NOT NULL DEFAULT (datetime('now')),
confirmed_by_user INTEGER NOT NULL DEFAULT 0
);

74
app/db/store.py
View file

@ -60,7 +60,9 @@ class Store:
# saved recipe columns
"style_tags",
# meal plan columns
"meal_types"):
"meal_types",
# captured_products columns
"allergens"):
if key in d and isinstance(d[key], str):
try:
d[key] = json.loads(d[key])
@ -1639,3 +1641,73 @@ class Store:
cur = self.conn.execute("DELETE FROM shopping_list_items")
self.conn.commit()
return cur.rowcount
# ── Captured products (visual label cache) ────────────────────────────────
def get_captured_product(self, barcode: str) -> dict | None:
"""Look up a locally-captured product by barcode.
Returns the row dict (ingredient_names and allergens already decoded as
lists) or None if the barcode has not been captured yet.
"""
return self._fetch_one(
"SELECT * FROM captured_products WHERE barcode = ?", (barcode,)
)
def save_captured_product(
self,
barcode: str,
*,
product_name: str | None = None,
brand: str | None = None,
serving_size_g: float | None = None,
calories: float | None = None,
fat_g: float | None = None,
saturated_fat_g: float | None = None,
carbs_g: float | None = None,
sugar_g: float | None = None,
fiber_g: float | None = None,
protein_g: float | None = None,
sodium_mg: float | None = None,
ingredient_names: list[str] | None = None,
allergens: list[str] | None = None,
confidence: float | None = None,
confirmed_by_user: bool = True,
source: str = "visual_capture",
) -> dict:
"""Insert or replace a captured product row, returning the saved dict."""
return self._insert_returning(
"""INSERT INTO captured_products
(barcode, product_name, brand, serving_size_g, calories,
fat_g, saturated_fat_g, carbs_g, sugar_g, fiber_g,
protein_g, sodium_mg, ingredient_names, allergens,
confidence, confirmed_by_user, source)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(barcode) DO UPDATE SET
product_name = excluded.product_name,
brand = excluded.brand,
serving_size_g = excluded.serving_size_g,
calories = excluded.calories,
fat_g = excluded.fat_g,
saturated_fat_g = excluded.saturated_fat_g,
carbs_g = excluded.carbs_g,
sugar_g = excluded.sugar_g,
fiber_g = excluded.fiber_g,
protein_g = excluded.protein_g,
sodium_mg = excluded.sodium_mg,
ingredient_names = excluded.ingredient_names,
allergens = excluded.allergens,
confidence = excluded.confidence,
confirmed_by_user = excluded.confirmed_by_user,
source = excluded.source,
captured_at = datetime('now')
RETURNING *""",
(
barcode, product_name, brand, serving_size_g, calories,
fat_g, saturated_fat_g, carbs_g, sugar_g, fiber_g,
protein_g, sodium_mg,
self._dump(ingredient_names or []),
self._dump(allergens or []),
confidence, 1 if confirmed_by_user else 0, source,
),
)

1
app/models/schemas/inventory.py
View file

@ -142,6 +142,7 @@ class BarcodeScanResult(BaseModel):
inventory_item: Optional[InventoryItemResponse]
added_to_inventory: bool
needs_manual_entry: bool = False
needs_visual_capture: bool = False # Paid tier offer when no product data found
message: str

59
app/models/schemas/label_capture.py Normal file
View file

@ -0,0 +1,59 @@
"""Pydantic schemas for visual label capture (kiwi#79)."""
from __future__ import annotations
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
class LabelCaptureResponse(BaseModel):
"""Extraction result returned after the user photographs a nutrition label."""
barcode: str
product_name: Optional[str] = None
brand: Optional[str] = None
serving_size_g: Optional[float] = None
calories: Optional[float] = None
fat_g: Optional[float] = None
saturated_fat_g: Optional[float] = None
carbs_g: Optional[float] = None
sugar_g: Optional[float] = None
fiber_g: Optional[float] = None
protein_g: Optional[float] = None
sodium_mg: Optional[float] = None
ingredient_names: List[str] = Field(default_factory=list)
allergens: List[str] = Field(default_factory=list)
confidence: float = 0.0
needs_review: bool = True # True when confidence < REVIEW_THRESHOLD
class LabelConfirmRequest(BaseModel):
"""User-confirmed extraction to save to the local product cache."""
barcode: str
product_name: Optional[str] = None
brand: Optional[str] = None
serving_size_g: Optional[float] = None
calories: Optional[float] = None
fat_g: Optional[float] = None
saturated_fat_g: Optional[float] = None
carbs_g: Optional[float] = None
sugar_g: Optional[float] = None
fiber_g: Optional[float] = None
protein_g: Optional[float] = None
sodium_mg: Optional[float] = None
ingredient_names: List[str] = Field(default_factory=list)
allergens: List[str] = Field(default_factory=list)
confidence: float = 0.0
# When True the confirmed product is also added to inventory
location: str = "pantry"
quantity: float = 1.0
auto_add: bool = True
class LabelConfirmResponse(BaseModel):
"""Result of confirming a captured product."""
ok: bool
barcode: str
product_id: Optional[int] = None
inventory_item_id: Optional[int] = None
message: str

140
app/services/label_capture.py Normal file
View file

@ -0,0 +1,140 @@
"""Visual label capture service for unenriched products (kiwi#79).
Wraps the cf-core VisionRouter to extract structured nutrition data from a
photographed nutrition facts panel. When the VisionRouter is not yet wired
(NotImplementedError) the service falls back to a mock extraction so the
barcode scan flow can be exercised end-to-end in development.
JSON contract returned by the vision model (and mock):
{
"product_name": str | null,
"brand": str | null,
"serving_size_g": number | null,
"calories": number | null,
"fat_g": number | null,
"saturated_fat_g": number | null,
"carbs_g": number | null,
"sugar_g": number | null,
"fiber_g": number | null,
"protein_g": number | null,
"sodium_mg": number | null,
"ingredient_names": [str],
"allergens": [str],
"confidence": number (0.0–1.0)
}
"""
from __future__ import annotations
import json
import logging
import os
from typing import Any
log = logging.getLogger(__name__)
# Confidence below this threshold surfaces amber highlights in the UI.
REVIEW_THRESHOLD = 0.7
_MOCK_EXTRACTION: dict[str, Any] = {
"product_name": "Unknown Product",
"brand": None,
"serving_size_g": None,
"calories": None,
"fat_g": None,
"saturated_fat_g": None,
"carbs_g": None,
"sugar_g": None,
"fiber_g": None,
"protein_g": None,
"sodium_mg": None,
"ingredient_names": [],
"allergens": [],
"confidence": 0.0,
}
_EXTRACTION_PROMPT = """You are reading a nutrition facts label photograph.
Extract the following fields as a JSON object with no extra text:
{
"product_name": <product name or null>,
"brand": <brand name or null>,
"serving_size_g": <serving size in grams as a number or null>,
"calories": <calories per serving as a number or null>,
"fat_g": <total fat grams or null>,
"saturated_fat_g": <saturated fat grams or null>,
"carbs_g": <total carbohydrates grams or null>,
"sugar_g": <sugars grams or null>,
"fiber_g": <dietary fiber grams or null>,
"protein_g": <protein grams or null>,
"sodium_mg": <sodium milligrams or null>,
"ingredient_names": [list of individual ingredients as strings],
"allergens": [list of allergens explicitly stated on label],
"confidence": <your confidence this extraction is correct, 0.0 to 1.0>
}
Use null for any field you cannot read clearly. Do not guess values.
Respond with JSON only."""
def extract_label(image_bytes: bytes) -> dict[str, Any]:
"""Run vision model extraction on raw label image bytes.
Returns a dict matching the nutrition JSON contract above.
Falls back to a zero-confidence mock if the VisionRouter is not yet
implemented (stub) or if the model returns unparseable output.
"""
# Allow unit tests to bypass the vision model entirely.
if os.environ.get("KIWI_LABEL_CAPTURE_MOCK") == "1":
log.debug("label_capture: mock mode active")
return dict(_MOCK_EXTRACTION)
try:
from circuitforge_core.vision import caption as vision_caption
result = vision_caption(image_bytes, prompt=_EXTRACTION_PROMPT)
raw = result.caption or ""
return _parse_extraction(raw)
except Exception as exc:
log.warning("label_capture: extraction failed (%s) — returning mock extraction", exc)
return dict(_MOCK_EXTRACTION)
def _parse_extraction(raw: str) -> dict[str, Any]:
"""Parse the JSON string returned by the vision model.
Strips markdown code fences if present. Validates required shape.
Returns the mock on any parse error.
"""
text = raw.strip()
if text.startswith("```"):
# Strip ```json ... ``` fences
lines = text.splitlines()
text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
try:
data = json.loads(text)
except json.JSONDecodeError as exc:
log.warning("label_capture: could not parse vision response: %s", exc)
return dict(_MOCK_EXTRACTION)
if not isinstance(data, dict):
log.warning("label_capture: vision response is not a dict")
return dict(_MOCK_EXTRACTION)
# Normalise list fields — model may return None instead of []
for list_key in ("ingredient_names", "allergens"):
if not isinstance(data.get(list_key), list):
data[list_key] = []
# Clamp confidence to [0, 1]
confidence = data.get("confidence")
if not isinstance(confidence, (int, float)):
confidence = 0.0
data["confidence"] = max(0.0, min(1.0, float(confidence)))
return data
def needs_review(extraction: dict[str, Any]) -> bool:
"""Return True when the extraction confidence is below REVIEW_THRESHOLD."""
return float(extraction.get("confidence", 0.0)) < REVIEW_THRESHOLD

1
app/tiers.py
View file

@ -44,6 +44,7 @@ KIWI_FEATURES: dict[str, str] = {
# Paid tier
"receipt_ocr": "paid", # BYOK-unlockable
"visual_label_capture": "paid", # Camera capture for unenriched barcodes (kiwi#79)
"recipe_suggestions": "paid", # BYOK-unlockable
"expiry_llm_matching": "paid", # BYOK-unlockable
"meal_planning": "free",

296
frontend/src/components/InventoryList.vue
View file

@ -138,6 +138,103 @@
</div>
</div>
<!-- Label Capture Panel (paid tier — appears after gap detection) -->
<div v-if="capturePhase !== null" class="label-capture-panel">
<!-- Offer phase -->
<div v-if="capturePhase === 'offer'" class="capture-offer">
<p class="capture-offer-text">We couldn't find this product. Photograph the nutrition label to add it.</p>
<div class="capture-offer-actions">
<button class="btn btn-primary" type="button" @click="triggerCaptureLabelInput">
Capture label
</button>
<button class="btn btn-ghost" type="button" @click="dismissCapture">
Skip
</button>
</div>
<input
ref="captureFileInput"
type="file"
accept="image/*"
capture="environment"
style="display: none"
@change="handleLabelPhotoSelect"
/>
</div>
<!-- Uploading / processing phase -->
<div v-else-if="capturePhase === 'uploading'" class="capture-processing">
<div class="loading-inline">
<div class="spinner spinner-sm"></div>
<span>Reading the label…</span>
</div>
</div>
<!-- Review phase -->
<div v-else-if="capturePhase === 'reviewing' && captureExtraction" class="capture-review">
<p class="capture-review-note">
Check the details below.
<span v-if="captureExtraction.needs_review" class="capture-review-low-conf">
Fields highlighted in amber weren't fully legible — please verify them.
</span>
</p>
<div class="form-row">
<div class="form-group">
<label class="form-label">Product name</label>
<input v-model="captureReview.product_name" type="text" class="form-input" placeholder="Product name" />
</div>
<div class="form-group">
<label class="form-label">Brand</label>
<input v-model="captureReview.brand" type="text" class="form-input" placeholder="Brand (optional)" />
</div>
</div>
<p class="form-section-label">Nutrition per serving</p>
<div class="capture-nutrition-grid">
<div
v-for="field in captureNutritionFields"
:key="field.key"
class="form-group"
>
<label
:class="['form-label', { 'capture-field-amber': captureExtraction.needs_review && captureExtraction[field.src as keyof typeof captureExtraction] == null }]"
>{{ field.label }}</label>
<input
v-model="captureReview[field.key as keyof typeof captureReview]"
type="number"
min="0"
step="0.1"
class="form-input"
:placeholder="field.unit"
/>
</div>
</div>
<div class="form-group" style="margin-top: var(--spacing-sm)">
<label class="form-label">Ingredients (comma-separated)</label>
<input v-model="captureReview.ingredients" type="text" class="form-input" placeholder="flour, water, salt…" />
</div>
<div class="form-group">
<label class="form-label">Allergens (comma-separated)</label>
<input v-model="captureReview.allergens" type="text" class="form-input" placeholder="wheat, milk…" />
</div>
<div class="capture-review-actions">
<button class="btn btn-primary" type="button" :disabled="captureLoading" @click="confirmCapture">
<span v-if="captureLoading"><div class="spinner spinner-sm"></div></span>
<span v-else>Looks good — save</span>
</button>
<button class="btn btn-ghost" type="button" @click="capturePhase = 'offer'">
Retake photo
</button>
<button class="btn btn-ghost" type="button" @click="dismissCapture">
Discard
</button>
</div>
</div>
</div>
<!-- Camera Scan Panel -->
<div v-if="scanMode === 'camera'" class="scan-panel">
<div class="upload-area" @click="triggerBarcodeInput">
@ -622,7 +719,7 @@ import { storeToRefs } from 'pinia'
import { useInventoryStore } from '../stores/inventory'
import { useSettingsStore } from '../stores/settings'
import { inventoryAPI } from '../services/api'
import type { InventoryItem } from '../services/api'
import type { InventoryItem, LabelCaptureResult } from '../services/api'
import { formatQuantity } from '../utils/units'
import EditItemModal from './EditItemModal.vue'
import ConfirmDialog from './ConfirmDialog.vue'
@ -684,6 +781,16 @@ function daysLabel(dateStr: string): string {
const scanMode = ref<'gun' | 'camera' | 'manual'>('gun')
// Options for button groups
// Label capture nutrition field descriptors used in the review form
const captureNutritionFields = [
{ key: 'calories', src: 'calories', label: 'Calories', unit: 'kcal' },
{ key: 'fat_g', src: 'fat_g', label: 'Total fat', unit: 'g' },
{ key: 'saturated_fat_g', src: 'saturated_fat_g', label: 'Saturated fat', unit: 'g' },
{ key: 'carbs_g', src: 'carbs_g', label: 'Carbs', unit: 'g' },
{ key: 'protein_g', src: 'protein_g', label: 'Protein', unit: 'g' },
{ key: 'sodium_mg', src: 'sodium_mg', label: 'Sodium', unit: 'mg' },
]
const locations = [
{ value: 'fridge', label: 'Fridge', icon: '🧊' },
{ value: 'freezer', label: 'Freezer', icon: '❄️' },
@ -780,6 +887,29 @@ const barcodeQuantity = ref(1)
const barcodeLoading = ref(false)
const barcodeResults = ref<Array<{ type: string; message: string }>>([])
// Label Capture Flow (kiwi#79)
type CapturePhase = 'offer' | 'uploading' | 'reviewing' | null
const capturePhase = ref<CapturePhase>(null)
const captureBarcode = ref('')
const captureLocation = ref('pantry')
const captureQuantity = ref(1)
const captureLoading = ref(false)
const captureFileInput = ref<HTMLInputElement | null>(null)
const captureExtraction = ref<LabelCaptureResult | null>(null)
// Editable review form — populated from extraction, user may correct fields
const captureReview = ref({
product_name: '',
brand: '',
calories: '' as string,
fat_g: '' as string,
saturated_fat_g: '' as string,
carbs_g: '' as string,
protein_g: '' as string,
sodium_mg: '' as string,
ingredients: '',
allergens: '',
})
// Manual Form
const manualForm = ref({
name: '',
@ -935,6 +1065,15 @@ async function handleScannerGunInput() {
message: `Added: ${productName}${productBrand} to ${scannerLocation.value}`,
})
await refreshItems()
} else if (item?.needs_visual_capture) {
captureBarcode.value = barcode
captureLocation.value = scannerLocation.value
captureQuantity.value = scannerQuantity.value
capturePhase.value = 'offer'
scannerResults.value.push({
type: 'info',
message: item.message,
})
} else if (item?.needs_manual_entry) {
// Barcode not found in any database — guide user to manual entry
scannerResults.value.push({
@ -1007,6 +1146,88 @@ async function handleBarcodeImageSelect(e: Event) {
}
}
// Label Capture Functions
function triggerCaptureLabelInput() {
captureFileInput.value?.click()
}
function dismissCapture() {
capturePhase.value = null
captureBarcode.value = ''
captureExtraction.value = null
}
async function handleLabelPhotoSelect(e: Event) {
const target = e.target as HTMLInputElement
const file = target.files?.[0]
if (!file) return
captureLoading.value = true
capturePhase.value = 'uploading'
try {
const result = await inventoryAPI.captureLabelPhoto(file, captureBarcode.value)
captureExtraction.value = result
// Pre-populate the review form with extracted values
captureReview.value = {
product_name: result.product_name || '',
brand: result.brand || '',
calories: result.calories != null ? String(result.calories) : '',
fat_g: result.fat_g != null ? String(result.fat_g) : '',
saturated_fat_g: result.saturated_fat_g != null ? String(result.saturated_fat_g) : '',
carbs_g: result.carbs_g != null ? String(result.carbs_g) : '',
protein_g: result.protein_g != null ? String(result.protein_g) : '',
sodium_mg: result.sodium_mg != null ? String(result.sodium_mg) : '',
ingredients: (result.ingredient_names || []).join(', '),
allergens: (result.allergens || []).join(', '),
}
capturePhase.value = 'reviewing'
} catch {
showToast('Could not read the label. Please try again or add manually.', 'error')
capturePhase.value = 'offer'
} finally {
captureLoading.value = false
if (target) target.value = ''
}
}
async function confirmCapture() {
if (!captureBarcode.value) return
captureLoading.value = true
try {
const toNum = (s: string) => s ? parseFloat(s) || null : null
const toList = (s: string) => s.split(',').map(x => x.trim()).filter(Boolean)
await inventoryAPI.confirmLabelCapture({
barcode: captureBarcode.value,
product_name: captureReview.value.product_name || null,
brand: captureReview.value.brand || null,
calories: toNum(captureReview.value.calories),
fat_g: toNum(captureReview.value.fat_g),
saturated_fat_g: toNum(captureReview.value.saturated_fat_g),
carbs_g: toNum(captureReview.value.carbs_g),
protein_g: toNum(captureReview.value.protein_g),
sodium_mg: toNum(captureReview.value.sodium_mg),
ingredient_names: toList(captureReview.value.ingredients),
allergens: toList(captureReview.value.allergens),
confidence: captureExtraction.value?.confidence ?? 0,
location: captureLocation.value,
quantity: captureQuantity.value,
auto_add: true,
})
const name = captureReview.value.product_name || 'item'
showToast(`${name} saved and added to ${captureLocation.value}`, 'success')
await refreshItems()
dismissCapture()
} catch {
showToast('Could not save. Please try again.', 'error')
} finally {
captureLoading.value = false
}
}
// Manual Add Functions
async function addManualItem() {
const { name, brand, quantity, unit, location, expirationDate } = manualForm.value
@ -1614,6 +1835,79 @@ function getItemClass(item: InventoryItem): string {
border: 1px solid var(--color-warning-border, #fcd34d);
}
/* ============================================
LABEL CAPTURE FLOW (kiwi#79)
============================================ */
.label-capture-panel {
margin: var(--spacing-md) 0;
padding: var(--spacing-md);
background: var(--color-surface);
border: 1px solid var(--color-border);
border-radius: var(--radius-lg);
}
.capture-offer-text {
font-size: var(--font-size-sm);
color: var(--color-text-secondary);
margin: 0 0 var(--spacing-md);
}
.capture-offer-actions {
display: flex;
gap: var(--spacing-sm);
flex-wrap: wrap;
}
.capture-processing {
display: flex;
justify-content: center;
padding: var(--spacing-md) 0;
}
.capture-review-note {
font-size: var(--font-size-sm);
color: var(--color-text-secondary);
margin: 0 0 var(--spacing-md);
}
.capture-review-low-conf {
color: var(--color-amber, #d97706);
font-size: var(--font-size-xs);
display: block;
margin-top: var(--spacing-xs);
}
.form-section-label {
font-size: var(--font-size-xs);
font-weight: 600;
color: var(--color-text-tertiary);
text-transform: uppercase;
letter-spacing: 0.05em;
margin: var(--spacing-md) 0 var(--spacing-sm);
}
.capture-nutrition-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(120px, 1fr));
gap: var(--spacing-sm);
}
/* Amber highlight for unread/low-confidence label fields */
.capture-field-amber {
color: var(--color-amber, #d97706);
}
.capture-field-amber + input {
border-color: var(--color-amber, #d97706);
}
.capture-review-actions {
display: flex;
gap: var(--spacing-sm);
flex-wrap: wrap;
margin-top: var(--spacing-md);
}
/* ============================================
EXPORT CARD
============================================ */

67
frontend/src/services/api.ts
View file

@ -111,9 +111,50 @@ export interface BarcodeScanResult {
inventory_item: InventoryItem | null
added_to_inventory: boolean
needs_manual_entry: boolean
needs_visual_capture: boolean
message: string
}
export interface LabelCaptureResult {
barcode: string
product_name: string | null
brand: string | null
serving_size_g: number | null
calories: number | null
fat_g: number | null
saturated_fat_g: number | null
carbs_g: number | null
sugar_g: number | null
fiber_g: number | null
protein_g: number | null
sodium_mg: number | null
ingredient_names: string[]
allergens: string[]
confidence: number
needs_review: boolean
}
export interface LabelConfirmRequest {
barcode: string
product_name?: string | null
brand?: string | null
serving_size_g?: number | null
calories?: number | null
fat_g?: number | null
saturated_fat_g?: number | null
carbs_g?: number | null
sugar_g?: number | null
fiber_g?: number | null
protein_g?: number | null
sodium_mg?: number | null
ingredient_names?: string[]
allergens?: string[]
confidence?: number
location?: string
quantity?: number
auto_add?: boolean
}
export interface BarcodeScanResponse {
success: boolean
barcodes_found: number
@ -344,6 +385,32 @@ export const inventoryAPI = {
})
return response.data
},
/**
* Upload a nutrition label photo for an unenriched barcode (paid tier).
* Returns extracted fields + confidence score for user review.
*/
async captureLabelPhoto(
file: File,
barcode: string
): Promise<LabelCaptureResult> {
const formData = new FormData()
formData.append('file', file)
formData.append('barcode', barcode)
const response = await api.post('/inventory/scan/label-capture', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
timeout: 60000, // vision inference can take ~5–10s
})
return response.data
},
/**
* Confirm a user-reviewed label extraction and save to the local cache.
*/
async confirmLabelCapture(data: LabelConfirmRequest): Promise<{ ok: boolean; product_id?: number; inventory_item_id?: number; message: string }> {
const response = await api.post('/inventory/scan/label-confirm', data)
return response.data
},
}
// ========== Receipts API ==========

270
tests/api/test_label_capture.py Normal file
View file

@ -0,0 +1,270 @@
"""
Tests for the visual label capture API endpoints (kiwi#79):
POST /api/v1/inventory/scan/label-capture
POST /api/v1/inventory/scan/label-confirm
GET /api/v1/inventory/scan/text — cache hit + needs_visual_capture flag
"""
import io
import os
import pytest
from fastapi.testclient import TestClient
from unittest.mock import MagicMock, patch
from app.main import app
from app.cloud_session import get_session
from app.db.session import get_store
client = TestClient(app)
def _session(tier: str = "paid", has_byok: bool = False) -> MagicMock:
m = MagicMock()
m.tier = tier
m.has_byok = has_byok
m.user_id = "test-user"
return m
def _store(**extra_returns) -> MagicMock:
m = MagicMock()
m.get_setting.return_value = None
m.get_captured_product.return_value = None
m.save_captured_product.return_value = {"id": 1, "barcode": "1234567890", "confirmed_by_user": 1}
m.get_or_create_product.return_value = (
{"id": 10, "name": "Test Product", "barcode": "1234567890",
"brand": None, "category": None, "description": None,
"image_url": None, "nutrition_data": {}, "source": "visual_capture",
"created_at": "2026-01-01", "updated_at": "2026-01-01"},
False,
)
m.add_inventory_item.return_value = {
"id": 99, "product_id": 10, "product_name": "Test Product",
"barcode": "1234567890", "category": None,
"quantity": 1.0, "unit": "count", "location": "pantry",
"sublocation": None, "purchase_date": None,
"expiration_date": None, "opened_date": None,
"opened_expiry_date": None, "secondary_state": None,
"secondary_uses": None, "secondary_warning": None,
"secondary_discard_signs": None, "status": "available",
"notes": None, "disposal_reason": None,
"source": "visual_capture", "created_at": "2026-01-01",
"updated_at": "2026-01-01",
}
for k, v in extra_returns.items():
setattr(m, k, v)
return m
@pytest.fixture(autouse=True)
def mock_env(monkeypatch):
monkeypatch.setenv("KIWI_LABEL_CAPTURE_MOCK", "1")
yield
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
# ── /scan/label-capture ───────────────────────────────────────────────────────
class TestLabelCaptureEndpoint:
def setup_method(self):
self.session = _session(tier="paid")
self.store = _store()
app.dependency_overrides[get_session] = lambda: self.session
app.dependency_overrides[get_store] = lambda: self.store
def teardown_method(self):
app.dependency_overrides.clear()
def test_returns_200_for_paid_tier(self):
resp = client.post(
"/api/v1/inventory/scan/label-capture",
data={"barcode": "1234567890"},
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
)
assert resp.status_code == 200
def test_response_contains_barcode(self):
resp = client.post(
"/api/v1/inventory/scan/label-capture",
data={"barcode": "5901234123457"},
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
)
data = resp.json()
assert data["barcode"] == "5901234123457"
def test_response_has_needs_review_field(self):
resp = client.post(
"/api/v1/inventory/scan/label-capture",
data={"barcode": "1234567890"},
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
)
data = resp.json()
assert "needs_review" in data
def test_mock_extraction_has_zero_confidence(self):
resp = client.post(
"/api/v1/inventory/scan/label-capture",
data={"barcode": "1234567890"},
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
)
data = resp.json()
assert data["confidence"] == 0.0
assert data["needs_review"] is True
def test_free_tier_returns_403(self):
app.dependency_overrides[get_session] = lambda: _session(tier="free")
resp = client.post(
"/api/v1/inventory/scan/label-capture",
data={"barcode": "1234567890"},
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
)
assert resp.status_code == 403
def test_local_tier_bypasses_gate(self):
app.dependency_overrides[get_session] = lambda: _session(tier="local")
resp = client.post(
"/api/v1/inventory/scan/label-capture",
data={"barcode": "1234567890"},
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
)
assert resp.status_code == 200
# ── /scan/label-confirm ───────────────────────────────────────────────────────
class TestLabelConfirmEndpoint:
def setup_method(self):
self.session = _session(tier="paid")
self.store = _store()
app.dependency_overrides[get_session] = lambda: self.session
app.dependency_overrides[get_store] = lambda: self.store
def teardown_method(self):
app.dependency_overrides.clear()
def test_returns_200(self):
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
"barcode": "1234567890",
"product_name": "Test Crackers",
"calories": 120.0,
"ingredient_names": ["flour", "salt"],
"allergens": ["wheat"],
"confidence": 0.88,
"auto_add": True,
"location": "pantry",
"quantity": 1.0,
})
assert resp.status_code == 200
def test_ok_true_in_response(self):
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
"barcode": "1234567890",
"auto_add": False,
})
assert resp.json()["ok"] is True
def test_save_captured_product_called(self):
client.post("/api/v1/inventory/scan/label-confirm", json={
"barcode": "1234567890",
"product_name": "Test",
"auto_add": False,
})
self.store.save_captured_product.assert_called_once()
call_kwargs = self.store.save_captured_product.call_args
assert call_kwargs[0][0] == "1234567890"
def test_auto_add_true_creates_inventory_item(self):
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
"barcode": "1234567890",
"auto_add": True,
})
data = resp.json()
assert data["inventory_item_id"] is not None
self.store.add_inventory_item.assert_called_once()
def test_auto_add_false_skips_inventory(self):
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
"barcode": "1234567890",
"auto_add": False,
})
data = resp.json()
assert data["inventory_item_id"] is None
self.store.add_inventory_item.assert_not_called()
def test_free_tier_blocked(self):
app.dependency_overrides[get_session] = lambda: _session(tier="free")
resp = client.post("/api/v1/inventory/scan/label-confirm", json={"barcode": "1234567890"})
assert resp.status_code == 403
# ── /scan/text cache hit + needs_visual_capture ───────────────────────────────
class TestScanTextWithCaptureGating:
def teardown_method(self):
app.dependency_overrides.clear()
def _setup(self, tier: str, cached=None, off_result=None):
store = _store()
store.get_captured_product.return_value = cached
session = _session(tier=tier)
app.dependency_overrides[get_session] = lambda: session
app.dependency_overrides[get_store] = lambda: store
return store
def _off_patch(self, result):
"""Patch OpenFoodFactsService.lookup_product at the class level."""
from unittest.mock import AsyncMock, patch
return patch(
"app.services.openfoodfacts.OpenFoodFactsService.lookup_product",
new=AsyncMock(return_value=result),
)
def test_paid_tier_no_product_sets_needs_visual_capture(self):
self._setup(tier="paid")
with self._off_patch(None):
resp = client.post("/api/v1/inventory/scan/text", json={
"barcode": "0000000000000", "location": "pantry",
})
assert resp.status_code == 200
result = resp.json()["results"][0]
assert result["needs_visual_capture"] is True
assert result["needs_manual_entry"] is False
def test_free_tier_no_product_sets_needs_manual_entry(self):
self._setup(tier="free")
with self._off_patch(None):
resp = client.post("/api/v1/inventory/scan/text", json={
"barcode": "0000000000000", "location": "pantry",
})
result = resp.json()["results"][0]
assert result["needs_visual_capture"] is False
assert result["needs_manual_entry"] is True
def test_cache_hit_uses_captured_product(self):
cached = {
"barcode": "9999999999999",
"product_name": "Cached Crackers",
"brand": "TestBrand",
"confirmed_by_user": 1,
"ingredient_names": ["flour"],
"allergens": [],
"calories": 110.0,
"fat_g": None, "saturated_fat_g": None, "carbs_g": None,
"sugar_g": None, "fiber_g": None, "protein_g": None,
"sodium_mg": None, "serving_size_g": None,
}
store = self._setup(tier="paid", cached=cached)
store.get_inventory_item.return_value = store.add_inventory_item.return_value
with self._off_patch(None): # OFF never called when cache hits
resp = client.post("/api/v1/inventory/scan/text", json={
"barcode": "9999999999999", "location": "pantry",
})
assert resp.status_code == 200
result = resp.json()["results"][0]
assert result["added_to_inventory"] is True
assert result["needs_visual_capture"] is False
# OFF was not called (cache resolved it)
# store.get_captured_product was called with the barcode
store.get_captured_product.assert_called_once_with("9999999999999")

116
tests/db/test_captured_products.py Normal file
View file

@ -0,0 +1,116 @@
"""Tests for captured_products store methods (kiwi#79)."""
import pytest
from pathlib import Path
from app.db.store import Store
@pytest.fixture
def store(tmp_path: Path) -> Store:
s = Store(tmp_path / "test.db")
yield s
s.close()
class TestMigration:
def test_captured_products_table_exists(self, store):
cur = store.conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='captured_products'"
)
assert cur.fetchone() is not None
def test_captured_products_columns(self, store):
cur = store.conn.execute("PRAGMA table_info(captured_products)")
# PRAGMA returns plain tuples: (cid, name, type, notnull, dflt_value, pk)
cols = {row[1] for row in cur.fetchall()}
expected = {
"id", "barcode", "product_name", "brand", "serving_size_g",
"calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g",
"fiber_g", "protein_g", "sodium_mg", "ingredient_names",
"allergens", "confidence", "source", "captured_at",
"confirmed_by_user",
}
assert expected.issubset(cols)
class TestGetCapturedProduct:
def test_returns_none_for_unknown_barcode(self, store):
assert store.get_captured_product("0000000000000") is None
def test_returns_row_after_save(self, store):
store.save_captured_product("1234567890123", product_name="Test Crackers")
result = store.get_captured_product("1234567890123")
assert result is not None
assert result["product_name"] == "Test Crackers"
def test_ingredient_names_decoded_as_list(self, store):
store.save_captured_product(
"1111111111111",
ingredient_names=["wheat flour", "salt"],
)
result = store.get_captured_product("1111111111111")
assert result["ingredient_names"] == ["wheat flour", "salt"]
def test_allergens_decoded_as_list(self, store):
store.save_captured_product(
"2222222222222",
allergens=["wheat", "milk"],
)
result = store.get_captured_product("2222222222222")
assert result["allergens"] == ["wheat", "milk"]
class TestSaveCapturedProduct:
def test_all_nutrition_fields_persisted(self, store):
store.save_captured_product(
"3333333333333",
product_name="Oat Crackers",
brand="TestBrand",
serving_size_g=30.0,
calories=120.0,
fat_g=4.0,
saturated_fat_g=0.5,
carbs_g=20.0,
sugar_g=2.0,
fiber_g=1.0,
protein_g=3.0,
sodium_mg=200.0,
confidence=0.92,
)
row = store.get_captured_product("3333333333333")
assert row["brand"] == "TestBrand"
assert row["calories"] == 120.0
assert row["protein_g"] == 3.0
assert row["confidence"] == 0.92
def test_confirmed_by_user_defaults_true(self, store):
store.save_captured_product("4444444444444")
row = store.get_captured_product("4444444444444")
assert row["confirmed_by_user"] == 1
def test_confirmed_by_user_false(self, store):
store.save_captured_product("5555555555555", confirmed_by_user=False)
row = store.get_captured_product("5555555555555")
assert row["confirmed_by_user"] == 0
def test_upsert_on_conflict(self, store):
"""Second save for same barcode updates in-place rather than erroring."""
store.save_captured_product("6666666666666", product_name="Old Name")
store.save_captured_product("6666666666666", product_name="New Name")
row = store.get_captured_product("6666666666666")
assert row["product_name"] == "New Name"
# Still only one row
cur = store.conn.execute(
"SELECT count(*) FROM captured_products WHERE barcode='6666666666666'"
)
assert cur.fetchone()[0] == 1
def test_empty_lists_stored_and_retrieved(self, store):
store.save_captured_product("7777777777777", ingredient_names=[], allergens=[])
row = store.get_captured_product("7777777777777")
assert row["ingredient_names"] == []
assert row["allergens"] == []
def test_source_default(self, store):
store.save_captured_product("8888888888888")
row = store.get_captured_product("8888888888888")
assert row["source"] == "visual_capture"

171
tests/services/test_label_capture.py Normal file
View file

@ -0,0 +1,171 @@
"""
Tests for app.services.label_capture.
All tests set KIWI_LABEL_CAPTURE_MOCK=1 so no vision model weights are needed.
"""
import json
import os
import pytest
@pytest.fixture(autouse=True)
def mock_vision(monkeypatch):
monkeypatch.setenv("KIWI_LABEL_CAPTURE_MOCK", "1")
yield
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
# ── TestExtractLabel ──────────────────────────────────────────────────────────
class TestExtractLabel:
def test_mock_returns_dict(self):
from app.services.label_capture import extract_label
result = extract_label(b"fake image bytes")
assert isinstance(result, dict)
def test_mock_returns_all_required_keys(self):
from app.services.label_capture import extract_label
result = extract_label(b"fake image bytes")
for key in ("product_name", "brand", "calories", "fat_g", "carbs_g",
"protein_g", "sodium_mg", "ingredient_names", "allergens",
"confidence"):
assert key in result, f"missing key: {key}"
def test_mock_ingredient_names_is_list(self):
from app.services.label_capture import extract_label
result = extract_label(b"fake")
assert isinstance(result["ingredient_names"], list)
def test_mock_allergens_is_list(self):
from app.services.label_capture import extract_label
result = extract_label(b"fake")
assert isinstance(result["allergens"], list)
def test_mock_confidence_zero(self):
from app.services.label_capture import extract_label
result = extract_label(b"fake")
assert result["confidence"] == 0.0
def _patch_vision(self, monkeypatch, caption_text: str):
"""Patch circuitforge_core.vision.caption to return a VisionResult with caption_text."""
from circuitforge_core.vision.backends.base import VisionResult
def _fake_caption(image_bytes, prompt=""):
return VisionResult(caption=caption_text)
import circuitforge_core.vision as vision_mod
monkeypatch.setattr(vision_mod, "caption", _fake_caption)
def test_exception_falls_back_to_mock(self, monkeypatch):
"""Any exception from the vision backend returns mock extraction."""
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
import circuitforge_core.vision as vision_mod
monkeypatch.setattr(vision_mod, "caption", lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("GPU unavailable")))
import app.services.label_capture as svc_mod
result = svc_mod.extract_label(b"image")
assert result["confidence"] == 0.0
assert isinstance(result["ingredient_names"], list)
def test_live_path_parses_json_string(self, monkeypatch):
"""When the vision backend returns valid JSON, it is parsed correctly."""
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
payload = {
"product_name": "Test Crackers",
"brand": "Test Brand",
"serving_size_g": 30.0,
"calories": 120.0,
"fat_g": 4.0,
"saturated_fat_g": 0.5,
"carbs_g": 20.0,
"sugar_g": 2.0,
"fiber_g": 1.0,
"protein_g": 3.0,
"sodium_mg": 200.0,
"ingredient_names": ["wheat flour", "canola oil", "salt"],
"allergens": ["wheat"],
"confidence": 0.92,
}
self._patch_vision(monkeypatch, json.dumps(payload))
import app.services.label_capture as svc_mod
result = svc_mod.extract_label(b"image")
assert result["product_name"] == "Test Crackers"
assert result["calories"] == 120.0
assert result["ingredient_names"] == ["wheat flour", "canola oil", "salt"]
assert result["allergens"] == ["wheat"]
assert result["confidence"] == 0.92
def test_live_path_strips_markdown_fences(self, monkeypatch):
"""JSON wrapped in ```json ... ``` fences is still parsed."""
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
payload = {"product_name": "Fancy", "brand": None, "serving_size_g": None,
"calories": None, "fat_g": None, "saturated_fat_g": None,
"carbs_g": None, "sugar_g": None, "fiber_g": None,
"protein_g": None, "sodium_mg": None,
"ingredient_names": [], "allergens": [], "confidence": 0.5}
self._patch_vision(monkeypatch, f"```json\n{json.dumps(payload)}\n```")
import app.services.label_capture as svc_mod
result = svc_mod.extract_label(b"image")
assert result["product_name"] == "Fancy"
def test_live_path_bad_json_falls_back(self, monkeypatch):
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
self._patch_vision(monkeypatch, "this is not json")
import app.services.label_capture as svc_mod
result = svc_mod.extract_label(b"image")
assert result["confidence"] == 0.0
def test_confidence_clamped_above_one(self, monkeypatch):
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
payload = {"product_name": None, "brand": None, "serving_size_g": None,
"calories": None, "fat_g": None, "saturated_fat_g": None,
"carbs_g": None, "sugar_g": None, "fiber_g": None,
"protein_g": None, "sodium_mg": None,
"ingredient_names": [], "allergens": [], "confidence": 5.0}
self._patch_vision(monkeypatch, json.dumps(payload))
import app.services.label_capture as svc_mod
result = svc_mod.extract_label(b"image")
assert result["confidence"] == 1.0
def test_none_list_fields_normalised_to_empty(self, monkeypatch):
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
payload = {"product_name": None, "brand": None, "serving_size_g": None,
"calories": None, "fat_g": None, "saturated_fat_g": None,
"carbs_g": None, "sugar_g": None, "fiber_g": None,
"protein_g": None, "sodium_mg": None,
"ingredient_names": None, "allergens": None, "confidence": 0.8}
self._patch_vision(monkeypatch, json.dumps(payload))
import app.services.label_capture as svc_mod
result = svc_mod.extract_label(b"image")
assert result["ingredient_names"] == []
assert result["allergens"] == []
# ── TestNeedsReview ───────────────────────────────────────────────────────────
class TestNeedsReview:
def test_below_threshold_needs_review(self):
from app.services.label_capture import needs_review, REVIEW_THRESHOLD
assert needs_review({"confidence": REVIEW_THRESHOLD - 0.01})
def test_at_threshold_no_review(self):
from app.services.label_capture import needs_review, REVIEW_THRESHOLD
assert not needs_review({"confidence": REVIEW_THRESHOLD})
def test_above_threshold_no_review(self):
from app.services.label_capture import needs_review
assert not needs_review({"confidence": 0.95})
def test_missing_confidence_needs_review(self):
from app.services.label_capture import needs_review
assert needs_review({})