From 17e62c451f73d8a487d081a4a919e4ecfe8a091a Mon Sep 17 00:00:00 2001 From: pyr0ball Date: Fri, 24 Apr 2026 17:57:25 -0700 Subject: [PATCH] feat: visual label capture for unenriched barcodes (kiwi#79) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a barcode scan finds no product in FDC/OFF, paid-tier users now see a "Capture label" offer instead of a dead-end "add manually" prompt. Backend: - Migration 036: captured_products local cache table (keyed by barcode, UPSERT on conflict so re-capture refreshes rather than errors) - store.get_captured_product / save_captured_product (with JSON decode for ingredient_names and allergens) - app/services/label_capture.py: wraps cf-core VisionRouter (caption API); graceful fallback to zero-confidence mock when stub/error; JSON fence stripping; confidence clamped to [0,1]; KIWI_LABEL_CAPTURE_MOCK=1 for tests - New schemas: LabelCaptureResponse, LabelConfirmRequest, LabelConfirmResponse - POST /inventory/scan/label-capture — image to extraction (paid+ gate, 403) - POST /inventory/scan/label-confirm — save confirmed product + optional inventory add - Both scan endpoints now: check captured_products cache before FDC/OFF; set needs_visual_capture=True for gap products on paid tier; BarcodeScanResult gains needs_visual_capture field - visual_label_capture feature gate added to tiers.py (paid) Tests: 42 new tests (service, store/migration, API endpoints) — 367 total passing Frontend: - InventoryList.vue: capturePhase state machine (offer => uploading => reviewing) - Offer card appears after scan gap (calm UX: no urgency, Discard always visible) - Review form: pre-populated from extraction; amber label highlights for unread fields (confidence < 0.7); comma-separated ingredients/allergens - api.ts: LabelCaptureResult + LabelConfirmRequest types; captureLabelPhoto() and confirmLabelCapture() API methods --- app/api/endpoints/inventory.py | 212 +++++++++++++- app/db/migrations/036_captured_products.sql | 26 ++ app/db/store.py | 74 ++++- app/models/schemas/inventory.py | 1 + app/models/schemas/label_capture.py | 59 ++++ app/services/label_capture.py | 140 +++++++++ app/tiers.py | 1 + frontend/src/components/InventoryList.vue | 296 +++++++++++++++++++- frontend/src/services/api.ts | 67 +++++ tests/api/test_label_capture.py | 270 ++++++++++++++++++ tests/db/test_captured_products.py | 116 ++++++++ tests/services/test_label_capture.py | 171 +++++++++++ 12 files changed, 1421 insertions(+), 12 deletions(-) create mode 100644 app/db/migrations/036_captured_products.sql create mode 100644 app/models/schemas/label_capture.py create mode 100644 app/services/label_capture.py create mode 100644 tests/api/test_label_capture.py create mode 100644 tests/db/test_captured_products.py create mode 100644 tests/services/test_label_capture.py diff --git a/app/api/endpoints/inventory.py b/app/api/endpoints/inventory.py index 543ae92..af0dca5 100644 --- a/app/api/endpoints/inventory.py +++ b/app/api/endpoints/inventory.py @@ -37,6 +37,7 @@ from app.models.schemas.inventory import ( TagCreate, TagResponse, ) +from app.models.schemas.label_capture import LabelConfirmRequest router = APIRouter() @@ -349,6 +350,31 @@ class BarcodeScanTextRequest(BaseModel): auto_add_to_inventory: bool = True +def _captured_to_product_info(row: dict) -> dict: + """Convert a captured_products row to the product_info dict shape used by + the barcode scan flow (mirrors what OpenFoodFactsService returns).""" + macros: dict = {} + for field in ("calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g", + "fiber_g", "protein_g", "sodium_mg", "serving_size_g"): + if row.get(field) is not None: + macros[field] = row[field] + return { + "name": row.get("product_name") or row.get("barcode", "Unknown Product"), + "brand": row.get("brand"), + "category": None, + "nutrition_data": macros, + "ingredient_names": row.get("ingredient_names") or [], + "allergens": row.get("allergens") or [], + "source": "visual_capture", + } + + +def _gap_message(tier: str, has_visual_capture: bool) -> str: + if has_visual_capture: + return "We couldn't find this product. Photograph the nutrition label to add it." + return "Not found in any product database — add manually" + + @router.post("/scan/text", response_model=BarcodeScanResponse) async def scan_barcode_text( body: BarcodeScanTextRequest, @@ -359,10 +385,21 @@ async def scan_barcode_text( log.info("scan auth=%s tier=%s barcode=%r", _auth_label(session.user_id), session.tier, body.barcode) from app.services.openfoodfacts import OpenFoodFactsService from app.services.expiration_predictor import ExpirationPredictor + from app.tiers import can_use - off = OpenFoodFactsService() predictor = ExpirationPredictor() - product_info = await off.lookup_product(body.barcode) + has_visual_capture = can_use("visual_label_capture", session.tier, session.has_byok) + + # 1. Check local captured-products cache before hitting FDC/OFF + cached = await asyncio.to_thread(store.get_captured_product, body.barcode) + if cached and cached.get("confirmed_by_user"): + product_info: dict | None = _captured_to_product_info(cached) + product_source = "visual_capture" + else: + off = OpenFoodFactsService() + product_info = await off.lookup_product(body.barcode) + product_source = "openfoodfacts" + inventory_item = None if product_info and body.auto_add_to_inventory: @@ -373,7 +410,7 @@ async def scan_barcode_text( brand=product_info.get("brand"), category=product_info.get("category"), nutrition_data=product_info.get("nutrition_data", {}), - source="openfoodfacts", + source=product_source, source_data=product_info, ) exp = predictor.predict_expiration( @@ -399,6 +436,7 @@ async def scan_barcode_text( result_product = None product_found = product_info is not None + needs_capture = not product_found and has_visual_capture return BarcodeScanResponse( success=True, barcodes_found=1, @@ -408,8 +446,9 @@ async def scan_barcode_text( "product": result_product, "inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None, "added_to_inventory": inventory_item is not None, - "needs_manual_entry": not product_found, - "message": "Added to inventory" if inventory_item else "Not found in any product database — add manually", + "needs_manual_entry": not product_found and not needs_capture, + "needs_visual_capture": needs_capture, + "message": "Added to inventory" if inventory_item else _gap_message(session.tier, needs_capture), }], message="Barcode processed", ) @@ -426,6 +465,9 @@ async def scan_barcode_image( ): """Scan a barcode from an uploaded image. Requires Phase 2 scanner integration.""" log.info("scan_image auth=%s tier=%s", _auth_label(session.user_id), session.tier) + from app.tiers import can_use + has_visual_capture = can_use("visual_label_capture", session.tier, session.has_byok) + temp_dir = Path("/tmp/kiwi_barcode_scans") temp_dir.mkdir(parents=True, exist_ok=True) temp_file = temp_dir / f"{uuid.uuid4()}_{file.filename}" @@ -448,7 +490,16 @@ async def scan_barcode_image( results = [] for bc in barcodes: code = bc["data"] - product_info = await off.lookup_product(code) + + # Check local visual-capture cache before hitting FDC/OFF + cached = await asyncio.to_thread(store.get_captured_product, code) + if cached and cached.get("confirmed_by_user"): + product_info: dict | None = _captured_to_product_info(cached) + product_source = "visual_capture" + else: + product_info = await off.lookup_product(code) + product_source = "openfoodfacts" + inventory_item = None if product_info and auto_add_to_inventory: product, _ = await asyncio.to_thread( @@ -458,7 +509,7 @@ async def scan_barcode_image( brand=product_info.get("brand"), category=product_info.get("category"), nutrition_data=product_info.get("nutrition_data", {}), - source="openfoodfacts", + source=product_source, source_data=product_info, ) exp = predictor.predict_expiration( @@ -466,7 +517,7 @@ async def scan_barcode_image( location, product_name=product_info.get("name", code), tier=session.tier, - has_byok=session.has_byok, + has_byok=session.has_byok, ) resolved_qty = product_info.get("pack_quantity") or quantity resolved_unit = product_info.get("pack_unit") or "count" @@ -478,13 +529,17 @@ async def scan_barcode_image( expiration_date=str(exp) if exp else None, source="barcode_scan", ) + product_found = product_info is not None + needs_capture = not product_found and has_visual_capture results.append({ "barcode": code, "barcode_type": bc.get("type", "unknown"), - "product": ProductResponse.model_validate(product) if product_info else None, + "product": ProductResponse.model_validate(product_info) if product_info else None, "inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None, "added_to_inventory": inventory_item is not None, - "message": "Added to inventory" if inventory_item else "Barcode scanned", + "needs_manual_entry": not product_found and not needs_capture, + "needs_visual_capture": needs_capture, + "message": "Added to inventory" if inventory_item else _gap_message(session.tier, needs_capture), }) return BarcodeScanResponse( success=True, barcodes_found=len(barcodes), results=results, @@ -495,6 +550,143 @@ async def scan_barcode_image( temp_file.unlink() +# ── Visual label capture (kiwi#79) ──────────────────────────────────────────── + +@router.post("/scan/label-capture") +async def capture_nutrition_label( + file: UploadFile = File(...), + barcode: str = Form(...), + store: Store = Depends(get_store), + session: CloudUser = Depends(get_session), +): + """Photograph a nutrition label for an unenriched product (paid tier). + + Sends the image to the vision model and returns structured nutrition data + for user review. Fields extracted with confidence < 0.7 should be + highlighted in amber in the UI. + """ + from app.tiers import can_use + from app.models.schemas.label_capture import LabelCaptureResponse + from app.services.label_capture import extract_label, needs_review as _needs_review + + if not can_use("visual_label_capture", session.tier, session.has_byok): + raise HTTPException(status_code=403, detail="Visual label capture requires a Paid tier or higher.") + log.info("label_capture tier=%s barcode=%r", session.tier, barcode) + + image_bytes = await file.read() + extraction = await asyncio.to_thread(extract_label, image_bytes) + + return LabelCaptureResponse( + barcode=barcode, + product_name=extraction.get("product_name"), + brand=extraction.get("brand"), + serving_size_g=extraction.get("serving_size_g"), + calories=extraction.get("calories"), + fat_g=extraction.get("fat_g"), + saturated_fat_g=extraction.get("saturated_fat_g"), + carbs_g=extraction.get("carbs_g"), + sugar_g=extraction.get("sugar_g"), + fiber_g=extraction.get("fiber_g"), + protein_g=extraction.get("protein_g"), + sodium_mg=extraction.get("sodium_mg"), + ingredient_names=extraction.get("ingredient_names") or [], + allergens=extraction.get("allergens") or [], + confidence=extraction.get("confidence", 0.0), + needs_review=_needs_review(extraction), + ) + + +@router.post("/scan/label-confirm") +async def confirm_nutrition_label( + body: LabelConfirmRequest, + store: Store = Depends(get_store), + session: CloudUser = Depends(get_session), +): + """Confirm and save a user-reviewed label extraction. + + Saves the product to the local cache so future scans of the same barcode + resolve instantly without another capture. Optionally adds the item to + the user's inventory. + """ + from app.tiers import can_use + from app.models.schemas.label_capture import LabelConfirmResponse + from app.services.expiration_predictor import ExpirationPredictor + + if not can_use("visual_label_capture", session.tier, session.has_byok): + raise HTTPException(status_code=403, detail="Visual label capture requires a Paid tier or higher.") + log.info("label_confirm tier=%s barcode=%r", session.tier, body.barcode) + + # Persist to local visual-capture cache + await asyncio.to_thread( + store.save_captured_product, + body.barcode, + product_name=body.product_name, + brand=body.brand, + serving_size_g=body.serving_size_g, + calories=body.calories, + fat_g=body.fat_g, + saturated_fat_g=body.saturated_fat_g, + carbs_g=body.carbs_g, + sugar_g=body.sugar_g, + fiber_g=body.fiber_g, + protein_g=body.protein_g, + sodium_mg=body.sodium_mg, + ingredient_names=body.ingredient_names, + allergens=body.allergens, + confidence=body.confidence, + confirmed_by_user=True, + ) + + product_id: int | None = None + inventory_item_id: int | None = None + + if body.auto_add: + predictor = ExpirationPredictor() + nutrition = {} + for field in ("calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g", + "fiber_g", "protein_g", "sodium_mg", "serving_size_g"): + val = getattr(body, field, None) + if val is not None: + nutrition[field] = val + + product, _ = await asyncio.to_thread( + store.get_or_create_product, + body.product_name or body.barcode, + body.barcode, + brand=body.brand, + category=None, + nutrition_data=nutrition, + source="visual_capture", + source_data={}, + ) + product_id = product["id"] + + exp = predictor.predict_expiration( + "", + body.location, + product_name=body.product_name or body.barcode, + tier=session.tier, + has_byok=session.has_byok, + ) + inv_item = await asyncio.to_thread( + store.add_inventory_item, + product_id, body.location, + quantity=body.quantity, + unit="count", + expiration_date=str(exp) if exp else None, + source="visual_capture", + ) + inventory_item_id = inv_item["id"] + + return LabelConfirmResponse( + ok=True, + barcode=body.barcode, + product_id=product_id, + inventory_item_id=inventory_item_id, + message="Product saved" + (" and added to inventory" if body.auto_add else ""), + ) + + # ── Tags ────────────────────────────────────────────────────────────────────── @router.post("/tags", response_model=TagResponse, status_code=status.HTTP_201_CREATED) diff --git a/app/db/migrations/036_captured_products.sql b/app/db/migrations/036_captured_products.sql new file mode 100644 index 0000000..25570db --- /dev/null +++ b/app/db/migrations/036_captured_products.sql @@ -0,0 +1,26 @@ +-- Migration 036: captured_products local cache +-- Products captured via visual label scanning (kiwi#79). +-- Keyed by barcode; checked before FDC/OFF on future scans so each product +-- is only captured once per device. + +CREATE TABLE IF NOT EXISTS captured_products ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + barcode TEXT UNIQUE NOT NULL, + product_name TEXT, + brand TEXT, + serving_size_g REAL, + calories REAL, + fat_g REAL, + saturated_fat_g REAL, + carbs_g REAL, + sugar_g REAL, + fiber_g REAL, + protein_g REAL, + sodium_mg REAL, + ingredient_names TEXT NOT NULL DEFAULT '[]', -- JSON array + allergens TEXT NOT NULL DEFAULT '[]', -- JSON array + confidence REAL, + source TEXT NOT NULL DEFAULT 'visual_capture', + captured_at TEXT NOT NULL DEFAULT (datetime('now')), + confirmed_by_user INTEGER NOT NULL DEFAULT 0 +); diff --git a/app/db/store.py b/app/db/store.py index b9fa0bf..818ca6c 100644 --- a/app/db/store.py +++ b/app/db/store.py @@ -60,7 +60,9 @@ class Store: # saved recipe columns "style_tags", # meal plan columns - "meal_types"): + "meal_types", + # captured_products columns + "allergens"): if key in d and isinstance(d[key], str): try: d[key] = json.loads(d[key]) @@ -1639,3 +1641,73 @@ class Store: cur = self.conn.execute("DELETE FROM shopping_list_items") self.conn.commit() return cur.rowcount + + # ── Captured products (visual label cache) ──────────────────────────────── + + def get_captured_product(self, barcode: str) -> dict | None: + """Look up a locally-captured product by barcode. + + Returns the row dict (ingredient_names and allergens already decoded as + lists) or None if the barcode has not been captured yet. + """ + return self._fetch_one( + "SELECT * FROM captured_products WHERE barcode = ?", (barcode,) + ) + + def save_captured_product( + self, + barcode: str, + *, + product_name: str | None = None, + brand: str | None = None, + serving_size_g: float | None = None, + calories: float | None = None, + fat_g: float | None = None, + saturated_fat_g: float | None = None, + carbs_g: float | None = None, + sugar_g: float | None = None, + fiber_g: float | None = None, + protein_g: float | None = None, + sodium_mg: float | None = None, + ingredient_names: list[str] | None = None, + allergens: list[str] | None = None, + confidence: float | None = None, + confirmed_by_user: bool = True, + source: str = "visual_capture", + ) -> dict: + """Insert or replace a captured product row, returning the saved dict.""" + return self._insert_returning( + """INSERT INTO captured_products + (barcode, product_name, brand, serving_size_g, calories, + fat_g, saturated_fat_g, carbs_g, sugar_g, fiber_g, + protein_g, sodium_mg, ingredient_names, allergens, + confidence, confirmed_by_user, source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(barcode) DO UPDATE SET + product_name = excluded.product_name, + brand = excluded.brand, + serving_size_g = excluded.serving_size_g, + calories = excluded.calories, + fat_g = excluded.fat_g, + saturated_fat_g = excluded.saturated_fat_g, + carbs_g = excluded.carbs_g, + sugar_g = excluded.sugar_g, + fiber_g = excluded.fiber_g, + protein_g = excluded.protein_g, + sodium_mg = excluded.sodium_mg, + ingredient_names = excluded.ingredient_names, + allergens = excluded.allergens, + confidence = excluded.confidence, + confirmed_by_user = excluded.confirmed_by_user, + source = excluded.source, + captured_at = datetime('now') + RETURNING *""", + ( + barcode, product_name, brand, serving_size_g, calories, + fat_g, saturated_fat_g, carbs_g, sugar_g, fiber_g, + protein_g, sodium_mg, + self._dump(ingredient_names or []), + self._dump(allergens or []), + confidence, 1 if confirmed_by_user else 0, source, + ), + ) diff --git a/app/models/schemas/inventory.py b/app/models/schemas/inventory.py index 147938f..fe42f03 100644 --- a/app/models/schemas/inventory.py +++ b/app/models/schemas/inventory.py @@ -142,6 +142,7 @@ class BarcodeScanResult(BaseModel): inventory_item: Optional[InventoryItemResponse] added_to_inventory: bool needs_manual_entry: bool = False + needs_visual_capture: bool = False # Paid tier offer when no product data found message: str diff --git a/app/models/schemas/label_capture.py b/app/models/schemas/label_capture.py new file mode 100644 index 0000000..ea97aae --- /dev/null +++ b/app/models/schemas/label_capture.py @@ -0,0 +1,59 @@ +"""Pydantic schemas for visual label capture (kiwi#79).""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + + +class LabelCaptureResponse(BaseModel): + """Extraction result returned after the user photographs a nutrition label.""" + barcode: str + product_name: Optional[str] = None + brand: Optional[str] = None + serving_size_g: Optional[float] = None + calories: Optional[float] = None + fat_g: Optional[float] = None + saturated_fat_g: Optional[float] = None + carbs_g: Optional[float] = None + sugar_g: Optional[float] = None + fiber_g: Optional[float] = None + protein_g: Optional[float] = None + sodium_mg: Optional[float] = None + ingredient_names: List[str] = Field(default_factory=list) + allergens: List[str] = Field(default_factory=list) + confidence: float = 0.0 + needs_review: bool = True # True when confidence < REVIEW_THRESHOLD + + +class LabelConfirmRequest(BaseModel): + """User-confirmed extraction to save to the local product cache.""" + barcode: str + product_name: Optional[str] = None + brand: Optional[str] = None + serving_size_g: Optional[float] = None + calories: Optional[float] = None + fat_g: Optional[float] = None + saturated_fat_g: Optional[float] = None + carbs_g: Optional[float] = None + sugar_g: Optional[float] = None + fiber_g: Optional[float] = None + protein_g: Optional[float] = None + sodium_mg: Optional[float] = None + ingredient_names: List[str] = Field(default_factory=list) + allergens: List[str] = Field(default_factory=list) + confidence: float = 0.0 + # When True the confirmed product is also added to inventory + location: str = "pantry" + quantity: float = 1.0 + auto_add: bool = True + + +class LabelConfirmResponse(BaseModel): + """Result of confirming a captured product.""" + ok: bool + barcode: str + product_id: Optional[int] = None + inventory_item_id: Optional[int] = None + message: str diff --git a/app/services/label_capture.py b/app/services/label_capture.py new file mode 100644 index 0000000..5c7a144 --- /dev/null +++ b/app/services/label_capture.py @@ -0,0 +1,140 @@ +"""Visual label capture service for unenriched products (kiwi#79). + +Wraps the cf-core VisionRouter to extract structured nutrition data from a +photographed nutrition facts panel. When the VisionRouter is not yet wired +(NotImplementedError) the service falls back to a mock extraction so the +barcode scan flow can be exercised end-to-end in development. + +JSON contract returned by the vision model (and mock): + { + "product_name": str | null, + "brand": str | null, + "serving_size_g": number | null, + "calories": number | null, + "fat_g": number | null, + "saturated_fat_g": number | null, + "carbs_g": number | null, + "sugar_g": number | null, + "fiber_g": number | null, + "protein_g": number | null, + "sodium_mg": number | null, + "ingredient_names": [str], + "allergens": [str], + "confidence": number (0.0–1.0) + } +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any + +log = logging.getLogger(__name__) + +# Confidence below this threshold surfaces amber highlights in the UI. +REVIEW_THRESHOLD = 0.7 + +_MOCK_EXTRACTION: dict[str, Any] = { + "product_name": "Unknown Product", + "brand": None, + "serving_size_g": None, + "calories": None, + "fat_g": None, + "saturated_fat_g": None, + "carbs_g": None, + "sugar_g": None, + "fiber_g": None, + "protein_g": None, + "sodium_mg": None, + "ingredient_names": [], + "allergens": [], + "confidence": 0.0, +} + +_EXTRACTION_PROMPT = """You are reading a nutrition facts label photograph. +Extract the following fields as a JSON object with no extra text: + +{ + "product_name": , + "brand": , + "serving_size_g": , + "calories": , + "fat_g": , + "saturated_fat_g": , + "carbs_g": , + "sugar_g": , + "fiber_g": , + "protein_g": , + "sodium_mg": , + "ingredient_names": [list of individual ingredients as strings], + "allergens": [list of allergens explicitly stated on label], + "confidence": +} + +Use null for any field you cannot read clearly. Do not guess values. +Respond with JSON only.""" + + +def extract_label(image_bytes: bytes) -> dict[str, Any]: + """Run vision model extraction on raw label image bytes. + + Returns a dict matching the nutrition JSON contract above. + Falls back to a zero-confidence mock if the VisionRouter is not yet + implemented (stub) or if the model returns unparseable output. + """ + # Allow unit tests to bypass the vision model entirely. + if os.environ.get("KIWI_LABEL_CAPTURE_MOCK") == "1": + log.debug("label_capture: mock mode active") + return dict(_MOCK_EXTRACTION) + + try: + from circuitforge_core.vision import caption as vision_caption + result = vision_caption(image_bytes, prompt=_EXTRACTION_PROMPT) + raw = result.caption or "" + return _parse_extraction(raw) + except Exception as exc: + log.warning("label_capture: extraction failed (%s) — returning mock extraction", exc) + return dict(_MOCK_EXTRACTION) + + +def _parse_extraction(raw: str) -> dict[str, Any]: + """Parse the JSON string returned by the vision model. + + Strips markdown code fences if present. Validates required shape. + Returns the mock on any parse error. + """ + text = raw.strip() + if text.startswith("```"): + # Strip ```json ... ``` fences + lines = text.splitlines() + text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:]) + + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + log.warning("label_capture: could not parse vision response: %s", exc) + return dict(_MOCK_EXTRACTION) + + if not isinstance(data, dict): + log.warning("label_capture: vision response is not a dict") + return dict(_MOCK_EXTRACTION) + + # Normalise list fields — model may return None instead of [] + for list_key in ("ingredient_names", "allergens"): + if not isinstance(data.get(list_key), list): + data[list_key] = [] + + # Clamp confidence to [0, 1] + confidence = data.get("confidence") + if not isinstance(confidence, (int, float)): + confidence = 0.0 + data["confidence"] = max(0.0, min(1.0, float(confidence))) + + return data + + +def needs_review(extraction: dict[str, Any]) -> bool: + """Return True when the extraction confidence is below REVIEW_THRESHOLD.""" + return float(extraction.get("confidence", 0.0)) < REVIEW_THRESHOLD diff --git a/app/tiers.py b/app/tiers.py index 1ce348c..b07ec82 100644 --- a/app/tiers.py +++ b/app/tiers.py @@ -44,6 +44,7 @@ KIWI_FEATURES: dict[str, str] = { # Paid tier "receipt_ocr": "paid", # BYOK-unlockable + "visual_label_capture": "paid", # Camera capture for unenriched barcodes (kiwi#79) "recipe_suggestions": "paid", # BYOK-unlockable "expiry_llm_matching": "paid", # BYOK-unlockable "meal_planning": "free", diff --git a/frontend/src/components/InventoryList.vue b/frontend/src/components/InventoryList.vue index cf38972..52e55f3 100644 --- a/frontend/src/components/InventoryList.vue +++ b/frontend/src/components/InventoryList.vue @@ -138,6 +138,103 @@ + +
+ + +
+

We couldn't find this product. Photograph the nutrition label to add it.

+
+ + +
+ +
+ + +
+
+ + Reading the label… +
+
+ + +
+

+ Check the details below. + + Fields highlighted in amber weren't fully legible — please verify them. + +

+ +
+
+ + +
+
+ + +
+
+ +

Nutrition per serving

+
+
+ + +
+
+ +
+ + +
+
+ + +
+ +
+ + + +
+
+
+
@@ -622,7 +719,7 @@ import { storeToRefs } from 'pinia' import { useInventoryStore } from '../stores/inventory' import { useSettingsStore } from '../stores/settings' import { inventoryAPI } from '../services/api' -import type { InventoryItem } from '../services/api' +import type { InventoryItem, LabelCaptureResult } from '../services/api' import { formatQuantity } from '../utils/units' import EditItemModal from './EditItemModal.vue' import ConfirmDialog from './ConfirmDialog.vue' @@ -684,6 +781,16 @@ function daysLabel(dateStr: string): string { const scanMode = ref<'gun' | 'camera' | 'manual'>('gun') // Options for button groups +// Label capture nutrition field descriptors used in the review form +const captureNutritionFields = [ + { key: 'calories', src: 'calories', label: 'Calories', unit: 'kcal' }, + { key: 'fat_g', src: 'fat_g', label: 'Total fat', unit: 'g' }, + { key: 'saturated_fat_g', src: 'saturated_fat_g', label: 'Saturated fat', unit: 'g' }, + { key: 'carbs_g', src: 'carbs_g', label: 'Carbs', unit: 'g' }, + { key: 'protein_g', src: 'protein_g', label: 'Protein', unit: 'g' }, + { key: 'sodium_mg', src: 'sodium_mg', label: 'Sodium', unit: 'mg' }, +] + const locations = [ { value: 'fridge', label: 'Fridge', icon: '🧊' }, { value: 'freezer', label: 'Freezer', icon: '❄️' }, @@ -780,6 +887,29 @@ const barcodeQuantity = ref(1) const barcodeLoading = ref(false) const barcodeResults = ref>([]) +// Label Capture Flow (kiwi#79) +type CapturePhase = 'offer' | 'uploading' | 'reviewing' | null +const capturePhase = ref(null) +const captureBarcode = ref('') +const captureLocation = ref('pantry') +const captureQuantity = ref(1) +const captureLoading = ref(false) +const captureFileInput = ref(null) +const captureExtraction = ref(null) +// Editable review form — populated from extraction, user may correct fields +const captureReview = ref({ + product_name: '', + brand: '', + calories: '' as string, + fat_g: '' as string, + saturated_fat_g: '' as string, + carbs_g: '' as string, + protein_g: '' as string, + sodium_mg: '' as string, + ingredients: '', + allergens: '', +}) + // Manual Form const manualForm = ref({ name: '', @@ -935,6 +1065,15 @@ async function handleScannerGunInput() { message: `Added: ${productName}${productBrand} to ${scannerLocation.value}`, }) await refreshItems() + } else if (item?.needs_visual_capture) { + captureBarcode.value = barcode + captureLocation.value = scannerLocation.value + captureQuantity.value = scannerQuantity.value + capturePhase.value = 'offer' + scannerResults.value.push({ + type: 'info', + message: item.message, + }) } else if (item?.needs_manual_entry) { // Barcode not found in any database — guide user to manual entry scannerResults.value.push({ @@ -1007,6 +1146,88 @@ async function handleBarcodeImageSelect(e: Event) { } } +// Label Capture Functions + +function triggerCaptureLabelInput() { + captureFileInput.value?.click() +} + +function dismissCapture() { + capturePhase.value = null + captureBarcode.value = '' + captureExtraction.value = null +} + +async function handleLabelPhotoSelect(e: Event) { + const target = e.target as HTMLInputElement + const file = target.files?.[0] + if (!file) return + + captureLoading.value = true + capturePhase.value = 'uploading' + + try { + const result = await inventoryAPI.captureLabelPhoto(file, captureBarcode.value) + captureExtraction.value = result + // Pre-populate the review form with extracted values + captureReview.value = { + product_name: result.product_name || '', + brand: result.brand || '', + calories: result.calories != null ? String(result.calories) : '', + fat_g: result.fat_g != null ? String(result.fat_g) : '', + saturated_fat_g: result.saturated_fat_g != null ? String(result.saturated_fat_g) : '', + carbs_g: result.carbs_g != null ? String(result.carbs_g) : '', + protein_g: result.protein_g != null ? String(result.protein_g) : '', + sodium_mg: result.sodium_mg != null ? String(result.sodium_mg) : '', + ingredients: (result.ingredient_names || []).join(', '), + allergens: (result.allergens || []).join(', '), + } + capturePhase.value = 'reviewing' + } catch { + showToast('Could not read the label. Please try again or add manually.', 'error') + capturePhase.value = 'offer' + } finally { + captureLoading.value = false + if (target) target.value = '' + } +} + +async function confirmCapture() { + if (!captureBarcode.value) return + + captureLoading.value = true + try { + const toNum = (s: string) => s ? parseFloat(s) || null : null + const toList = (s: string) => s.split(',').map(x => x.trim()).filter(Boolean) + + await inventoryAPI.confirmLabelCapture({ + barcode: captureBarcode.value, + product_name: captureReview.value.product_name || null, + brand: captureReview.value.brand || null, + calories: toNum(captureReview.value.calories), + fat_g: toNum(captureReview.value.fat_g), + saturated_fat_g: toNum(captureReview.value.saturated_fat_g), + carbs_g: toNum(captureReview.value.carbs_g), + protein_g: toNum(captureReview.value.protein_g), + sodium_mg: toNum(captureReview.value.sodium_mg), + ingredient_names: toList(captureReview.value.ingredients), + allergens: toList(captureReview.value.allergens), + confidence: captureExtraction.value?.confidence ?? 0, + location: captureLocation.value, + quantity: captureQuantity.value, + auto_add: true, + }) + const name = captureReview.value.product_name || 'item' + showToast(`${name} saved and added to ${captureLocation.value}`, 'success') + await refreshItems() + dismissCapture() + } catch { + showToast('Could not save. Please try again.', 'error') + } finally { + captureLoading.value = false + } +} + // Manual Add Functions async function addManualItem() { const { name, brand, quantity, unit, location, expirationDate } = manualForm.value @@ -1614,6 +1835,79 @@ function getItemClass(item: InventoryItem): string { border: 1px solid var(--color-warning-border, #fcd34d); } +/* ============================================ + LABEL CAPTURE FLOW (kiwi#79) + ============================================ */ +.label-capture-panel { + margin: var(--spacing-md) 0; + padding: var(--spacing-md); + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: var(--radius-lg); +} + +.capture-offer-text { + font-size: var(--font-size-sm); + color: var(--color-text-secondary); + margin: 0 0 var(--spacing-md); +} + +.capture-offer-actions { + display: flex; + gap: var(--spacing-sm); + flex-wrap: wrap; +} + +.capture-processing { + display: flex; + justify-content: center; + padding: var(--spacing-md) 0; +} + +.capture-review-note { + font-size: var(--font-size-sm); + color: var(--color-text-secondary); + margin: 0 0 var(--spacing-md); +} + +.capture-review-low-conf { + color: var(--color-amber, #d97706); + font-size: var(--font-size-xs); + display: block; + margin-top: var(--spacing-xs); +} + +.form-section-label { + font-size: var(--font-size-xs); + font-weight: 600; + color: var(--color-text-tertiary); + text-transform: uppercase; + letter-spacing: 0.05em; + margin: var(--spacing-md) 0 var(--spacing-sm); +} + +.capture-nutrition-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(120px, 1fr)); + gap: var(--spacing-sm); +} + +/* Amber highlight for unread/low-confidence label fields */ +.capture-field-amber { + color: var(--color-amber, #d97706); +} + +.capture-field-amber + input { + border-color: var(--color-amber, #d97706); +} + +.capture-review-actions { + display: flex; + gap: var(--spacing-sm); + flex-wrap: wrap; + margin-top: var(--spacing-md); +} + /* ============================================ EXPORT CARD ============================================ */ diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 094f5a0..9bc298c 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -111,9 +111,50 @@ export interface BarcodeScanResult { inventory_item: InventoryItem | null added_to_inventory: boolean needs_manual_entry: boolean + needs_visual_capture: boolean message: string } +export interface LabelCaptureResult { + barcode: string + product_name: string | null + brand: string | null + serving_size_g: number | null + calories: number | null + fat_g: number | null + saturated_fat_g: number | null + carbs_g: number | null + sugar_g: number | null + fiber_g: number | null + protein_g: number | null + sodium_mg: number | null + ingredient_names: string[] + allergens: string[] + confidence: number + needs_review: boolean +} + +export interface LabelConfirmRequest { + barcode: string + product_name?: string | null + brand?: string | null + serving_size_g?: number | null + calories?: number | null + fat_g?: number | null + saturated_fat_g?: number | null + carbs_g?: number | null + sugar_g?: number | null + fiber_g?: number | null + protein_g?: number | null + sodium_mg?: number | null + ingredient_names?: string[] + allergens?: string[] + confidence?: number + location?: string + quantity?: number + auto_add?: boolean +} + export interface BarcodeScanResponse { success: boolean barcodes_found: number @@ -344,6 +385,32 @@ export const inventoryAPI = { }) return response.data }, + + /** + * Upload a nutrition label photo for an unenriched barcode (paid tier). + * Returns extracted fields + confidence score for user review. + */ + async captureLabelPhoto( + file: File, + barcode: string + ): Promise { + const formData = new FormData() + formData.append('file', file) + formData.append('barcode', barcode) + const response = await api.post('/inventory/scan/label-capture', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + timeout: 60000, // vision inference can take ~5–10s + }) + return response.data + }, + + /** + * Confirm a user-reviewed label extraction and save to the local cache. + */ + async confirmLabelCapture(data: LabelConfirmRequest): Promise<{ ok: boolean; product_id?: number; inventory_item_id?: number; message: string }> { + const response = await api.post('/inventory/scan/label-confirm', data) + return response.data + }, } // ========== Receipts API ========== diff --git a/tests/api/test_label_capture.py b/tests/api/test_label_capture.py new file mode 100644 index 0000000..766c997 --- /dev/null +++ b/tests/api/test_label_capture.py @@ -0,0 +1,270 @@ +""" +Tests for the visual label capture API endpoints (kiwi#79): + POST /api/v1/inventory/scan/label-capture + POST /api/v1/inventory/scan/label-confirm + GET /api/v1/inventory/scan/text — cache hit + needs_visual_capture flag +""" +import io +import os +import pytest +from fastapi.testclient import TestClient +from unittest.mock import MagicMock, patch + +from app.main import app +from app.cloud_session import get_session +from app.db.session import get_store + + +client = TestClient(app) + + +def _session(tier: str = "paid", has_byok: bool = False) -> MagicMock: + m = MagicMock() + m.tier = tier + m.has_byok = has_byok + m.user_id = "test-user" + return m + + +def _store(**extra_returns) -> MagicMock: + m = MagicMock() + m.get_setting.return_value = None + m.get_captured_product.return_value = None + m.save_captured_product.return_value = {"id": 1, "barcode": "1234567890", "confirmed_by_user": 1} + m.get_or_create_product.return_value = ( + {"id": 10, "name": "Test Product", "barcode": "1234567890", + "brand": None, "category": None, "description": None, + "image_url": None, "nutrition_data": {}, "source": "visual_capture", + "created_at": "2026-01-01", "updated_at": "2026-01-01"}, + False, + ) + m.add_inventory_item.return_value = { + "id": 99, "product_id": 10, "product_name": "Test Product", + "barcode": "1234567890", "category": None, + "quantity": 1.0, "unit": "count", "location": "pantry", + "sublocation": None, "purchase_date": None, + "expiration_date": None, "opened_date": None, + "opened_expiry_date": None, "secondary_state": None, + "secondary_uses": None, "secondary_warning": None, + "secondary_discard_signs": None, "status": "available", + "notes": None, "disposal_reason": None, + "source": "visual_capture", "created_at": "2026-01-01", + "updated_at": "2026-01-01", + } + for k, v in extra_returns.items(): + setattr(m, k, v) + return m + + +@pytest.fixture(autouse=True) +def mock_env(monkeypatch): + monkeypatch.setenv("KIWI_LABEL_CAPTURE_MOCK", "1") + yield + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + + +# ── /scan/label-capture ─────────────────────────────────────────────────────── + +class TestLabelCaptureEndpoint: + def setup_method(self): + self.session = _session(tier="paid") + self.store = _store() + app.dependency_overrides[get_session] = lambda: self.session + app.dependency_overrides[get_store] = lambda: self.store + + def teardown_method(self): + app.dependency_overrides.clear() + + def test_returns_200_for_paid_tier(self): + resp = client.post( + "/api/v1/inventory/scan/label-capture", + data={"barcode": "1234567890"}, + files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")}, + ) + assert resp.status_code == 200 + + def test_response_contains_barcode(self): + resp = client.post( + "/api/v1/inventory/scan/label-capture", + data={"barcode": "5901234123457"}, + files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")}, + ) + data = resp.json() + assert data["barcode"] == "5901234123457" + + def test_response_has_needs_review_field(self): + resp = client.post( + "/api/v1/inventory/scan/label-capture", + data={"barcode": "1234567890"}, + files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")}, + ) + data = resp.json() + assert "needs_review" in data + + def test_mock_extraction_has_zero_confidence(self): + resp = client.post( + "/api/v1/inventory/scan/label-capture", + data={"barcode": "1234567890"}, + files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")}, + ) + data = resp.json() + assert data["confidence"] == 0.0 + assert data["needs_review"] is True + + def test_free_tier_returns_403(self): + app.dependency_overrides[get_session] = lambda: _session(tier="free") + resp = client.post( + "/api/v1/inventory/scan/label-capture", + data={"barcode": "1234567890"}, + files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")}, + ) + assert resp.status_code == 403 + + def test_local_tier_bypasses_gate(self): + app.dependency_overrides[get_session] = lambda: _session(tier="local") + resp = client.post( + "/api/v1/inventory/scan/label-capture", + data={"barcode": "1234567890"}, + files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")}, + ) + assert resp.status_code == 200 + + +# ── /scan/label-confirm ─────────────────────────────────────────────────────── + +class TestLabelConfirmEndpoint: + def setup_method(self): + self.session = _session(tier="paid") + self.store = _store() + app.dependency_overrides[get_session] = lambda: self.session + app.dependency_overrides[get_store] = lambda: self.store + + def teardown_method(self): + app.dependency_overrides.clear() + + def test_returns_200(self): + resp = client.post("/api/v1/inventory/scan/label-confirm", json={ + "barcode": "1234567890", + "product_name": "Test Crackers", + "calories": 120.0, + "ingredient_names": ["flour", "salt"], + "allergens": ["wheat"], + "confidence": 0.88, + "auto_add": True, + "location": "pantry", + "quantity": 1.0, + }) + assert resp.status_code == 200 + + def test_ok_true_in_response(self): + resp = client.post("/api/v1/inventory/scan/label-confirm", json={ + "barcode": "1234567890", + "auto_add": False, + }) + assert resp.json()["ok"] is True + + def test_save_captured_product_called(self): + client.post("/api/v1/inventory/scan/label-confirm", json={ + "barcode": "1234567890", + "product_name": "Test", + "auto_add": False, + }) + self.store.save_captured_product.assert_called_once() + call_kwargs = self.store.save_captured_product.call_args + assert call_kwargs[0][0] == "1234567890" + + def test_auto_add_true_creates_inventory_item(self): + resp = client.post("/api/v1/inventory/scan/label-confirm", json={ + "barcode": "1234567890", + "auto_add": True, + }) + data = resp.json() + assert data["inventory_item_id"] is not None + self.store.add_inventory_item.assert_called_once() + + def test_auto_add_false_skips_inventory(self): + resp = client.post("/api/v1/inventory/scan/label-confirm", json={ + "barcode": "1234567890", + "auto_add": False, + }) + data = resp.json() + assert data["inventory_item_id"] is None + self.store.add_inventory_item.assert_not_called() + + def test_free_tier_blocked(self): + app.dependency_overrides[get_session] = lambda: _session(tier="free") + resp = client.post("/api/v1/inventory/scan/label-confirm", json={"barcode": "1234567890"}) + assert resp.status_code == 403 + + +# ── /scan/text cache hit + needs_visual_capture ─────────────────────────────── + +class TestScanTextWithCaptureGating: + def teardown_method(self): + app.dependency_overrides.clear() + + def _setup(self, tier: str, cached=None, off_result=None): + store = _store() + store.get_captured_product.return_value = cached + session = _session(tier=tier) + app.dependency_overrides[get_session] = lambda: session + app.dependency_overrides[get_store] = lambda: store + return store + + def _off_patch(self, result): + """Patch OpenFoodFactsService.lookup_product at the class level.""" + from unittest.mock import AsyncMock, patch + return patch( + "app.services.openfoodfacts.OpenFoodFactsService.lookup_product", + new=AsyncMock(return_value=result), + ) + + def test_paid_tier_no_product_sets_needs_visual_capture(self): + self._setup(tier="paid") + with self._off_patch(None): + resp = client.post("/api/v1/inventory/scan/text", json={ + "barcode": "0000000000000", "location": "pantry", + }) + assert resp.status_code == 200 + result = resp.json()["results"][0] + assert result["needs_visual_capture"] is True + assert result["needs_manual_entry"] is False + + def test_free_tier_no_product_sets_needs_manual_entry(self): + self._setup(tier="free") + with self._off_patch(None): + resp = client.post("/api/v1/inventory/scan/text", json={ + "barcode": "0000000000000", "location": "pantry", + }) + result = resp.json()["results"][0] + assert result["needs_visual_capture"] is False + assert result["needs_manual_entry"] is True + + def test_cache_hit_uses_captured_product(self): + cached = { + "barcode": "9999999999999", + "product_name": "Cached Crackers", + "brand": "TestBrand", + "confirmed_by_user": 1, + "ingredient_names": ["flour"], + "allergens": [], + "calories": 110.0, + "fat_g": None, "saturated_fat_g": None, "carbs_g": None, + "sugar_g": None, "fiber_g": None, "protein_g": None, + "sodium_mg": None, "serving_size_g": None, + } + store = self._setup(tier="paid", cached=cached) + store.get_inventory_item.return_value = store.add_inventory_item.return_value + + with self._off_patch(None): # OFF never called when cache hits + resp = client.post("/api/v1/inventory/scan/text", json={ + "barcode": "9999999999999", "location": "pantry", + }) + + assert resp.status_code == 200 + result = resp.json()["results"][0] + assert result["added_to_inventory"] is True + assert result["needs_visual_capture"] is False + # OFF was not called (cache resolved it) + # store.get_captured_product was called with the barcode + store.get_captured_product.assert_called_once_with("9999999999999") diff --git a/tests/db/test_captured_products.py b/tests/db/test_captured_products.py new file mode 100644 index 0000000..28942fb --- /dev/null +++ b/tests/db/test_captured_products.py @@ -0,0 +1,116 @@ +"""Tests for captured_products store methods (kiwi#79).""" +import pytest +from pathlib import Path +from app.db.store import Store + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + s = Store(tmp_path / "test.db") + yield s + s.close() + + +class TestMigration: + def test_captured_products_table_exists(self, store): + cur = store.conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='captured_products'" + ) + assert cur.fetchone() is not None + + def test_captured_products_columns(self, store): + cur = store.conn.execute("PRAGMA table_info(captured_products)") + # PRAGMA returns plain tuples: (cid, name, type, notnull, dflt_value, pk) + cols = {row[1] for row in cur.fetchall()} + expected = { + "id", "barcode", "product_name", "brand", "serving_size_g", + "calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g", + "fiber_g", "protein_g", "sodium_mg", "ingredient_names", + "allergens", "confidence", "source", "captured_at", + "confirmed_by_user", + } + assert expected.issubset(cols) + + +class TestGetCapturedProduct: + def test_returns_none_for_unknown_barcode(self, store): + assert store.get_captured_product("0000000000000") is None + + def test_returns_row_after_save(self, store): + store.save_captured_product("1234567890123", product_name="Test Crackers") + result = store.get_captured_product("1234567890123") + assert result is not None + assert result["product_name"] == "Test Crackers" + + def test_ingredient_names_decoded_as_list(self, store): + store.save_captured_product( + "1111111111111", + ingredient_names=["wheat flour", "salt"], + ) + result = store.get_captured_product("1111111111111") + assert result["ingredient_names"] == ["wheat flour", "salt"] + + def test_allergens_decoded_as_list(self, store): + store.save_captured_product( + "2222222222222", + allergens=["wheat", "milk"], + ) + result = store.get_captured_product("2222222222222") + assert result["allergens"] == ["wheat", "milk"] + + +class TestSaveCapturedProduct: + def test_all_nutrition_fields_persisted(self, store): + store.save_captured_product( + "3333333333333", + product_name="Oat Crackers", + brand="TestBrand", + serving_size_g=30.0, + calories=120.0, + fat_g=4.0, + saturated_fat_g=0.5, + carbs_g=20.0, + sugar_g=2.0, + fiber_g=1.0, + protein_g=3.0, + sodium_mg=200.0, + confidence=0.92, + ) + row = store.get_captured_product("3333333333333") + assert row["brand"] == "TestBrand" + assert row["calories"] == 120.0 + assert row["protein_g"] == 3.0 + assert row["confidence"] == 0.92 + + def test_confirmed_by_user_defaults_true(self, store): + store.save_captured_product("4444444444444") + row = store.get_captured_product("4444444444444") + assert row["confirmed_by_user"] == 1 + + def test_confirmed_by_user_false(self, store): + store.save_captured_product("5555555555555", confirmed_by_user=False) + row = store.get_captured_product("5555555555555") + assert row["confirmed_by_user"] == 0 + + def test_upsert_on_conflict(self, store): + """Second save for same barcode updates in-place rather than erroring.""" + store.save_captured_product("6666666666666", product_name="Old Name") + store.save_captured_product("6666666666666", product_name="New Name") + row = store.get_captured_product("6666666666666") + assert row["product_name"] == "New Name" + # Still only one row + cur = store.conn.execute( + "SELECT count(*) FROM captured_products WHERE barcode='6666666666666'" + ) + assert cur.fetchone()[0] == 1 + + def test_empty_lists_stored_and_retrieved(self, store): + store.save_captured_product("7777777777777", ingredient_names=[], allergens=[]) + row = store.get_captured_product("7777777777777") + assert row["ingredient_names"] == [] + assert row["allergens"] == [] + + def test_source_default(self, store): + store.save_captured_product("8888888888888") + row = store.get_captured_product("8888888888888") + assert row["source"] == "visual_capture" diff --git a/tests/services/test_label_capture.py b/tests/services/test_label_capture.py new file mode 100644 index 0000000..f660af3 --- /dev/null +++ b/tests/services/test_label_capture.py @@ -0,0 +1,171 @@ +""" +Tests for app.services.label_capture. + +All tests set KIWI_LABEL_CAPTURE_MOCK=1 so no vision model weights are needed. +""" +import json +import os +import pytest + + +@pytest.fixture(autouse=True) +def mock_vision(monkeypatch): + monkeypatch.setenv("KIWI_LABEL_CAPTURE_MOCK", "1") + yield + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + + +# ── TestExtractLabel ────────────────────────────────────────────────────────── + +class TestExtractLabel: + def test_mock_returns_dict(self): + from app.services.label_capture import extract_label + result = extract_label(b"fake image bytes") + assert isinstance(result, dict) + + def test_mock_returns_all_required_keys(self): + from app.services.label_capture import extract_label + result = extract_label(b"fake image bytes") + for key in ("product_name", "brand", "calories", "fat_g", "carbs_g", + "protein_g", "sodium_mg", "ingredient_names", "allergens", + "confidence"): + assert key in result, f"missing key: {key}" + + def test_mock_ingredient_names_is_list(self): + from app.services.label_capture import extract_label + result = extract_label(b"fake") + assert isinstance(result["ingredient_names"], list) + + def test_mock_allergens_is_list(self): + from app.services.label_capture import extract_label + result = extract_label(b"fake") + assert isinstance(result["allergens"], list) + + def test_mock_confidence_zero(self): + from app.services.label_capture import extract_label + result = extract_label(b"fake") + assert result["confidence"] == 0.0 + + def _patch_vision(self, monkeypatch, caption_text: str): + """Patch circuitforge_core.vision.caption to return a VisionResult with caption_text.""" + from circuitforge_core.vision.backends.base import VisionResult + + def _fake_caption(image_bytes, prompt=""): + return VisionResult(caption=caption_text) + + import circuitforge_core.vision as vision_mod + monkeypatch.setattr(vision_mod, "caption", _fake_caption) + + def test_exception_falls_back_to_mock(self, monkeypatch): + """Any exception from the vision backend returns mock extraction.""" + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + + import circuitforge_core.vision as vision_mod + monkeypatch.setattr(vision_mod, "caption", lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("GPU unavailable"))) + + import app.services.label_capture as svc_mod + result = svc_mod.extract_label(b"image") + assert result["confidence"] == 0.0 + assert isinstance(result["ingredient_names"], list) + + def test_live_path_parses_json_string(self, monkeypatch): + """When the vision backend returns valid JSON, it is parsed correctly.""" + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + + payload = { + "product_name": "Test Crackers", + "brand": "Test Brand", + "serving_size_g": 30.0, + "calories": 120.0, + "fat_g": 4.0, + "saturated_fat_g": 0.5, + "carbs_g": 20.0, + "sugar_g": 2.0, + "fiber_g": 1.0, + "protein_g": 3.0, + "sodium_mg": 200.0, + "ingredient_names": ["wheat flour", "canola oil", "salt"], + "allergens": ["wheat"], + "confidence": 0.92, + } + self._patch_vision(monkeypatch, json.dumps(payload)) + + import app.services.label_capture as svc_mod + result = svc_mod.extract_label(b"image") + assert result["product_name"] == "Test Crackers" + assert result["calories"] == 120.0 + assert result["ingredient_names"] == ["wheat flour", "canola oil", "salt"] + assert result["allergens"] == ["wheat"] + assert result["confidence"] == 0.92 + + def test_live_path_strips_markdown_fences(self, monkeypatch): + """JSON wrapped in ```json ... ``` fences is still parsed.""" + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + + payload = {"product_name": "Fancy", "brand": None, "serving_size_g": None, + "calories": None, "fat_g": None, "saturated_fat_g": None, + "carbs_g": None, "sugar_g": None, "fiber_g": None, + "protein_g": None, "sodium_mg": None, + "ingredient_names": [], "allergens": [], "confidence": 0.5} + self._patch_vision(monkeypatch, f"```json\n{json.dumps(payload)}\n```") + + import app.services.label_capture as svc_mod + result = svc_mod.extract_label(b"image") + assert result["product_name"] == "Fancy" + + def test_live_path_bad_json_falls_back(self, monkeypatch): + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + self._patch_vision(monkeypatch, "this is not json") + + import app.services.label_capture as svc_mod + result = svc_mod.extract_label(b"image") + assert result["confidence"] == 0.0 + + def test_confidence_clamped_above_one(self, monkeypatch): + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + + payload = {"product_name": None, "brand": None, "serving_size_g": None, + "calories": None, "fat_g": None, "saturated_fat_g": None, + "carbs_g": None, "sugar_g": None, "fiber_g": None, + "protein_g": None, "sodium_mg": None, + "ingredient_names": [], "allergens": [], "confidence": 5.0} + self._patch_vision(monkeypatch, json.dumps(payload)) + + import app.services.label_capture as svc_mod + result = svc_mod.extract_label(b"image") + assert result["confidence"] == 1.0 + + def test_none_list_fields_normalised_to_empty(self, monkeypatch): + monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False) + + payload = {"product_name": None, "brand": None, "serving_size_g": None, + "calories": None, "fat_g": None, "saturated_fat_g": None, + "carbs_g": None, "sugar_g": None, "fiber_g": None, + "protein_g": None, "sodium_mg": None, + "ingredient_names": None, "allergens": None, "confidence": 0.8} + self._patch_vision(monkeypatch, json.dumps(payload)) + + import app.services.label_capture as svc_mod + result = svc_mod.extract_label(b"image") + assert result["ingredient_names"] == [] + assert result["allergens"] == [] + + +# ── TestNeedsReview ─────────────────────────────────────────────────────────── + +class TestNeedsReview: + def test_below_threshold_needs_review(self): + from app.services.label_capture import needs_review, REVIEW_THRESHOLD + assert needs_review({"confidence": REVIEW_THRESHOLD - 0.01}) + + def test_at_threshold_no_review(self): + from app.services.label_capture import needs_review, REVIEW_THRESHOLD + assert not needs_review({"confidence": REVIEW_THRESHOLD}) + + def test_above_threshold_no_review(self): + from app.services.label_capture import needs_review + assert not needs_review({"confidence": 0.95}) + + def test_missing_confidence_needs_review(self): + from app.services.label_capture import needs_review + assert needs_review({})