feat: visual label capture for unenriched barcodes (kiwi#79)
When a barcode scan finds no product in FDC/OFF, paid-tier users now see a "Capture label" offer instead of a dead-end "add manually" prompt. Backend: - Migration 036: captured_products local cache table (keyed by barcode, UPSERT on conflict so re-capture refreshes rather than errors) - store.get_captured_product / save_captured_product (with JSON decode for ingredient_names and allergens) - app/services/label_capture.py: wraps cf-core VisionRouter (caption API); graceful fallback to zero-confidence mock when stub/error; JSON fence stripping; confidence clamped to [0,1]; KIWI_LABEL_CAPTURE_MOCK=1 for tests - New schemas: LabelCaptureResponse, LabelConfirmRequest, LabelConfirmResponse - POST /inventory/scan/label-capture — image to extraction (paid+ gate, 403) - POST /inventory/scan/label-confirm — save confirmed product + optional inventory add - Both scan endpoints now: check captured_products cache before FDC/OFF; set needs_visual_capture=True for gap products on paid tier; BarcodeScanResult gains needs_visual_capture field - visual_label_capture feature gate added to tiers.py (paid) Tests: 42 new tests (service, store/migration, API endpoints) — 367 total passing Frontend: - InventoryList.vue: capturePhase state machine (offer => uploading => reviewing) - Offer card appears after scan gap (calm UX: no urgency, Discard always visible) - Review form: pre-populated from extraction; amber label highlights for unread fields (confidence < 0.7); comma-separated ingredients/allergens - api.ts: LabelCaptureResult + LabelConfirmRequest types; captureLabelPhoto() and confirmLabelCapture() API methods
This commit is contained in:
parent
3463aa1e17
commit
17e62c451f
12 changed files with 1421 additions and 12 deletions
|
|
@ -37,6 +37,7 @@ from app.models.schemas.inventory import (
|
||||||
TagCreate,
|
TagCreate,
|
||||||
TagResponse,
|
TagResponse,
|
||||||
)
|
)
|
||||||
|
from app.models.schemas.label_capture import LabelConfirmRequest
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
@ -349,6 +350,31 @@ class BarcodeScanTextRequest(BaseModel):
|
||||||
auto_add_to_inventory: bool = True
|
auto_add_to_inventory: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
def _captured_to_product_info(row: dict) -> dict:
|
||||||
|
"""Convert a captured_products row to the product_info dict shape used by
|
||||||
|
the barcode scan flow (mirrors what OpenFoodFactsService returns)."""
|
||||||
|
macros: dict = {}
|
||||||
|
for field in ("calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g",
|
||||||
|
"fiber_g", "protein_g", "sodium_mg", "serving_size_g"):
|
||||||
|
if row.get(field) is not None:
|
||||||
|
macros[field] = row[field]
|
||||||
|
return {
|
||||||
|
"name": row.get("product_name") or row.get("barcode", "Unknown Product"),
|
||||||
|
"brand": row.get("brand"),
|
||||||
|
"category": None,
|
||||||
|
"nutrition_data": macros,
|
||||||
|
"ingredient_names": row.get("ingredient_names") or [],
|
||||||
|
"allergens": row.get("allergens") or [],
|
||||||
|
"source": "visual_capture",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _gap_message(tier: str, has_visual_capture: bool) -> str:
|
||||||
|
if has_visual_capture:
|
||||||
|
return "We couldn't find this product. Photograph the nutrition label to add it."
|
||||||
|
return "Not found in any product database — add manually"
|
||||||
|
|
||||||
|
|
||||||
@router.post("/scan/text", response_model=BarcodeScanResponse)
|
@router.post("/scan/text", response_model=BarcodeScanResponse)
|
||||||
async def scan_barcode_text(
|
async def scan_barcode_text(
|
||||||
body: BarcodeScanTextRequest,
|
body: BarcodeScanTextRequest,
|
||||||
|
|
@ -359,10 +385,21 @@ async def scan_barcode_text(
|
||||||
log.info("scan auth=%s tier=%s barcode=%r", _auth_label(session.user_id), session.tier, body.barcode)
|
log.info("scan auth=%s tier=%s barcode=%r", _auth_label(session.user_id), session.tier, body.barcode)
|
||||||
from app.services.openfoodfacts import OpenFoodFactsService
|
from app.services.openfoodfacts import OpenFoodFactsService
|
||||||
from app.services.expiration_predictor import ExpirationPredictor
|
from app.services.expiration_predictor import ExpirationPredictor
|
||||||
|
from app.tiers import can_use
|
||||||
|
|
||||||
off = OpenFoodFactsService()
|
|
||||||
predictor = ExpirationPredictor()
|
predictor = ExpirationPredictor()
|
||||||
product_info = await off.lookup_product(body.barcode)
|
has_visual_capture = can_use("visual_label_capture", session.tier, session.has_byok)
|
||||||
|
|
||||||
|
# 1. Check local captured-products cache before hitting FDC/OFF
|
||||||
|
cached = await asyncio.to_thread(store.get_captured_product, body.barcode)
|
||||||
|
if cached and cached.get("confirmed_by_user"):
|
||||||
|
product_info: dict | None = _captured_to_product_info(cached)
|
||||||
|
product_source = "visual_capture"
|
||||||
|
else:
|
||||||
|
off = OpenFoodFactsService()
|
||||||
|
product_info = await off.lookup_product(body.barcode)
|
||||||
|
product_source = "openfoodfacts"
|
||||||
|
|
||||||
inventory_item = None
|
inventory_item = None
|
||||||
|
|
||||||
if product_info and body.auto_add_to_inventory:
|
if product_info and body.auto_add_to_inventory:
|
||||||
|
|
@ -373,7 +410,7 @@ async def scan_barcode_text(
|
||||||
brand=product_info.get("brand"),
|
brand=product_info.get("brand"),
|
||||||
category=product_info.get("category"),
|
category=product_info.get("category"),
|
||||||
nutrition_data=product_info.get("nutrition_data", {}),
|
nutrition_data=product_info.get("nutrition_data", {}),
|
||||||
source="openfoodfacts",
|
source=product_source,
|
||||||
source_data=product_info,
|
source_data=product_info,
|
||||||
)
|
)
|
||||||
exp = predictor.predict_expiration(
|
exp = predictor.predict_expiration(
|
||||||
|
|
@ -399,6 +436,7 @@ async def scan_barcode_text(
|
||||||
result_product = None
|
result_product = None
|
||||||
|
|
||||||
product_found = product_info is not None
|
product_found = product_info is not None
|
||||||
|
needs_capture = not product_found and has_visual_capture
|
||||||
return BarcodeScanResponse(
|
return BarcodeScanResponse(
|
||||||
success=True,
|
success=True,
|
||||||
barcodes_found=1,
|
barcodes_found=1,
|
||||||
|
|
@ -408,8 +446,9 @@ async def scan_barcode_text(
|
||||||
"product": result_product,
|
"product": result_product,
|
||||||
"inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None,
|
"inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None,
|
||||||
"added_to_inventory": inventory_item is not None,
|
"added_to_inventory": inventory_item is not None,
|
||||||
"needs_manual_entry": not product_found,
|
"needs_manual_entry": not product_found and not needs_capture,
|
||||||
"message": "Added to inventory" if inventory_item else "Not found in any product database — add manually",
|
"needs_visual_capture": needs_capture,
|
||||||
|
"message": "Added to inventory" if inventory_item else _gap_message(session.tier, needs_capture),
|
||||||
}],
|
}],
|
||||||
message="Barcode processed",
|
message="Barcode processed",
|
||||||
)
|
)
|
||||||
|
|
@ -426,6 +465,9 @@ async def scan_barcode_image(
|
||||||
):
|
):
|
||||||
"""Scan a barcode from an uploaded image. Requires Phase 2 scanner integration."""
|
"""Scan a barcode from an uploaded image. Requires Phase 2 scanner integration."""
|
||||||
log.info("scan_image auth=%s tier=%s", _auth_label(session.user_id), session.tier)
|
log.info("scan_image auth=%s tier=%s", _auth_label(session.user_id), session.tier)
|
||||||
|
from app.tiers import can_use
|
||||||
|
has_visual_capture = can_use("visual_label_capture", session.tier, session.has_byok)
|
||||||
|
|
||||||
temp_dir = Path("/tmp/kiwi_barcode_scans")
|
temp_dir = Path("/tmp/kiwi_barcode_scans")
|
||||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||||
temp_file = temp_dir / f"{uuid.uuid4()}_{file.filename}"
|
temp_file = temp_dir / f"{uuid.uuid4()}_{file.filename}"
|
||||||
|
|
@ -448,7 +490,16 @@ async def scan_barcode_image(
|
||||||
results = []
|
results = []
|
||||||
for bc in barcodes:
|
for bc in barcodes:
|
||||||
code = bc["data"]
|
code = bc["data"]
|
||||||
product_info = await off.lookup_product(code)
|
|
||||||
|
# Check local visual-capture cache before hitting FDC/OFF
|
||||||
|
cached = await asyncio.to_thread(store.get_captured_product, code)
|
||||||
|
if cached and cached.get("confirmed_by_user"):
|
||||||
|
product_info: dict | None = _captured_to_product_info(cached)
|
||||||
|
product_source = "visual_capture"
|
||||||
|
else:
|
||||||
|
product_info = await off.lookup_product(code)
|
||||||
|
product_source = "openfoodfacts"
|
||||||
|
|
||||||
inventory_item = None
|
inventory_item = None
|
||||||
if product_info and auto_add_to_inventory:
|
if product_info and auto_add_to_inventory:
|
||||||
product, _ = await asyncio.to_thread(
|
product, _ = await asyncio.to_thread(
|
||||||
|
|
@ -458,7 +509,7 @@ async def scan_barcode_image(
|
||||||
brand=product_info.get("brand"),
|
brand=product_info.get("brand"),
|
||||||
category=product_info.get("category"),
|
category=product_info.get("category"),
|
||||||
nutrition_data=product_info.get("nutrition_data", {}),
|
nutrition_data=product_info.get("nutrition_data", {}),
|
||||||
source="openfoodfacts",
|
source=product_source,
|
||||||
source_data=product_info,
|
source_data=product_info,
|
||||||
)
|
)
|
||||||
exp = predictor.predict_expiration(
|
exp = predictor.predict_expiration(
|
||||||
|
|
@ -466,7 +517,7 @@ async def scan_barcode_image(
|
||||||
location,
|
location,
|
||||||
product_name=product_info.get("name", code),
|
product_name=product_info.get("name", code),
|
||||||
tier=session.tier,
|
tier=session.tier,
|
||||||
has_byok=session.has_byok,
|
has_byok=session.has_byok,
|
||||||
)
|
)
|
||||||
resolved_qty = product_info.get("pack_quantity") or quantity
|
resolved_qty = product_info.get("pack_quantity") or quantity
|
||||||
resolved_unit = product_info.get("pack_unit") or "count"
|
resolved_unit = product_info.get("pack_unit") or "count"
|
||||||
|
|
@ -478,13 +529,17 @@ async def scan_barcode_image(
|
||||||
expiration_date=str(exp) if exp else None,
|
expiration_date=str(exp) if exp else None,
|
||||||
source="barcode_scan",
|
source="barcode_scan",
|
||||||
)
|
)
|
||||||
|
product_found = product_info is not None
|
||||||
|
needs_capture = not product_found and has_visual_capture
|
||||||
results.append({
|
results.append({
|
||||||
"barcode": code,
|
"barcode": code,
|
||||||
"barcode_type": bc.get("type", "unknown"),
|
"barcode_type": bc.get("type", "unknown"),
|
||||||
"product": ProductResponse.model_validate(product) if product_info else None,
|
"product": ProductResponse.model_validate(product_info) if product_info else None,
|
||||||
"inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None,
|
"inventory_item": InventoryItemResponse.model_validate(inventory_item) if inventory_item else None,
|
||||||
"added_to_inventory": inventory_item is not None,
|
"added_to_inventory": inventory_item is not None,
|
||||||
"message": "Added to inventory" if inventory_item else "Barcode scanned",
|
"needs_manual_entry": not product_found and not needs_capture,
|
||||||
|
"needs_visual_capture": needs_capture,
|
||||||
|
"message": "Added to inventory" if inventory_item else _gap_message(session.tier, needs_capture),
|
||||||
})
|
})
|
||||||
return BarcodeScanResponse(
|
return BarcodeScanResponse(
|
||||||
success=True, barcodes_found=len(barcodes), results=results,
|
success=True, barcodes_found=len(barcodes), results=results,
|
||||||
|
|
@ -495,6 +550,143 @@ async def scan_barcode_image(
|
||||||
temp_file.unlink()
|
temp_file.unlink()
|
||||||
|
|
||||||
|
|
||||||
|
# ── Visual label capture (kiwi#79) ────────────────────────────────────────────
|
||||||
|
|
||||||
|
@router.post("/scan/label-capture")
|
||||||
|
async def capture_nutrition_label(
|
||||||
|
file: UploadFile = File(...),
|
||||||
|
barcode: str = Form(...),
|
||||||
|
store: Store = Depends(get_store),
|
||||||
|
session: CloudUser = Depends(get_session),
|
||||||
|
):
|
||||||
|
"""Photograph a nutrition label for an unenriched product (paid tier).
|
||||||
|
|
||||||
|
Sends the image to the vision model and returns structured nutrition data
|
||||||
|
for user review. Fields extracted with confidence < 0.7 should be
|
||||||
|
highlighted in amber in the UI.
|
||||||
|
"""
|
||||||
|
from app.tiers import can_use
|
||||||
|
from app.models.schemas.label_capture import LabelCaptureResponse
|
||||||
|
from app.services.label_capture import extract_label, needs_review as _needs_review
|
||||||
|
|
||||||
|
if not can_use("visual_label_capture", session.tier, session.has_byok):
|
||||||
|
raise HTTPException(status_code=403, detail="Visual label capture requires a Paid tier or higher.")
|
||||||
|
log.info("label_capture tier=%s barcode=%r", session.tier, barcode)
|
||||||
|
|
||||||
|
image_bytes = await file.read()
|
||||||
|
extraction = await asyncio.to_thread(extract_label, image_bytes)
|
||||||
|
|
||||||
|
return LabelCaptureResponse(
|
||||||
|
barcode=barcode,
|
||||||
|
product_name=extraction.get("product_name"),
|
||||||
|
brand=extraction.get("brand"),
|
||||||
|
serving_size_g=extraction.get("serving_size_g"),
|
||||||
|
calories=extraction.get("calories"),
|
||||||
|
fat_g=extraction.get("fat_g"),
|
||||||
|
saturated_fat_g=extraction.get("saturated_fat_g"),
|
||||||
|
carbs_g=extraction.get("carbs_g"),
|
||||||
|
sugar_g=extraction.get("sugar_g"),
|
||||||
|
fiber_g=extraction.get("fiber_g"),
|
||||||
|
protein_g=extraction.get("protein_g"),
|
||||||
|
sodium_mg=extraction.get("sodium_mg"),
|
||||||
|
ingredient_names=extraction.get("ingredient_names") or [],
|
||||||
|
allergens=extraction.get("allergens") or [],
|
||||||
|
confidence=extraction.get("confidence", 0.0),
|
||||||
|
needs_review=_needs_review(extraction),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/scan/label-confirm")
|
||||||
|
async def confirm_nutrition_label(
|
||||||
|
body: LabelConfirmRequest,
|
||||||
|
store: Store = Depends(get_store),
|
||||||
|
session: CloudUser = Depends(get_session),
|
||||||
|
):
|
||||||
|
"""Confirm and save a user-reviewed label extraction.
|
||||||
|
|
||||||
|
Saves the product to the local cache so future scans of the same barcode
|
||||||
|
resolve instantly without another capture. Optionally adds the item to
|
||||||
|
the user's inventory.
|
||||||
|
"""
|
||||||
|
from app.tiers import can_use
|
||||||
|
from app.models.schemas.label_capture import LabelConfirmResponse
|
||||||
|
from app.services.expiration_predictor import ExpirationPredictor
|
||||||
|
|
||||||
|
if not can_use("visual_label_capture", session.tier, session.has_byok):
|
||||||
|
raise HTTPException(status_code=403, detail="Visual label capture requires a Paid tier or higher.")
|
||||||
|
log.info("label_confirm tier=%s barcode=%r", session.tier, body.barcode)
|
||||||
|
|
||||||
|
# Persist to local visual-capture cache
|
||||||
|
await asyncio.to_thread(
|
||||||
|
store.save_captured_product,
|
||||||
|
body.barcode,
|
||||||
|
product_name=body.product_name,
|
||||||
|
brand=body.brand,
|
||||||
|
serving_size_g=body.serving_size_g,
|
||||||
|
calories=body.calories,
|
||||||
|
fat_g=body.fat_g,
|
||||||
|
saturated_fat_g=body.saturated_fat_g,
|
||||||
|
carbs_g=body.carbs_g,
|
||||||
|
sugar_g=body.sugar_g,
|
||||||
|
fiber_g=body.fiber_g,
|
||||||
|
protein_g=body.protein_g,
|
||||||
|
sodium_mg=body.sodium_mg,
|
||||||
|
ingredient_names=body.ingredient_names,
|
||||||
|
allergens=body.allergens,
|
||||||
|
confidence=body.confidence,
|
||||||
|
confirmed_by_user=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
product_id: int | None = None
|
||||||
|
inventory_item_id: int | None = None
|
||||||
|
|
||||||
|
if body.auto_add:
|
||||||
|
predictor = ExpirationPredictor()
|
||||||
|
nutrition = {}
|
||||||
|
for field in ("calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g",
|
||||||
|
"fiber_g", "protein_g", "sodium_mg", "serving_size_g"):
|
||||||
|
val = getattr(body, field, None)
|
||||||
|
if val is not None:
|
||||||
|
nutrition[field] = val
|
||||||
|
|
||||||
|
product, _ = await asyncio.to_thread(
|
||||||
|
store.get_or_create_product,
|
||||||
|
body.product_name or body.barcode,
|
||||||
|
body.barcode,
|
||||||
|
brand=body.brand,
|
||||||
|
category=None,
|
||||||
|
nutrition_data=nutrition,
|
||||||
|
source="visual_capture",
|
||||||
|
source_data={},
|
||||||
|
)
|
||||||
|
product_id = product["id"]
|
||||||
|
|
||||||
|
exp = predictor.predict_expiration(
|
||||||
|
"",
|
||||||
|
body.location,
|
||||||
|
product_name=body.product_name or body.barcode,
|
||||||
|
tier=session.tier,
|
||||||
|
has_byok=session.has_byok,
|
||||||
|
)
|
||||||
|
inv_item = await asyncio.to_thread(
|
||||||
|
store.add_inventory_item,
|
||||||
|
product_id, body.location,
|
||||||
|
quantity=body.quantity,
|
||||||
|
unit="count",
|
||||||
|
expiration_date=str(exp) if exp else None,
|
||||||
|
source="visual_capture",
|
||||||
|
)
|
||||||
|
inventory_item_id = inv_item["id"]
|
||||||
|
|
||||||
|
return LabelConfirmResponse(
|
||||||
|
ok=True,
|
||||||
|
barcode=body.barcode,
|
||||||
|
product_id=product_id,
|
||||||
|
inventory_item_id=inventory_item_id,
|
||||||
|
message="Product saved" + (" and added to inventory" if body.auto_add else ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ── Tags ──────────────────────────────────────────────────────────────────────
|
# ── Tags ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@router.post("/tags", response_model=TagResponse, status_code=status.HTTP_201_CREATED)
|
@router.post("/tags", response_model=TagResponse, status_code=status.HTTP_201_CREATED)
|
||||||
|
|
|
||||||
26
app/db/migrations/036_captured_products.sql
Normal file
26
app/db/migrations/036_captured_products.sql
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
-- Migration 036: captured_products local cache
|
||||||
|
-- Products captured via visual label scanning (kiwi#79).
|
||||||
|
-- Keyed by barcode; checked before FDC/OFF on future scans so each product
|
||||||
|
-- is only captured once per device.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS captured_products (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
barcode TEXT UNIQUE NOT NULL,
|
||||||
|
product_name TEXT,
|
||||||
|
brand TEXT,
|
||||||
|
serving_size_g REAL,
|
||||||
|
calories REAL,
|
||||||
|
fat_g REAL,
|
||||||
|
saturated_fat_g REAL,
|
||||||
|
carbs_g REAL,
|
||||||
|
sugar_g REAL,
|
||||||
|
fiber_g REAL,
|
||||||
|
protein_g REAL,
|
||||||
|
sodium_mg REAL,
|
||||||
|
ingredient_names TEXT NOT NULL DEFAULT '[]', -- JSON array
|
||||||
|
allergens TEXT NOT NULL DEFAULT '[]', -- JSON array
|
||||||
|
confidence REAL,
|
||||||
|
source TEXT NOT NULL DEFAULT 'visual_capture',
|
||||||
|
captured_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
confirmed_by_user INTEGER NOT NULL DEFAULT 0
|
||||||
|
);
|
||||||
|
|
@ -60,7 +60,9 @@ class Store:
|
||||||
# saved recipe columns
|
# saved recipe columns
|
||||||
"style_tags",
|
"style_tags",
|
||||||
# meal plan columns
|
# meal plan columns
|
||||||
"meal_types"):
|
"meal_types",
|
||||||
|
# captured_products columns
|
||||||
|
"allergens"):
|
||||||
if key in d and isinstance(d[key], str):
|
if key in d and isinstance(d[key], str):
|
||||||
try:
|
try:
|
||||||
d[key] = json.loads(d[key])
|
d[key] = json.loads(d[key])
|
||||||
|
|
@ -1639,3 +1641,73 @@ class Store:
|
||||||
cur = self.conn.execute("DELETE FROM shopping_list_items")
|
cur = self.conn.execute("DELETE FROM shopping_list_items")
|
||||||
self.conn.commit()
|
self.conn.commit()
|
||||||
return cur.rowcount
|
return cur.rowcount
|
||||||
|
|
||||||
|
# ── Captured products (visual label cache) ────────────────────────────────
|
||||||
|
|
||||||
|
def get_captured_product(self, barcode: str) -> dict | None:
|
||||||
|
"""Look up a locally-captured product by barcode.
|
||||||
|
|
||||||
|
Returns the row dict (ingredient_names and allergens already decoded as
|
||||||
|
lists) or None if the barcode has not been captured yet.
|
||||||
|
"""
|
||||||
|
return self._fetch_one(
|
||||||
|
"SELECT * FROM captured_products WHERE barcode = ?", (barcode,)
|
||||||
|
)
|
||||||
|
|
||||||
|
def save_captured_product(
|
||||||
|
self,
|
||||||
|
barcode: str,
|
||||||
|
*,
|
||||||
|
product_name: str | None = None,
|
||||||
|
brand: str | None = None,
|
||||||
|
serving_size_g: float | None = None,
|
||||||
|
calories: float | None = None,
|
||||||
|
fat_g: float | None = None,
|
||||||
|
saturated_fat_g: float | None = None,
|
||||||
|
carbs_g: float | None = None,
|
||||||
|
sugar_g: float | None = None,
|
||||||
|
fiber_g: float | None = None,
|
||||||
|
protein_g: float | None = None,
|
||||||
|
sodium_mg: float | None = None,
|
||||||
|
ingredient_names: list[str] | None = None,
|
||||||
|
allergens: list[str] | None = None,
|
||||||
|
confidence: float | None = None,
|
||||||
|
confirmed_by_user: bool = True,
|
||||||
|
source: str = "visual_capture",
|
||||||
|
) -> dict:
|
||||||
|
"""Insert or replace a captured product row, returning the saved dict."""
|
||||||
|
return self._insert_returning(
|
||||||
|
"""INSERT INTO captured_products
|
||||||
|
(barcode, product_name, brand, serving_size_g, calories,
|
||||||
|
fat_g, saturated_fat_g, carbs_g, sugar_g, fiber_g,
|
||||||
|
protein_g, sodium_mg, ingredient_names, allergens,
|
||||||
|
confidence, confirmed_by_user, source)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(barcode) DO UPDATE SET
|
||||||
|
product_name = excluded.product_name,
|
||||||
|
brand = excluded.brand,
|
||||||
|
serving_size_g = excluded.serving_size_g,
|
||||||
|
calories = excluded.calories,
|
||||||
|
fat_g = excluded.fat_g,
|
||||||
|
saturated_fat_g = excluded.saturated_fat_g,
|
||||||
|
carbs_g = excluded.carbs_g,
|
||||||
|
sugar_g = excluded.sugar_g,
|
||||||
|
fiber_g = excluded.fiber_g,
|
||||||
|
protein_g = excluded.protein_g,
|
||||||
|
sodium_mg = excluded.sodium_mg,
|
||||||
|
ingredient_names = excluded.ingredient_names,
|
||||||
|
allergens = excluded.allergens,
|
||||||
|
confidence = excluded.confidence,
|
||||||
|
confirmed_by_user = excluded.confirmed_by_user,
|
||||||
|
source = excluded.source,
|
||||||
|
captured_at = datetime('now')
|
||||||
|
RETURNING *""",
|
||||||
|
(
|
||||||
|
barcode, product_name, brand, serving_size_g, calories,
|
||||||
|
fat_g, saturated_fat_g, carbs_g, sugar_g, fiber_g,
|
||||||
|
protein_g, sodium_mg,
|
||||||
|
self._dump(ingredient_names or []),
|
||||||
|
self._dump(allergens or []),
|
||||||
|
confidence, 1 if confirmed_by_user else 0, source,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -142,6 +142,7 @@ class BarcodeScanResult(BaseModel):
|
||||||
inventory_item: Optional[InventoryItemResponse]
|
inventory_item: Optional[InventoryItemResponse]
|
||||||
added_to_inventory: bool
|
added_to_inventory: bool
|
||||||
needs_manual_entry: bool = False
|
needs_manual_entry: bool = False
|
||||||
|
needs_visual_capture: bool = False # Paid tier offer when no product data found
|
||||||
message: str
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
59
app/models/schemas/label_capture.py
Normal file
59
app/models/schemas/label_capture.py
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
"""Pydantic schemas for visual label capture (kiwi#79)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
class LabelCaptureResponse(BaseModel):
|
||||||
|
"""Extraction result returned after the user photographs a nutrition label."""
|
||||||
|
barcode: str
|
||||||
|
product_name: Optional[str] = None
|
||||||
|
brand: Optional[str] = None
|
||||||
|
serving_size_g: Optional[float] = None
|
||||||
|
calories: Optional[float] = None
|
||||||
|
fat_g: Optional[float] = None
|
||||||
|
saturated_fat_g: Optional[float] = None
|
||||||
|
carbs_g: Optional[float] = None
|
||||||
|
sugar_g: Optional[float] = None
|
||||||
|
fiber_g: Optional[float] = None
|
||||||
|
protein_g: Optional[float] = None
|
||||||
|
sodium_mg: Optional[float] = None
|
||||||
|
ingredient_names: List[str] = Field(default_factory=list)
|
||||||
|
allergens: List[str] = Field(default_factory=list)
|
||||||
|
confidence: float = 0.0
|
||||||
|
needs_review: bool = True # True when confidence < REVIEW_THRESHOLD
|
||||||
|
|
||||||
|
|
||||||
|
class LabelConfirmRequest(BaseModel):
|
||||||
|
"""User-confirmed extraction to save to the local product cache."""
|
||||||
|
barcode: str
|
||||||
|
product_name: Optional[str] = None
|
||||||
|
brand: Optional[str] = None
|
||||||
|
serving_size_g: Optional[float] = None
|
||||||
|
calories: Optional[float] = None
|
||||||
|
fat_g: Optional[float] = None
|
||||||
|
saturated_fat_g: Optional[float] = None
|
||||||
|
carbs_g: Optional[float] = None
|
||||||
|
sugar_g: Optional[float] = None
|
||||||
|
fiber_g: Optional[float] = None
|
||||||
|
protein_g: Optional[float] = None
|
||||||
|
sodium_mg: Optional[float] = None
|
||||||
|
ingredient_names: List[str] = Field(default_factory=list)
|
||||||
|
allergens: List[str] = Field(default_factory=list)
|
||||||
|
confidence: float = 0.0
|
||||||
|
# When True the confirmed product is also added to inventory
|
||||||
|
location: str = "pantry"
|
||||||
|
quantity: float = 1.0
|
||||||
|
auto_add: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class LabelConfirmResponse(BaseModel):
|
||||||
|
"""Result of confirming a captured product."""
|
||||||
|
ok: bool
|
||||||
|
barcode: str
|
||||||
|
product_id: Optional[int] = None
|
||||||
|
inventory_item_id: Optional[int] = None
|
||||||
|
message: str
|
||||||
140
app/services/label_capture.py
Normal file
140
app/services/label_capture.py
Normal file
|
|
@ -0,0 +1,140 @@
|
||||||
|
"""Visual label capture service for unenriched products (kiwi#79).
|
||||||
|
|
||||||
|
Wraps the cf-core VisionRouter to extract structured nutrition data from a
|
||||||
|
photographed nutrition facts panel. When the VisionRouter is not yet wired
|
||||||
|
(NotImplementedError) the service falls back to a mock extraction so the
|
||||||
|
barcode scan flow can be exercised end-to-end in development.
|
||||||
|
|
||||||
|
JSON contract returned by the vision model (and mock):
|
||||||
|
{
|
||||||
|
"product_name": str | null,
|
||||||
|
"brand": str | null,
|
||||||
|
"serving_size_g": number | null,
|
||||||
|
"calories": number | null,
|
||||||
|
"fat_g": number | null,
|
||||||
|
"saturated_fat_g": number | null,
|
||||||
|
"carbs_g": number | null,
|
||||||
|
"sugar_g": number | null,
|
||||||
|
"fiber_g": number | null,
|
||||||
|
"protein_g": number | null,
|
||||||
|
"sodium_mg": number | null,
|
||||||
|
"ingredient_names": [str],
|
||||||
|
"allergens": [str],
|
||||||
|
"confidence": number (0.0–1.0)
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Confidence below this threshold surfaces amber highlights in the UI.
|
||||||
|
REVIEW_THRESHOLD = 0.7
|
||||||
|
|
||||||
|
_MOCK_EXTRACTION: dict[str, Any] = {
|
||||||
|
"product_name": "Unknown Product",
|
||||||
|
"brand": None,
|
||||||
|
"serving_size_g": None,
|
||||||
|
"calories": None,
|
||||||
|
"fat_g": None,
|
||||||
|
"saturated_fat_g": None,
|
||||||
|
"carbs_g": None,
|
||||||
|
"sugar_g": None,
|
||||||
|
"fiber_g": None,
|
||||||
|
"protein_g": None,
|
||||||
|
"sodium_mg": None,
|
||||||
|
"ingredient_names": [],
|
||||||
|
"allergens": [],
|
||||||
|
"confidence": 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
_EXTRACTION_PROMPT = """You are reading a nutrition facts label photograph.
|
||||||
|
Extract the following fields as a JSON object with no extra text:
|
||||||
|
|
||||||
|
{
|
||||||
|
"product_name": <product name or null>,
|
||||||
|
"brand": <brand name or null>,
|
||||||
|
"serving_size_g": <serving size in grams as a number or null>,
|
||||||
|
"calories": <calories per serving as a number or null>,
|
||||||
|
"fat_g": <total fat grams or null>,
|
||||||
|
"saturated_fat_g": <saturated fat grams or null>,
|
||||||
|
"carbs_g": <total carbohydrates grams or null>,
|
||||||
|
"sugar_g": <sugars grams or null>,
|
||||||
|
"fiber_g": <dietary fiber grams or null>,
|
||||||
|
"protein_g": <protein grams or null>,
|
||||||
|
"sodium_mg": <sodium milligrams or null>,
|
||||||
|
"ingredient_names": [list of individual ingredients as strings],
|
||||||
|
"allergens": [list of allergens explicitly stated on label],
|
||||||
|
"confidence": <your confidence this extraction is correct, 0.0 to 1.0>
|
||||||
|
}
|
||||||
|
|
||||||
|
Use null for any field you cannot read clearly. Do not guess values.
|
||||||
|
Respond with JSON only."""
|
||||||
|
|
||||||
|
|
||||||
|
def extract_label(image_bytes: bytes) -> dict[str, Any]:
|
||||||
|
"""Run vision model extraction on raw label image bytes.
|
||||||
|
|
||||||
|
Returns a dict matching the nutrition JSON contract above.
|
||||||
|
Falls back to a zero-confidence mock if the VisionRouter is not yet
|
||||||
|
implemented (stub) or if the model returns unparseable output.
|
||||||
|
"""
|
||||||
|
# Allow unit tests to bypass the vision model entirely.
|
||||||
|
if os.environ.get("KIWI_LABEL_CAPTURE_MOCK") == "1":
|
||||||
|
log.debug("label_capture: mock mode active")
|
||||||
|
return dict(_MOCK_EXTRACTION)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from circuitforge_core.vision import caption as vision_caption
|
||||||
|
result = vision_caption(image_bytes, prompt=_EXTRACTION_PROMPT)
|
||||||
|
raw = result.caption or ""
|
||||||
|
return _parse_extraction(raw)
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("label_capture: extraction failed (%s) — returning mock extraction", exc)
|
||||||
|
return dict(_MOCK_EXTRACTION)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_extraction(raw: str) -> dict[str, Any]:
|
||||||
|
"""Parse the JSON string returned by the vision model.
|
||||||
|
|
||||||
|
Strips markdown code fences if present. Validates required shape.
|
||||||
|
Returns the mock on any parse error.
|
||||||
|
"""
|
||||||
|
text = raw.strip()
|
||||||
|
if text.startswith("```"):
|
||||||
|
# Strip ```json ... ``` fences
|
||||||
|
lines = text.splitlines()
|
||||||
|
text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = json.loads(text)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
log.warning("label_capture: could not parse vision response: %s", exc)
|
||||||
|
return dict(_MOCK_EXTRACTION)
|
||||||
|
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
log.warning("label_capture: vision response is not a dict")
|
||||||
|
return dict(_MOCK_EXTRACTION)
|
||||||
|
|
||||||
|
# Normalise list fields — model may return None instead of []
|
||||||
|
for list_key in ("ingredient_names", "allergens"):
|
||||||
|
if not isinstance(data.get(list_key), list):
|
||||||
|
data[list_key] = []
|
||||||
|
|
||||||
|
# Clamp confidence to [0, 1]
|
||||||
|
confidence = data.get("confidence")
|
||||||
|
if not isinstance(confidence, (int, float)):
|
||||||
|
confidence = 0.0
|
||||||
|
data["confidence"] = max(0.0, min(1.0, float(confidence)))
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def needs_review(extraction: dict[str, Any]) -> bool:
|
||||||
|
"""Return True when the extraction confidence is below REVIEW_THRESHOLD."""
|
||||||
|
return float(extraction.get("confidence", 0.0)) < REVIEW_THRESHOLD
|
||||||
|
|
@ -44,6 +44,7 @@ KIWI_FEATURES: dict[str, str] = {
|
||||||
|
|
||||||
# Paid tier
|
# Paid tier
|
||||||
"receipt_ocr": "paid", # BYOK-unlockable
|
"receipt_ocr": "paid", # BYOK-unlockable
|
||||||
|
"visual_label_capture": "paid", # Camera capture for unenriched barcodes (kiwi#79)
|
||||||
"recipe_suggestions": "paid", # BYOK-unlockable
|
"recipe_suggestions": "paid", # BYOK-unlockable
|
||||||
"expiry_llm_matching": "paid", # BYOK-unlockable
|
"expiry_llm_matching": "paid", # BYOK-unlockable
|
||||||
"meal_planning": "free",
|
"meal_planning": "free",
|
||||||
|
|
|
||||||
|
|
@ -138,6 +138,103 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Label Capture Panel (paid tier — appears after gap detection) -->
|
||||||
|
<div v-if="capturePhase !== null" class="label-capture-panel">
|
||||||
|
|
||||||
|
<!-- Offer phase -->
|
||||||
|
<div v-if="capturePhase === 'offer'" class="capture-offer">
|
||||||
|
<p class="capture-offer-text">We couldn't find this product. Photograph the nutrition label to add it.</p>
|
||||||
|
<div class="capture-offer-actions">
|
||||||
|
<button class="btn btn-primary" type="button" @click="triggerCaptureLabelInput">
|
||||||
|
Capture label
|
||||||
|
</button>
|
||||||
|
<button class="btn btn-ghost" type="button" @click="dismissCapture">
|
||||||
|
Skip
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
ref="captureFileInput"
|
||||||
|
type="file"
|
||||||
|
accept="image/*"
|
||||||
|
capture="environment"
|
||||||
|
style="display: none"
|
||||||
|
@change="handleLabelPhotoSelect"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Uploading / processing phase -->
|
||||||
|
<div v-else-if="capturePhase === 'uploading'" class="capture-processing">
|
||||||
|
<div class="loading-inline">
|
||||||
|
<div class="spinner spinner-sm"></div>
|
||||||
|
<span>Reading the label…</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Review phase -->
|
||||||
|
<div v-else-if="capturePhase === 'reviewing' && captureExtraction" class="capture-review">
|
||||||
|
<p class="capture-review-note">
|
||||||
|
Check the details below.
|
||||||
|
<span v-if="captureExtraction.needs_review" class="capture-review-low-conf">
|
||||||
|
Fields highlighted in amber weren't fully legible — please verify them.
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div class="form-row">
|
||||||
|
<div class="form-group">
|
||||||
|
<label class="form-label">Product name</label>
|
||||||
|
<input v-model="captureReview.product_name" type="text" class="form-input" placeholder="Product name" />
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label class="form-label">Brand</label>
|
||||||
|
<input v-model="captureReview.brand" type="text" class="form-input" placeholder="Brand (optional)" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p class="form-section-label">Nutrition per serving</p>
|
||||||
|
<div class="capture-nutrition-grid">
|
||||||
|
<div
|
||||||
|
v-for="field in captureNutritionFields"
|
||||||
|
:key="field.key"
|
||||||
|
class="form-group"
|
||||||
|
>
|
||||||
|
<label
|
||||||
|
:class="['form-label', { 'capture-field-amber': captureExtraction.needs_review && captureExtraction[field.src as keyof typeof captureExtraction] == null }]"
|
||||||
|
>{{ field.label }}</label>
|
||||||
|
<input
|
||||||
|
v-model="captureReview[field.key as keyof typeof captureReview]"
|
||||||
|
type="number"
|
||||||
|
min="0"
|
||||||
|
step="0.1"
|
||||||
|
class="form-input"
|
||||||
|
:placeholder="field.unit"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group" style="margin-top: var(--spacing-sm)">
|
||||||
|
<label class="form-label">Ingredients (comma-separated)</label>
|
||||||
|
<input v-model="captureReview.ingredients" type="text" class="form-input" placeholder="flour, water, salt…" />
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label class="form-label">Allergens (comma-separated)</label>
|
||||||
|
<input v-model="captureReview.allergens" type="text" class="form-input" placeholder="wheat, milk…" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="capture-review-actions">
|
||||||
|
<button class="btn btn-primary" type="button" :disabled="captureLoading" @click="confirmCapture">
|
||||||
|
<span v-if="captureLoading"><div class="spinner spinner-sm"></div></span>
|
||||||
|
<span v-else>Looks good — save</span>
|
||||||
|
</button>
|
||||||
|
<button class="btn btn-ghost" type="button" @click="capturePhase = 'offer'">
|
||||||
|
Retake photo
|
||||||
|
</button>
|
||||||
|
<button class="btn btn-ghost" type="button" @click="dismissCapture">
|
||||||
|
Discard
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Camera Scan Panel -->
|
<!-- Camera Scan Panel -->
|
||||||
<div v-if="scanMode === 'camera'" class="scan-panel">
|
<div v-if="scanMode === 'camera'" class="scan-panel">
|
||||||
<div class="upload-area" @click="triggerBarcodeInput">
|
<div class="upload-area" @click="triggerBarcodeInput">
|
||||||
|
|
@ -622,7 +719,7 @@ import { storeToRefs } from 'pinia'
|
||||||
import { useInventoryStore } from '../stores/inventory'
|
import { useInventoryStore } from '../stores/inventory'
|
||||||
import { useSettingsStore } from '../stores/settings'
|
import { useSettingsStore } from '../stores/settings'
|
||||||
import { inventoryAPI } from '../services/api'
|
import { inventoryAPI } from '../services/api'
|
||||||
import type { InventoryItem } from '../services/api'
|
import type { InventoryItem, LabelCaptureResult } from '../services/api'
|
||||||
import { formatQuantity } from '../utils/units'
|
import { formatQuantity } from '../utils/units'
|
||||||
import EditItemModal from './EditItemModal.vue'
|
import EditItemModal from './EditItemModal.vue'
|
||||||
import ConfirmDialog from './ConfirmDialog.vue'
|
import ConfirmDialog from './ConfirmDialog.vue'
|
||||||
|
|
@ -684,6 +781,16 @@ function daysLabel(dateStr: string): string {
|
||||||
const scanMode = ref<'gun' | 'camera' | 'manual'>('gun')
|
const scanMode = ref<'gun' | 'camera' | 'manual'>('gun')
|
||||||
|
|
||||||
// Options for button groups
|
// Options for button groups
|
||||||
|
// Label capture nutrition field descriptors used in the review form
|
||||||
|
const captureNutritionFields = [
|
||||||
|
{ key: 'calories', src: 'calories', label: 'Calories', unit: 'kcal' },
|
||||||
|
{ key: 'fat_g', src: 'fat_g', label: 'Total fat', unit: 'g' },
|
||||||
|
{ key: 'saturated_fat_g', src: 'saturated_fat_g', label: 'Saturated fat', unit: 'g' },
|
||||||
|
{ key: 'carbs_g', src: 'carbs_g', label: 'Carbs', unit: 'g' },
|
||||||
|
{ key: 'protein_g', src: 'protein_g', label: 'Protein', unit: 'g' },
|
||||||
|
{ key: 'sodium_mg', src: 'sodium_mg', label: 'Sodium', unit: 'mg' },
|
||||||
|
]
|
||||||
|
|
||||||
const locations = [
|
const locations = [
|
||||||
{ value: 'fridge', label: 'Fridge', icon: '🧊' },
|
{ value: 'fridge', label: 'Fridge', icon: '🧊' },
|
||||||
{ value: 'freezer', label: 'Freezer', icon: '❄️' },
|
{ value: 'freezer', label: 'Freezer', icon: '❄️' },
|
||||||
|
|
@ -780,6 +887,29 @@ const barcodeQuantity = ref(1)
|
||||||
const barcodeLoading = ref(false)
|
const barcodeLoading = ref(false)
|
||||||
const barcodeResults = ref<Array<{ type: string; message: string }>>([])
|
const barcodeResults = ref<Array<{ type: string; message: string }>>([])
|
||||||
|
|
||||||
|
// Label Capture Flow (kiwi#79)
|
||||||
|
type CapturePhase = 'offer' | 'uploading' | 'reviewing' | null
|
||||||
|
const capturePhase = ref<CapturePhase>(null)
|
||||||
|
const captureBarcode = ref('')
|
||||||
|
const captureLocation = ref('pantry')
|
||||||
|
const captureQuantity = ref(1)
|
||||||
|
const captureLoading = ref(false)
|
||||||
|
const captureFileInput = ref<HTMLInputElement | null>(null)
|
||||||
|
const captureExtraction = ref<LabelCaptureResult | null>(null)
|
||||||
|
// Editable review form — populated from extraction, user may correct fields
|
||||||
|
const captureReview = ref({
|
||||||
|
product_name: '',
|
||||||
|
brand: '',
|
||||||
|
calories: '' as string,
|
||||||
|
fat_g: '' as string,
|
||||||
|
saturated_fat_g: '' as string,
|
||||||
|
carbs_g: '' as string,
|
||||||
|
protein_g: '' as string,
|
||||||
|
sodium_mg: '' as string,
|
||||||
|
ingredients: '',
|
||||||
|
allergens: '',
|
||||||
|
})
|
||||||
|
|
||||||
// Manual Form
|
// Manual Form
|
||||||
const manualForm = ref({
|
const manualForm = ref({
|
||||||
name: '',
|
name: '',
|
||||||
|
|
@ -935,6 +1065,15 @@ async function handleScannerGunInput() {
|
||||||
message: `Added: ${productName}${productBrand} to ${scannerLocation.value}`,
|
message: `Added: ${productName}${productBrand} to ${scannerLocation.value}`,
|
||||||
})
|
})
|
||||||
await refreshItems()
|
await refreshItems()
|
||||||
|
} else if (item?.needs_visual_capture) {
|
||||||
|
captureBarcode.value = barcode
|
||||||
|
captureLocation.value = scannerLocation.value
|
||||||
|
captureQuantity.value = scannerQuantity.value
|
||||||
|
capturePhase.value = 'offer'
|
||||||
|
scannerResults.value.push({
|
||||||
|
type: 'info',
|
||||||
|
message: item.message,
|
||||||
|
})
|
||||||
} else if (item?.needs_manual_entry) {
|
} else if (item?.needs_manual_entry) {
|
||||||
// Barcode not found in any database — guide user to manual entry
|
// Barcode not found in any database — guide user to manual entry
|
||||||
scannerResults.value.push({
|
scannerResults.value.push({
|
||||||
|
|
@ -1007,6 +1146,88 @@ async function handleBarcodeImageSelect(e: Event) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Label Capture Functions
|
||||||
|
|
||||||
|
function triggerCaptureLabelInput() {
|
||||||
|
captureFileInput.value?.click()
|
||||||
|
}
|
||||||
|
|
||||||
|
function dismissCapture() {
|
||||||
|
capturePhase.value = null
|
||||||
|
captureBarcode.value = ''
|
||||||
|
captureExtraction.value = null
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleLabelPhotoSelect(e: Event) {
|
||||||
|
const target = e.target as HTMLInputElement
|
||||||
|
const file = target.files?.[0]
|
||||||
|
if (!file) return
|
||||||
|
|
||||||
|
captureLoading.value = true
|
||||||
|
capturePhase.value = 'uploading'
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await inventoryAPI.captureLabelPhoto(file, captureBarcode.value)
|
||||||
|
captureExtraction.value = result
|
||||||
|
// Pre-populate the review form with extracted values
|
||||||
|
captureReview.value = {
|
||||||
|
product_name: result.product_name || '',
|
||||||
|
brand: result.brand || '',
|
||||||
|
calories: result.calories != null ? String(result.calories) : '',
|
||||||
|
fat_g: result.fat_g != null ? String(result.fat_g) : '',
|
||||||
|
saturated_fat_g: result.saturated_fat_g != null ? String(result.saturated_fat_g) : '',
|
||||||
|
carbs_g: result.carbs_g != null ? String(result.carbs_g) : '',
|
||||||
|
protein_g: result.protein_g != null ? String(result.protein_g) : '',
|
||||||
|
sodium_mg: result.sodium_mg != null ? String(result.sodium_mg) : '',
|
||||||
|
ingredients: (result.ingredient_names || []).join(', '),
|
||||||
|
allergens: (result.allergens || []).join(', '),
|
||||||
|
}
|
||||||
|
capturePhase.value = 'reviewing'
|
||||||
|
} catch {
|
||||||
|
showToast('Could not read the label. Please try again or add manually.', 'error')
|
||||||
|
capturePhase.value = 'offer'
|
||||||
|
} finally {
|
||||||
|
captureLoading.value = false
|
||||||
|
if (target) target.value = ''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function confirmCapture() {
|
||||||
|
if (!captureBarcode.value) return
|
||||||
|
|
||||||
|
captureLoading.value = true
|
||||||
|
try {
|
||||||
|
const toNum = (s: string) => s ? parseFloat(s) || null : null
|
||||||
|
const toList = (s: string) => s.split(',').map(x => x.trim()).filter(Boolean)
|
||||||
|
|
||||||
|
await inventoryAPI.confirmLabelCapture({
|
||||||
|
barcode: captureBarcode.value,
|
||||||
|
product_name: captureReview.value.product_name || null,
|
||||||
|
brand: captureReview.value.brand || null,
|
||||||
|
calories: toNum(captureReview.value.calories),
|
||||||
|
fat_g: toNum(captureReview.value.fat_g),
|
||||||
|
saturated_fat_g: toNum(captureReview.value.saturated_fat_g),
|
||||||
|
carbs_g: toNum(captureReview.value.carbs_g),
|
||||||
|
protein_g: toNum(captureReview.value.protein_g),
|
||||||
|
sodium_mg: toNum(captureReview.value.sodium_mg),
|
||||||
|
ingredient_names: toList(captureReview.value.ingredients),
|
||||||
|
allergens: toList(captureReview.value.allergens),
|
||||||
|
confidence: captureExtraction.value?.confidence ?? 0,
|
||||||
|
location: captureLocation.value,
|
||||||
|
quantity: captureQuantity.value,
|
||||||
|
auto_add: true,
|
||||||
|
})
|
||||||
|
const name = captureReview.value.product_name || 'item'
|
||||||
|
showToast(`${name} saved and added to ${captureLocation.value}`, 'success')
|
||||||
|
await refreshItems()
|
||||||
|
dismissCapture()
|
||||||
|
} catch {
|
||||||
|
showToast('Could not save. Please try again.', 'error')
|
||||||
|
} finally {
|
||||||
|
captureLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Manual Add Functions
|
// Manual Add Functions
|
||||||
async function addManualItem() {
|
async function addManualItem() {
|
||||||
const { name, brand, quantity, unit, location, expirationDate } = manualForm.value
|
const { name, brand, quantity, unit, location, expirationDate } = manualForm.value
|
||||||
|
|
@ -1614,6 +1835,79 @@ function getItemClass(item: InventoryItem): string {
|
||||||
border: 1px solid var(--color-warning-border, #fcd34d);
|
border: 1px solid var(--color-warning-border, #fcd34d);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ============================================
|
||||||
|
LABEL CAPTURE FLOW (kiwi#79)
|
||||||
|
============================================ */
|
||||||
|
.label-capture-panel {
|
||||||
|
margin: var(--spacing-md) 0;
|
||||||
|
padding: var(--spacing-md);
|
||||||
|
background: var(--color-surface);
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-lg);
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-offer-text {
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
color: var(--color-text-secondary);
|
||||||
|
margin: 0 0 var(--spacing-md);
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-offer-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: var(--spacing-sm);
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-processing {
|
||||||
|
display: flex;
|
||||||
|
justify-content: center;
|
||||||
|
padding: var(--spacing-md) 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-review-note {
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
color: var(--color-text-secondary);
|
||||||
|
margin: 0 0 var(--spacing-md);
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-review-low-conf {
|
||||||
|
color: var(--color-amber, #d97706);
|
||||||
|
font-size: var(--font-size-xs);
|
||||||
|
display: block;
|
||||||
|
margin-top: var(--spacing-xs);
|
||||||
|
}
|
||||||
|
|
||||||
|
.form-section-label {
|
||||||
|
font-size: var(--font-size-xs);
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--color-text-tertiary);
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.05em;
|
||||||
|
margin: var(--spacing-md) 0 var(--spacing-sm);
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-nutrition-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fill, minmax(120px, 1fr));
|
||||||
|
gap: var(--spacing-sm);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Amber highlight for unread/low-confidence label fields */
|
||||||
|
.capture-field-amber {
|
||||||
|
color: var(--color-amber, #d97706);
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-field-amber + input {
|
||||||
|
border-color: var(--color-amber, #d97706);
|
||||||
|
}
|
||||||
|
|
||||||
|
.capture-review-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: var(--spacing-sm);
|
||||||
|
flex-wrap: wrap;
|
||||||
|
margin-top: var(--spacing-md);
|
||||||
|
}
|
||||||
|
|
||||||
/* ============================================
|
/* ============================================
|
||||||
EXPORT CARD
|
EXPORT CARD
|
||||||
============================================ */
|
============================================ */
|
||||||
|
|
|
||||||
|
|
@ -111,9 +111,50 @@ export interface BarcodeScanResult {
|
||||||
inventory_item: InventoryItem | null
|
inventory_item: InventoryItem | null
|
||||||
added_to_inventory: boolean
|
added_to_inventory: boolean
|
||||||
needs_manual_entry: boolean
|
needs_manual_entry: boolean
|
||||||
|
needs_visual_capture: boolean
|
||||||
message: string
|
message: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface LabelCaptureResult {
|
||||||
|
barcode: string
|
||||||
|
product_name: string | null
|
||||||
|
brand: string | null
|
||||||
|
serving_size_g: number | null
|
||||||
|
calories: number | null
|
||||||
|
fat_g: number | null
|
||||||
|
saturated_fat_g: number | null
|
||||||
|
carbs_g: number | null
|
||||||
|
sugar_g: number | null
|
||||||
|
fiber_g: number | null
|
||||||
|
protein_g: number | null
|
||||||
|
sodium_mg: number | null
|
||||||
|
ingredient_names: string[]
|
||||||
|
allergens: string[]
|
||||||
|
confidence: number
|
||||||
|
needs_review: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LabelConfirmRequest {
|
||||||
|
barcode: string
|
||||||
|
product_name?: string | null
|
||||||
|
brand?: string | null
|
||||||
|
serving_size_g?: number | null
|
||||||
|
calories?: number | null
|
||||||
|
fat_g?: number | null
|
||||||
|
saturated_fat_g?: number | null
|
||||||
|
carbs_g?: number | null
|
||||||
|
sugar_g?: number | null
|
||||||
|
fiber_g?: number | null
|
||||||
|
protein_g?: number | null
|
||||||
|
sodium_mg?: number | null
|
||||||
|
ingredient_names?: string[]
|
||||||
|
allergens?: string[]
|
||||||
|
confidence?: number
|
||||||
|
location?: string
|
||||||
|
quantity?: number
|
||||||
|
auto_add?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
export interface BarcodeScanResponse {
|
export interface BarcodeScanResponse {
|
||||||
success: boolean
|
success: boolean
|
||||||
barcodes_found: number
|
barcodes_found: number
|
||||||
|
|
@ -344,6 +385,32 @@ export const inventoryAPI = {
|
||||||
})
|
})
|
||||||
return response.data
|
return response.data
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload a nutrition label photo for an unenriched barcode (paid tier).
|
||||||
|
* Returns extracted fields + confidence score for user review.
|
||||||
|
*/
|
||||||
|
async captureLabelPhoto(
|
||||||
|
file: File,
|
||||||
|
barcode: string
|
||||||
|
): Promise<LabelCaptureResult> {
|
||||||
|
const formData = new FormData()
|
||||||
|
formData.append('file', file)
|
||||||
|
formData.append('barcode', barcode)
|
||||||
|
const response = await api.post('/inventory/scan/label-capture', formData, {
|
||||||
|
headers: { 'Content-Type': 'multipart/form-data' },
|
||||||
|
timeout: 60000, // vision inference can take ~5–10s
|
||||||
|
})
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Confirm a user-reviewed label extraction and save to the local cache.
|
||||||
|
*/
|
||||||
|
async confirmLabelCapture(data: LabelConfirmRequest): Promise<{ ok: boolean; product_id?: number; inventory_item_id?: number; message: string }> {
|
||||||
|
const response = await api.post('/inventory/scan/label-confirm', data)
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// ========== Receipts API ==========
|
// ========== Receipts API ==========
|
||||||
|
|
|
||||||
270
tests/api/test_label_capture.py
Normal file
270
tests/api/test_label_capture.py
Normal file
|
|
@ -0,0 +1,270 @@
|
||||||
|
"""
|
||||||
|
Tests for the visual label capture API endpoints (kiwi#79):
|
||||||
|
POST /api/v1/inventory/scan/label-capture
|
||||||
|
POST /api/v1/inventory/scan/label-confirm
|
||||||
|
GET /api/v1/inventory/scan/text — cache hit + needs_visual_capture flag
|
||||||
|
"""
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from app.main import app
|
||||||
|
from app.cloud_session import get_session
|
||||||
|
from app.db.session import get_store
|
||||||
|
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(tier: str = "paid", has_byok: bool = False) -> MagicMock:
|
||||||
|
m = MagicMock()
|
||||||
|
m.tier = tier
|
||||||
|
m.has_byok = has_byok
|
||||||
|
m.user_id = "test-user"
|
||||||
|
return m
|
||||||
|
|
||||||
|
|
||||||
|
def _store(**extra_returns) -> MagicMock:
|
||||||
|
m = MagicMock()
|
||||||
|
m.get_setting.return_value = None
|
||||||
|
m.get_captured_product.return_value = None
|
||||||
|
m.save_captured_product.return_value = {"id": 1, "barcode": "1234567890", "confirmed_by_user": 1}
|
||||||
|
m.get_or_create_product.return_value = (
|
||||||
|
{"id": 10, "name": "Test Product", "barcode": "1234567890",
|
||||||
|
"brand": None, "category": None, "description": None,
|
||||||
|
"image_url": None, "nutrition_data": {}, "source": "visual_capture",
|
||||||
|
"created_at": "2026-01-01", "updated_at": "2026-01-01"},
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
m.add_inventory_item.return_value = {
|
||||||
|
"id": 99, "product_id": 10, "product_name": "Test Product",
|
||||||
|
"barcode": "1234567890", "category": None,
|
||||||
|
"quantity": 1.0, "unit": "count", "location": "pantry",
|
||||||
|
"sublocation": None, "purchase_date": None,
|
||||||
|
"expiration_date": None, "opened_date": None,
|
||||||
|
"opened_expiry_date": None, "secondary_state": None,
|
||||||
|
"secondary_uses": None, "secondary_warning": None,
|
||||||
|
"secondary_discard_signs": None, "status": "available",
|
||||||
|
"notes": None, "disposal_reason": None,
|
||||||
|
"source": "visual_capture", "created_at": "2026-01-01",
|
||||||
|
"updated_at": "2026-01-01",
|
||||||
|
}
|
||||||
|
for k, v in extra_returns.items():
|
||||||
|
setattr(m, k, v)
|
||||||
|
return m
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def mock_env(monkeypatch):
|
||||||
|
monkeypatch.setenv("KIWI_LABEL_CAPTURE_MOCK", "1")
|
||||||
|
yield
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
|
||||||
|
|
||||||
|
# ── /scan/label-capture ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
class TestLabelCaptureEndpoint:
|
||||||
|
def setup_method(self):
|
||||||
|
self.session = _session(tier="paid")
|
||||||
|
self.store = _store()
|
||||||
|
app.dependency_overrides[get_session] = lambda: self.session
|
||||||
|
app.dependency_overrides[get_store] = lambda: self.store
|
||||||
|
|
||||||
|
def teardown_method(self):
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
def test_returns_200_for_paid_tier(self):
|
||||||
|
resp = client.post(
|
||||||
|
"/api/v1/inventory/scan/label-capture",
|
||||||
|
data={"barcode": "1234567890"},
|
||||||
|
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
|
||||||
|
def test_response_contains_barcode(self):
|
||||||
|
resp = client.post(
|
||||||
|
"/api/v1/inventory/scan/label-capture",
|
||||||
|
data={"barcode": "5901234123457"},
|
||||||
|
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
|
||||||
|
)
|
||||||
|
data = resp.json()
|
||||||
|
assert data["barcode"] == "5901234123457"
|
||||||
|
|
||||||
|
def test_response_has_needs_review_field(self):
|
||||||
|
resp = client.post(
|
||||||
|
"/api/v1/inventory/scan/label-capture",
|
||||||
|
data={"barcode": "1234567890"},
|
||||||
|
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
|
||||||
|
)
|
||||||
|
data = resp.json()
|
||||||
|
assert "needs_review" in data
|
||||||
|
|
||||||
|
def test_mock_extraction_has_zero_confidence(self):
|
||||||
|
resp = client.post(
|
||||||
|
"/api/v1/inventory/scan/label-capture",
|
||||||
|
data={"barcode": "1234567890"},
|
||||||
|
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
|
||||||
|
)
|
||||||
|
data = resp.json()
|
||||||
|
assert data["confidence"] == 0.0
|
||||||
|
assert data["needs_review"] is True
|
||||||
|
|
||||||
|
def test_free_tier_returns_403(self):
|
||||||
|
app.dependency_overrides[get_session] = lambda: _session(tier="free")
|
||||||
|
resp = client.post(
|
||||||
|
"/api/v1/inventory/scan/label-capture",
|
||||||
|
data={"barcode": "1234567890"},
|
||||||
|
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 403
|
||||||
|
|
||||||
|
def test_local_tier_bypasses_gate(self):
|
||||||
|
app.dependency_overrides[get_session] = lambda: _session(tier="local")
|
||||||
|
resp = client.post(
|
||||||
|
"/api/v1/inventory/scan/label-capture",
|
||||||
|
data={"barcode": "1234567890"},
|
||||||
|
files={"file": ("label.jpg", io.BytesIO(b"fake_image"), "image/jpeg")},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
# ── /scan/label-confirm ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
class TestLabelConfirmEndpoint:
|
||||||
|
def setup_method(self):
|
||||||
|
self.session = _session(tier="paid")
|
||||||
|
self.store = _store()
|
||||||
|
app.dependency_overrides[get_session] = lambda: self.session
|
||||||
|
app.dependency_overrides[get_store] = lambda: self.store
|
||||||
|
|
||||||
|
def teardown_method(self):
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
def test_returns_200(self):
|
||||||
|
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
|
||||||
|
"barcode": "1234567890",
|
||||||
|
"product_name": "Test Crackers",
|
||||||
|
"calories": 120.0,
|
||||||
|
"ingredient_names": ["flour", "salt"],
|
||||||
|
"allergens": ["wheat"],
|
||||||
|
"confidence": 0.88,
|
||||||
|
"auto_add": True,
|
||||||
|
"location": "pantry",
|
||||||
|
"quantity": 1.0,
|
||||||
|
})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
|
||||||
|
def test_ok_true_in_response(self):
|
||||||
|
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
|
||||||
|
"barcode": "1234567890",
|
||||||
|
"auto_add": False,
|
||||||
|
})
|
||||||
|
assert resp.json()["ok"] is True
|
||||||
|
|
||||||
|
def test_save_captured_product_called(self):
|
||||||
|
client.post("/api/v1/inventory/scan/label-confirm", json={
|
||||||
|
"barcode": "1234567890",
|
||||||
|
"product_name": "Test",
|
||||||
|
"auto_add": False,
|
||||||
|
})
|
||||||
|
self.store.save_captured_product.assert_called_once()
|
||||||
|
call_kwargs = self.store.save_captured_product.call_args
|
||||||
|
assert call_kwargs[0][0] == "1234567890"
|
||||||
|
|
||||||
|
def test_auto_add_true_creates_inventory_item(self):
|
||||||
|
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
|
||||||
|
"barcode": "1234567890",
|
||||||
|
"auto_add": True,
|
||||||
|
})
|
||||||
|
data = resp.json()
|
||||||
|
assert data["inventory_item_id"] is not None
|
||||||
|
self.store.add_inventory_item.assert_called_once()
|
||||||
|
|
||||||
|
def test_auto_add_false_skips_inventory(self):
|
||||||
|
resp = client.post("/api/v1/inventory/scan/label-confirm", json={
|
||||||
|
"barcode": "1234567890",
|
||||||
|
"auto_add": False,
|
||||||
|
})
|
||||||
|
data = resp.json()
|
||||||
|
assert data["inventory_item_id"] is None
|
||||||
|
self.store.add_inventory_item.assert_not_called()
|
||||||
|
|
||||||
|
def test_free_tier_blocked(self):
|
||||||
|
app.dependency_overrides[get_session] = lambda: _session(tier="free")
|
||||||
|
resp = client.post("/api/v1/inventory/scan/label-confirm", json={"barcode": "1234567890"})
|
||||||
|
assert resp.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
# ── /scan/text cache hit + needs_visual_capture ───────────────────────────────
|
||||||
|
|
||||||
|
class TestScanTextWithCaptureGating:
|
||||||
|
def teardown_method(self):
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
def _setup(self, tier: str, cached=None, off_result=None):
|
||||||
|
store = _store()
|
||||||
|
store.get_captured_product.return_value = cached
|
||||||
|
session = _session(tier=tier)
|
||||||
|
app.dependency_overrides[get_session] = lambda: session
|
||||||
|
app.dependency_overrides[get_store] = lambda: store
|
||||||
|
return store
|
||||||
|
|
||||||
|
def _off_patch(self, result):
|
||||||
|
"""Patch OpenFoodFactsService.lookup_product at the class level."""
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
return patch(
|
||||||
|
"app.services.openfoodfacts.OpenFoodFactsService.lookup_product",
|
||||||
|
new=AsyncMock(return_value=result),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_paid_tier_no_product_sets_needs_visual_capture(self):
|
||||||
|
self._setup(tier="paid")
|
||||||
|
with self._off_patch(None):
|
||||||
|
resp = client.post("/api/v1/inventory/scan/text", json={
|
||||||
|
"barcode": "0000000000000", "location": "pantry",
|
||||||
|
})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
result = resp.json()["results"][0]
|
||||||
|
assert result["needs_visual_capture"] is True
|
||||||
|
assert result["needs_manual_entry"] is False
|
||||||
|
|
||||||
|
def test_free_tier_no_product_sets_needs_manual_entry(self):
|
||||||
|
self._setup(tier="free")
|
||||||
|
with self._off_patch(None):
|
||||||
|
resp = client.post("/api/v1/inventory/scan/text", json={
|
||||||
|
"barcode": "0000000000000", "location": "pantry",
|
||||||
|
})
|
||||||
|
result = resp.json()["results"][0]
|
||||||
|
assert result["needs_visual_capture"] is False
|
||||||
|
assert result["needs_manual_entry"] is True
|
||||||
|
|
||||||
|
def test_cache_hit_uses_captured_product(self):
|
||||||
|
cached = {
|
||||||
|
"barcode": "9999999999999",
|
||||||
|
"product_name": "Cached Crackers",
|
||||||
|
"brand": "TestBrand",
|
||||||
|
"confirmed_by_user": 1,
|
||||||
|
"ingredient_names": ["flour"],
|
||||||
|
"allergens": [],
|
||||||
|
"calories": 110.0,
|
||||||
|
"fat_g": None, "saturated_fat_g": None, "carbs_g": None,
|
||||||
|
"sugar_g": None, "fiber_g": None, "protein_g": None,
|
||||||
|
"sodium_mg": None, "serving_size_g": None,
|
||||||
|
}
|
||||||
|
store = self._setup(tier="paid", cached=cached)
|
||||||
|
store.get_inventory_item.return_value = store.add_inventory_item.return_value
|
||||||
|
|
||||||
|
with self._off_patch(None): # OFF never called when cache hits
|
||||||
|
resp = client.post("/api/v1/inventory/scan/text", json={
|
||||||
|
"barcode": "9999999999999", "location": "pantry",
|
||||||
|
})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
result = resp.json()["results"][0]
|
||||||
|
assert result["added_to_inventory"] is True
|
||||||
|
assert result["needs_visual_capture"] is False
|
||||||
|
# OFF was not called (cache resolved it)
|
||||||
|
# store.get_captured_product was called with the barcode
|
||||||
|
store.get_captured_product.assert_called_once_with("9999999999999")
|
||||||
116
tests/db/test_captured_products.py
Normal file
116
tests/db/test_captured_products.py
Normal file
|
|
@ -0,0 +1,116 @@
|
||||||
|
"""Tests for captured_products store methods (kiwi#79)."""
|
||||||
|
import pytest
|
||||||
|
from pathlib import Path
|
||||||
|
from app.db.store import Store
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def store(tmp_path: Path) -> Store:
|
||||||
|
s = Store(tmp_path / "test.db")
|
||||||
|
yield s
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
|
class TestMigration:
|
||||||
|
def test_captured_products_table_exists(self, store):
|
||||||
|
cur = store.conn.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table' AND name='captured_products'"
|
||||||
|
)
|
||||||
|
assert cur.fetchone() is not None
|
||||||
|
|
||||||
|
def test_captured_products_columns(self, store):
|
||||||
|
cur = store.conn.execute("PRAGMA table_info(captured_products)")
|
||||||
|
# PRAGMA returns plain tuples: (cid, name, type, notnull, dflt_value, pk)
|
||||||
|
cols = {row[1] for row in cur.fetchall()}
|
||||||
|
expected = {
|
||||||
|
"id", "barcode", "product_name", "brand", "serving_size_g",
|
||||||
|
"calories", "fat_g", "saturated_fat_g", "carbs_g", "sugar_g",
|
||||||
|
"fiber_g", "protein_g", "sodium_mg", "ingredient_names",
|
||||||
|
"allergens", "confidence", "source", "captured_at",
|
||||||
|
"confirmed_by_user",
|
||||||
|
}
|
||||||
|
assert expected.issubset(cols)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetCapturedProduct:
|
||||||
|
def test_returns_none_for_unknown_barcode(self, store):
|
||||||
|
assert store.get_captured_product("0000000000000") is None
|
||||||
|
|
||||||
|
def test_returns_row_after_save(self, store):
|
||||||
|
store.save_captured_product("1234567890123", product_name="Test Crackers")
|
||||||
|
result = store.get_captured_product("1234567890123")
|
||||||
|
assert result is not None
|
||||||
|
assert result["product_name"] == "Test Crackers"
|
||||||
|
|
||||||
|
def test_ingredient_names_decoded_as_list(self, store):
|
||||||
|
store.save_captured_product(
|
||||||
|
"1111111111111",
|
||||||
|
ingredient_names=["wheat flour", "salt"],
|
||||||
|
)
|
||||||
|
result = store.get_captured_product("1111111111111")
|
||||||
|
assert result["ingredient_names"] == ["wheat flour", "salt"]
|
||||||
|
|
||||||
|
def test_allergens_decoded_as_list(self, store):
|
||||||
|
store.save_captured_product(
|
||||||
|
"2222222222222",
|
||||||
|
allergens=["wheat", "milk"],
|
||||||
|
)
|
||||||
|
result = store.get_captured_product("2222222222222")
|
||||||
|
assert result["allergens"] == ["wheat", "milk"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestSaveCapturedProduct:
|
||||||
|
def test_all_nutrition_fields_persisted(self, store):
|
||||||
|
store.save_captured_product(
|
||||||
|
"3333333333333",
|
||||||
|
product_name="Oat Crackers",
|
||||||
|
brand="TestBrand",
|
||||||
|
serving_size_g=30.0,
|
||||||
|
calories=120.0,
|
||||||
|
fat_g=4.0,
|
||||||
|
saturated_fat_g=0.5,
|
||||||
|
carbs_g=20.0,
|
||||||
|
sugar_g=2.0,
|
||||||
|
fiber_g=1.0,
|
||||||
|
protein_g=3.0,
|
||||||
|
sodium_mg=200.0,
|
||||||
|
confidence=0.92,
|
||||||
|
)
|
||||||
|
row = store.get_captured_product("3333333333333")
|
||||||
|
assert row["brand"] == "TestBrand"
|
||||||
|
assert row["calories"] == 120.0
|
||||||
|
assert row["protein_g"] == 3.0
|
||||||
|
assert row["confidence"] == 0.92
|
||||||
|
|
||||||
|
def test_confirmed_by_user_defaults_true(self, store):
|
||||||
|
store.save_captured_product("4444444444444")
|
||||||
|
row = store.get_captured_product("4444444444444")
|
||||||
|
assert row["confirmed_by_user"] == 1
|
||||||
|
|
||||||
|
def test_confirmed_by_user_false(self, store):
|
||||||
|
store.save_captured_product("5555555555555", confirmed_by_user=False)
|
||||||
|
row = store.get_captured_product("5555555555555")
|
||||||
|
assert row["confirmed_by_user"] == 0
|
||||||
|
|
||||||
|
def test_upsert_on_conflict(self, store):
|
||||||
|
"""Second save for same barcode updates in-place rather than erroring."""
|
||||||
|
store.save_captured_product("6666666666666", product_name="Old Name")
|
||||||
|
store.save_captured_product("6666666666666", product_name="New Name")
|
||||||
|
row = store.get_captured_product("6666666666666")
|
||||||
|
assert row["product_name"] == "New Name"
|
||||||
|
# Still only one row
|
||||||
|
cur = store.conn.execute(
|
||||||
|
"SELECT count(*) FROM captured_products WHERE barcode='6666666666666'"
|
||||||
|
)
|
||||||
|
assert cur.fetchone()[0] == 1
|
||||||
|
|
||||||
|
def test_empty_lists_stored_and_retrieved(self, store):
|
||||||
|
store.save_captured_product("7777777777777", ingredient_names=[], allergens=[])
|
||||||
|
row = store.get_captured_product("7777777777777")
|
||||||
|
assert row["ingredient_names"] == []
|
||||||
|
assert row["allergens"] == []
|
||||||
|
|
||||||
|
def test_source_default(self, store):
|
||||||
|
store.save_captured_product("8888888888888")
|
||||||
|
row = store.get_captured_product("8888888888888")
|
||||||
|
assert row["source"] == "visual_capture"
|
||||||
171
tests/services/test_label_capture.py
Normal file
171
tests/services/test_label_capture.py
Normal file
|
|
@ -0,0 +1,171 @@
|
||||||
|
"""
|
||||||
|
Tests for app.services.label_capture.
|
||||||
|
|
||||||
|
All tests set KIWI_LABEL_CAPTURE_MOCK=1 so no vision model weights are needed.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def mock_vision(monkeypatch):
|
||||||
|
monkeypatch.setenv("KIWI_LABEL_CAPTURE_MOCK", "1")
|
||||||
|
yield
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
|
||||||
|
|
||||||
|
# ── TestExtractLabel ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
class TestExtractLabel:
|
||||||
|
def test_mock_returns_dict(self):
|
||||||
|
from app.services.label_capture import extract_label
|
||||||
|
result = extract_label(b"fake image bytes")
|
||||||
|
assert isinstance(result, dict)
|
||||||
|
|
||||||
|
def test_mock_returns_all_required_keys(self):
|
||||||
|
from app.services.label_capture import extract_label
|
||||||
|
result = extract_label(b"fake image bytes")
|
||||||
|
for key in ("product_name", "brand", "calories", "fat_g", "carbs_g",
|
||||||
|
"protein_g", "sodium_mg", "ingredient_names", "allergens",
|
||||||
|
"confidence"):
|
||||||
|
assert key in result, f"missing key: {key}"
|
||||||
|
|
||||||
|
def test_mock_ingredient_names_is_list(self):
|
||||||
|
from app.services.label_capture import extract_label
|
||||||
|
result = extract_label(b"fake")
|
||||||
|
assert isinstance(result["ingredient_names"], list)
|
||||||
|
|
||||||
|
def test_mock_allergens_is_list(self):
|
||||||
|
from app.services.label_capture import extract_label
|
||||||
|
result = extract_label(b"fake")
|
||||||
|
assert isinstance(result["allergens"], list)
|
||||||
|
|
||||||
|
def test_mock_confidence_zero(self):
|
||||||
|
from app.services.label_capture import extract_label
|
||||||
|
result = extract_label(b"fake")
|
||||||
|
assert result["confidence"] == 0.0
|
||||||
|
|
||||||
|
def _patch_vision(self, monkeypatch, caption_text: str):
|
||||||
|
"""Patch circuitforge_core.vision.caption to return a VisionResult with caption_text."""
|
||||||
|
from circuitforge_core.vision.backends.base import VisionResult
|
||||||
|
|
||||||
|
def _fake_caption(image_bytes, prompt=""):
|
||||||
|
return VisionResult(caption=caption_text)
|
||||||
|
|
||||||
|
import circuitforge_core.vision as vision_mod
|
||||||
|
monkeypatch.setattr(vision_mod, "caption", _fake_caption)
|
||||||
|
|
||||||
|
def test_exception_falls_back_to_mock(self, monkeypatch):
|
||||||
|
"""Any exception from the vision backend returns mock extraction."""
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
|
||||||
|
import circuitforge_core.vision as vision_mod
|
||||||
|
monkeypatch.setattr(vision_mod, "caption", lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("GPU unavailable")))
|
||||||
|
|
||||||
|
import app.services.label_capture as svc_mod
|
||||||
|
result = svc_mod.extract_label(b"image")
|
||||||
|
assert result["confidence"] == 0.0
|
||||||
|
assert isinstance(result["ingredient_names"], list)
|
||||||
|
|
||||||
|
def test_live_path_parses_json_string(self, monkeypatch):
|
||||||
|
"""When the vision backend returns valid JSON, it is parsed correctly."""
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"product_name": "Test Crackers",
|
||||||
|
"brand": "Test Brand",
|
||||||
|
"serving_size_g": 30.0,
|
||||||
|
"calories": 120.0,
|
||||||
|
"fat_g": 4.0,
|
||||||
|
"saturated_fat_g": 0.5,
|
||||||
|
"carbs_g": 20.0,
|
||||||
|
"sugar_g": 2.0,
|
||||||
|
"fiber_g": 1.0,
|
||||||
|
"protein_g": 3.0,
|
||||||
|
"sodium_mg": 200.0,
|
||||||
|
"ingredient_names": ["wheat flour", "canola oil", "salt"],
|
||||||
|
"allergens": ["wheat"],
|
||||||
|
"confidence": 0.92,
|
||||||
|
}
|
||||||
|
self._patch_vision(monkeypatch, json.dumps(payload))
|
||||||
|
|
||||||
|
import app.services.label_capture as svc_mod
|
||||||
|
result = svc_mod.extract_label(b"image")
|
||||||
|
assert result["product_name"] == "Test Crackers"
|
||||||
|
assert result["calories"] == 120.0
|
||||||
|
assert result["ingredient_names"] == ["wheat flour", "canola oil", "salt"]
|
||||||
|
assert result["allergens"] == ["wheat"]
|
||||||
|
assert result["confidence"] == 0.92
|
||||||
|
|
||||||
|
def test_live_path_strips_markdown_fences(self, monkeypatch):
|
||||||
|
"""JSON wrapped in ```json ... ``` fences is still parsed."""
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
|
||||||
|
payload = {"product_name": "Fancy", "brand": None, "serving_size_g": None,
|
||||||
|
"calories": None, "fat_g": None, "saturated_fat_g": None,
|
||||||
|
"carbs_g": None, "sugar_g": None, "fiber_g": None,
|
||||||
|
"protein_g": None, "sodium_mg": None,
|
||||||
|
"ingredient_names": [], "allergens": [], "confidence": 0.5}
|
||||||
|
self._patch_vision(monkeypatch, f"```json\n{json.dumps(payload)}\n```")
|
||||||
|
|
||||||
|
import app.services.label_capture as svc_mod
|
||||||
|
result = svc_mod.extract_label(b"image")
|
||||||
|
assert result["product_name"] == "Fancy"
|
||||||
|
|
||||||
|
def test_live_path_bad_json_falls_back(self, monkeypatch):
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
self._patch_vision(monkeypatch, "this is not json")
|
||||||
|
|
||||||
|
import app.services.label_capture as svc_mod
|
||||||
|
result = svc_mod.extract_label(b"image")
|
||||||
|
assert result["confidence"] == 0.0
|
||||||
|
|
||||||
|
def test_confidence_clamped_above_one(self, monkeypatch):
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
|
||||||
|
payload = {"product_name": None, "brand": None, "serving_size_g": None,
|
||||||
|
"calories": None, "fat_g": None, "saturated_fat_g": None,
|
||||||
|
"carbs_g": None, "sugar_g": None, "fiber_g": None,
|
||||||
|
"protein_g": None, "sodium_mg": None,
|
||||||
|
"ingredient_names": [], "allergens": [], "confidence": 5.0}
|
||||||
|
self._patch_vision(monkeypatch, json.dumps(payload))
|
||||||
|
|
||||||
|
import app.services.label_capture as svc_mod
|
||||||
|
result = svc_mod.extract_label(b"image")
|
||||||
|
assert result["confidence"] == 1.0
|
||||||
|
|
||||||
|
def test_none_list_fields_normalised_to_empty(self, monkeypatch):
|
||||||
|
monkeypatch.delenv("KIWI_LABEL_CAPTURE_MOCK", raising=False)
|
||||||
|
|
||||||
|
payload = {"product_name": None, "brand": None, "serving_size_g": None,
|
||||||
|
"calories": None, "fat_g": None, "saturated_fat_g": None,
|
||||||
|
"carbs_g": None, "sugar_g": None, "fiber_g": None,
|
||||||
|
"protein_g": None, "sodium_mg": None,
|
||||||
|
"ingredient_names": None, "allergens": None, "confidence": 0.8}
|
||||||
|
self._patch_vision(monkeypatch, json.dumps(payload))
|
||||||
|
|
||||||
|
import app.services.label_capture as svc_mod
|
||||||
|
result = svc_mod.extract_label(b"image")
|
||||||
|
assert result["ingredient_names"] == []
|
||||||
|
assert result["allergens"] == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── TestNeedsReview ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
class TestNeedsReview:
|
||||||
|
def test_below_threshold_needs_review(self):
|
||||||
|
from app.services.label_capture import needs_review, REVIEW_THRESHOLD
|
||||||
|
assert needs_review({"confidence": REVIEW_THRESHOLD - 0.01})
|
||||||
|
|
||||||
|
def test_at_threshold_no_review(self):
|
||||||
|
from app.services.label_capture import needs_review, REVIEW_THRESHOLD
|
||||||
|
assert not needs_review({"confidence": REVIEW_THRESHOLD})
|
||||||
|
|
||||||
|
def test_above_threshold_no_review(self):
|
||||||
|
from app.services.label_capture import needs_review
|
||||||
|
assert not needs_review({"confidence": 0.95})
|
||||||
|
|
||||||
|
def test_missing_confidence_needs_review(self):
|
||||||
|
from app.services.label_capture import needs_review
|
||||||
|
assert needs_review({})
|
||||||
Loading…
Reference in a new issue