"""Map cf-harvest SegmentKnowledge → KnowledgeFact rows. Idempotent: re-ingesting the same video batch is a no-op (INSERT OR IGNORE on fact_hash). Partial success: malformed individual facts are collected in rejected[] without aborting the batch — the caller receives a per-item rejection report. """ from __future__ import annotations import hashlib import json import logging import sqlite3 from typing import Any from app.models.schemas.knowledge import ( IngestResponse, KnowledgeIngestRequest, RejectedFact, ) logger = logging.getLogger(__name__) # Maps fact_type → how to derive the searchable subject from the payload. # Subject is stored denormalized for fast filtering; it is not the ground truth # — the full payload is authoritative. _SUBJECT_EXTRACTORS: dict[str, Any] = { "companion_planting": lambda p: (p.get("plants") or [None])[0], "soil_amendment": lambda p: (p.get("target_plants") or [None])[0] or p.get("amendment"), "propagation": lambda p: p.get("plant"), "pest_diagnosis": lambda p: p.get("pest_or_disease"), "harvest_timing": lambda p: p.get("plant"), "instruction": lambda p: p.get("subject") or p.get("action"), "medicinal": lambda p: p.get("plant"), "history_context": lambda p: p.get("subject"), } def _derive_subject(fact_type: str, payload: dict) -> str | None: extractor = _SUBJECT_EXTRACTORS.get(fact_type) if extractor is None: return None try: result = extractor(payload) return str(result).strip() if result else None except Exception: return None def _fact_hash(fact_type: str, payload: dict, video_path: str) -> str: canonical = json.dumps( {"t": fact_type, "p": payload, "v": video_path}, sort_keys=True, ensure_ascii=True, ) return hashlib.sha256(canonical.encode()).hexdigest() def ingest(conn: sqlite3.Connection, request: KnowledgeIngestRequest) -> IngestResponse: """Upsert source, then insert facts. Returns counts and per-item rejections.""" src = request.source # Upsert knowledge_source — idempotent on video_path conn.execute( """ INSERT INTO knowledge_sources (video_path, presenter, location, source_domain) VALUES (?, ?, ?, ?) ON CONFLICT(video_path) DO UPDATE SET presenter = excluded.presenter, location = excluded.location, source_domain = excluded.source_domain """, (src.video_path, src.presenter, src.location, src.source_domain), ) source_id: int = conn.execute( "SELECT id FROM knowledge_sources WHERE video_path = ?", (src.video_path,) ).fetchone()["id"] inserted = 0 skipped = 0 rejected: list[RejectedFact] = [] for idx, fact in enumerate(request.facts): try: subject = _derive_subject(fact.fact_type, fact.payload) fhash = _fact_hash(fact.fact_type, fact.payload, src.video_path) payload_json = json.dumps(fact.payload, ensure_ascii=False) cursor = conn.execute( """ INSERT OR IGNORE INTO knowledge_facts (source_id, fact_type, subject, payload, presenter, confidence, fact_hash) VALUES (?, ?, ?, ?, ?, ?, ?) """, ( source_id, fact.fact_type, subject, payload_json, src.presenter, fact.confidence, fhash, ), ) if cursor.rowcount == 1: inserted += 1 else: skipped += 1 except Exception as exc: logger.warning("Rejected fact %d (%s): %s", idx, fact.fact_type, exc) rejected.append(RejectedFact(index=idx, reason=str(exc))) conn.commit() return IngestResponse( source_id=source_id, video_path=src.video_path, facts_received=len(request.facts), facts_inserted=inserted, facts_skipped_duplicate=skipped, rejected=rejected, )