Circuit-Forge/kiwi
2
0
Fork 0
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: pipeline scripts — connection safety, remove unused recipes_path arg, fix inserted counter, pre-load profile index

Browse source
This commit is contained in:
pyr0ball 2026-03-30 23:10:52 -07:00
parent e57ae74e27
commit e44d36e32f
2 changed files with 125 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
scripts/pipeline/build_recipe_index.py
View file

@ -55,7 +55,17 @@ def compute_element_coverage(profiles: list[dict]) -> dict[str, float]:
def build(db_path: Path, recipes_path: Path, batch_size: int = 10000) -> None:
conn = sqlite3.connect(db_path)
try:
conn.execute("PRAGMA journal_mode=WAL")
# Pre-load ingredient element profiles to avoid N+1 queries
profile_index: dict[str, list[str]] = {}
for row in conn.execute("SELECT name, elements FROM ingredient_profiles"):
try:
profile_index[row[0]] = json.loads(row[1])
except Exception:
pass
df = pd.read_parquet(recipes_path)
inserted = 0
batch = []
@ -72,11 +82,8 @@ def build(db_path: Path, recipes_path: Path, batch_size: int = 10000) -> None:
profiles = []
for name in ingredient_names:
row_p = conn.execute(
"SELECT elements FROM ingredient_profiles WHERE name = ?", (name,)
).fetchone()
if row_p:
profiles.append({"elements": json.loads(row_p[0])})
if name in profile_index:
profiles.append({"elements": profile_index[name]})
coverage = compute_element_coverage(profiles)
directions = row.get("RecipeInstructions", [])
@ -102,6 +109,7 @@ def build(db_path: Path, recipes_path: Path, batch_size: int = 10000) -> None:
))
if len(batch) >= batch_size:
before = conn.total_changes
conn.executemany("""
INSERT OR IGNORE INTO recipes
(external_id, title, ingredients, ingredient_names, directions,
@ -109,11 +117,12 @@ def build(db_path: Path, recipes_path: Path, batch_size: int = 10000) -> None:
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
""", batch)
conn.commit()
inserted += len(batch)
inserted += conn.total_changes - before
print(f" {inserted} recipes inserted...")
batch = []
if batch:
before = conn.total_changes
conn.executemany("""
INSERT OR IGNORE INTO recipes
(external_id, title, ingredients, ingredient_names, directions,
@ -121,8 +130,10 @@ def build(db_path: Path, recipes_path: Path, batch_size: int = 10000) -> None:
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
""", batch)
conn.commit()
inserted += len(batch)
inserted += conn.total_changes - before
conn.commit()
finally:
conn.close()
print(f"Total: {inserted} recipes inserted")

11
scripts/pipeline/derive_substitutions.py
View file

@ -5,8 +5,7 @@ GPL-3.0 source -- derived annotations only, raw pairs not shipped.
Usage:
conda run -n job-seeker python scripts/pipeline/derive_substitutions.py \
--db /path/to/kiwi.db \
--recipepairs data/recipepairs.parquet \
--recipes data/recipes_foodcom.parquet
--recipepairs data/recipepairs.parquet
"""
from __future__ import annotations
import argparse
@ -31,9 +30,9 @@ def diff_ingredients(base: list[str], target: list[str]) -> tuple[list[str], lis
return removed, added
def build(db_path: Path, recipepairs_path: Path, recipes_path: Path) -> None:
def build(db_path: Path, recipepairs_path: Path) -> None:
conn = sqlite3.connect(db_path)
try:
print("Loading recipe ingredient index...")
recipe_ingredients: dict[str, list[str]] = {}
for row in conn.execute("SELECT external_id, ingredient_names FROM recipes"):
@ -96,6 +95,7 @@ def build(db_path: Path, recipepairs_path: Path, recipes_path: Path) -> None:
inserted += 1
conn.commit()
finally:
conn.close()
print(f"Inserted {inserted} substitution pairs (min 3 occurrences)")
@ -104,6 +104,5 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--db", required=True, type=Path)
parser.add_argument("--recipepairs", required=True, type=Path)
parser.add_argument("--recipes", required=True, type=Path)
args = parser.parse_args()
build(args.db, args.recipepairs, args.recipes)
build(args.db, args.recipepairs)