From 41584de5dfd15ac60aace7370193942c1b83f568 Mon Sep 17 00:00:00 2001
From: pyr0ball <pyroballpcs@gmail.com>
Date: Tue, 5 May 2026 12:41:46 -0700
Subject: [PATCH] fix(benchmark): guard empty exemplars, warn on malformed JSON
 in build_exemplars_from_jsonl

---
 scripts/benchmark_classifier.py    |  9 +++++++--
 tests/test_benchmark_classifier.py | 18 ++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/scripts/benchmark_classifier.py b/scripts/benchmark_classifier.py
index d979bd3..be935b2 100644
--- a/scripts/benchmark_classifier.py
+++ b/scripts/benchmark_classifier.py
@@ -202,15 +202,20 @@ def build_exemplars_from_jsonl(path: str, k_per_label: int = 10) -> dict[str, li
                 continue
             try:
                 row = json.loads(line)
-            except json.JSONDecodeError:
+            except json.JSONDecodeError as exc:
+                print(f"[build_exemplars] WARN: skipping malformed line: {exc}", flush=True)
                 continue
             label = row.get("label")
             if not label:
                 continue
+            subject = row.get("subject", "")
+            body = row.get("body", "")
+            if not subject and not body:
+                continue
             texts = result.setdefault(label, [])
             if len(texts) < k_per_label:
                 texts.append(
-                    f"Subject: {row.get('subject', '')}\n\n{row.get('body', '')[:600]}"
+                    f"Subject: {subject}\n\n{body[:600]}"
                 )
     return result
 
diff --git a/tests/test_benchmark_classifier.py b/tests/test_benchmark_classifier.py
index f076aff..d0b2155 100644
--- a/tests/test_benchmark_classifier.py
+++ b/tests/test_benchmark_classifier.py
@@ -225,3 +225,21 @@ def test_build_exemplars_truncates_body_at_600(tmp_path):
     result = build_exemplars_from_jsonl(str(f))
     body_part = result["neutral"][0].split("\n\n", 1)[1]
     assert len(body_part) == 600
+
+
+def test_build_exemplars_skips_rows_with_no_content(tmp_path):
+    from scripts.benchmark_classifier import build_exemplars_from_jsonl
+    import json
+
+    rows = [
+        {"label": "neutral"},                                  # no subject, no body -> skip
+        {"subject": "S", "body": "B", "label": "neutral"},    # valid -> keep
+        {"label": "rejected", "subject": "", "body": ""},      # empty strings -> skip
+    ]
+    f = tmp_path / "score.jsonl"
+    lines = [json.dumps(r) for r in rows]
+    f.write_text("\n".join(lines))
+
+    result = build_exemplars_from_jsonl(str(f))
+    assert list(result.keys()) == ["neutral"]
+    assert len(result["neutral"]) == 1