From a271278dc98c3d4da4e31869973d8f1694ef6094 Mon Sep 17 00:00:00 2001 From: pyr0ball Date: Thu, 9 Apr 2026 12:26:44 -0700 Subject: [PATCH] feat(#10): env var LLM config + cf-orch coordinator auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _load_cforch_config() falls back to CF_ORCH_URL / CF_LICENSE_KEY / OLLAMA_HOST / OLLAMA_MODEL env vars when label_tool.yaml cforch: key is absent or empty (yaml wins when both present) - CF_LICENSE_KEY forwarded to benchmark subprocess env so cf-orch agent can authenticate without it appearing in command args - GET /api/cforch/config endpoint — returns resolved connection state; redacts license key (returns license_key_set bool only) - SettingsView: connection status pill (cf-orch / Ollama / unconfigured) loaded from /api/cforch/config on mount; shows env vs yaml source - .env.example documenting all relevant vars - config/label_tool.yaml.example: full cforch: section with all keys - environment.yml: add circuitforge-core>=0.9.0 dependency - .gitignore: add .env - 4 new tests (17 total in test_cforch.py); 136 passing overall Closes #10 --- .env.example | 19 ++++++++ app/cforch.py | 66 +++++++++++++++++++++----- config/label_tool.yaml.example | 20 ++++++++ environment.yml | 3 ++ tests/test_cforch.py | 84 ++++++++++++++++++++++++++++++++++ web/src/views/SettingsView.vue | 64 +++++++++++++++++++++++++- 6 files changed, 243 insertions(+), 13 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..03c3c9f --- /dev/null +++ b/.env.example @@ -0,0 +1,19 @@ +# Avocet — environment variable configuration +# Copy to .env and fill in values. All keys are optional. +# label_tool.yaml takes precedence over env vars where both exist. + +# ── Local inference (Ollama) ─────────────────────────────────────────────────── +# OLLAMA_HOST defaults to http://localhost:11434 if unset. +OLLAMA_HOST=http://localhost:11434 +OLLAMA_MODEL=llama3.2:3b + +# ── cf-orch coordinator (paid/premium tiers) ─────────────────────────────────── +# Required for multi-GPU LLM benchmarking via the cf-orch benchmark harness. +# Free-tier users can leave these unset and use Ollama only. +CF_ORCH_URL=http://localhost:7700 +CF_LICENSE_KEY=CFG-AVCT-xxxx-xxxx-xxxx + +# ── Cloud LLM backends (optional — paid/premium) ────────────────────────────── +# Set one of these to use a cloud LLM instead of a local model. +# ANTHROPIC_API_KEY=sk-ant-... +# OPENAI_API_KEY=sk-... diff --git a/app/cforch.py b/app/cforch.py index 27ca050..5942ab3 100644 --- a/app/cforch.py +++ b/app/cforch.py @@ -14,6 +14,7 @@ from __future__ import annotations import json import logging +import os import re import subprocess as _subprocess from pathlib import Path @@ -49,16 +50,32 @@ def _config_file() -> Path: def _load_cforch_config() -> dict: - """Read label_tool.yaml and return the cforch sub-dict (or {} if absent/malformed).""" + """Read label_tool.yaml cforch section, falling back to environment variables. + + Priority (highest to lowest): + 1. label_tool.yaml cforch: key + 2. Environment variables (CF_ORCH_URL, CF_LICENSE_KEY, OLLAMA_HOST, OLLAMA_MODEL) + """ f = _config_file() - if not f.exists(): - return {} - try: - raw = yaml.safe_load(f.read_text(encoding="utf-8")) or {} - except yaml.YAMLError as exc: - logger.warning("Failed to parse cforch config %s: %s", f, exc) - return {} - return raw.get("cforch", {}) or {} + file_cfg: dict = {} + if f.exists(): + try: + raw = yaml.safe_load(f.read_text(encoding="utf-8")) or {} + file_cfg = raw.get("cforch", {}) or {} + except yaml.YAMLError as exc: + logger.warning("Failed to parse cforch config %s: %s", f, exc) + + # Env var fallbacks — only used when the yaml key is absent or empty + def _coalesce(file_val: str, env_key: str) -> str: + return file_val if file_val else os.environ.get(env_key, "") + + return { + **file_cfg, + "coordinator_url": _coalesce(file_cfg.get("coordinator_url", ""), "CF_ORCH_URL"), + "license_key": _coalesce(file_cfg.get("license_key", ""), "CF_LICENSE_KEY"), + "ollama_url": _coalesce(file_cfg.get("ollama_url", ""), "OLLAMA_HOST"), + "ollama_model": _coalesce(file_cfg.get("ollama_model", ""), "OLLAMA_MODEL"), + } def _strip_ansi(text: str) -> str: @@ -184,7 +201,8 @@ def run_benchmark( results_dir = cfg.get("results_dir", "") python_bin = cfg.get("python_bin", "/devl/miniconda3/envs/cf/bin/python") cfg_coordinator = cfg.get("coordinator_url", "") - cfg_ollama = cfg.get("ollama_url", "") + cfg_ollama = cfg.get("ollama_url", "") + cfg_license_key = cfg.get("license_key", "") def generate(): global _BENCH_RUNNING, _bench_proc @@ -206,13 +224,19 @@ def run_benchmark( if model_tags: cmd.extend(["--filter-tags"] + model_tags.split(",")) + # query param overrides config, config overrides env var (already resolved by _load_cforch_config) effective_coordinator = coordinator_url if coordinator_url else cfg_coordinator - effective_ollama = ollama_url if ollama_url else cfg_ollama + effective_ollama = ollama_url if ollama_url else cfg_ollama if effective_coordinator: cmd.extend(["--coordinator", effective_coordinator]) if effective_ollama: cmd.extend(["--ollama-url", effective_ollama]) + # Pass license key as env var so subprocess can authenticate with cf-orch + proc_env = {**os.environ} + if cfg_license_key: + proc_env["CF_LICENSE_KEY"] = cfg_license_key + _BENCH_RUNNING = True try: proc = _subprocess.Popen( @@ -221,6 +245,7 @@ def run_benchmark( stderr=_subprocess.STDOUT, text=True, bufsize=1, + env=proc_env, ) _bench_proc = proc try: @@ -254,6 +279,25 @@ def run_benchmark( ) +# ── GET /config ──────────────────────────────────────────────────────────────── + +@router.get("/config") +def get_cforch_config() -> dict: + """Return resolved cf-orch connection config (env vars merged with yaml). + + Redacts license_key — only returns whether it is set, not the value. + Used by the Settings UI to show current connection state. + """ + cfg = _load_cforch_config() + return { + "coordinator_url": cfg.get("coordinator_url", ""), + "ollama_url": cfg.get("ollama_url", ""), + "ollama_model": cfg.get("ollama_model", ""), + "license_key_set": bool(cfg.get("license_key", "")), + "source": "env" if not _config_file().exists() else "yaml+env", + } + + # ── GET /results ─────────────────────────────────────────────────────────────── @router.get("/results") diff --git a/config/label_tool.yaml.example b/config/label_tool.yaml.example index 9310d21..8aedb2b 100644 --- a/config/label_tool.yaml.example +++ b/config/label_tool.yaml.example @@ -26,3 +26,23 @@ max_per_account: 500 # produced by circuitforge-orch's benchmark harness. sft: bench_results_dir: /path/to/circuitforge-orch/scripts/bench_results + +# cf-orch integration — LLM benchmark harness via cf-orch coordinator. +# All keys here override the corresponding environment variables. +# Omit any key to fall back to the env var (see .env.example). +cforch: + # Path to cf-orch's benchmark.py script + bench_script: /path/to/circuitforge-orch/scripts/benchmark.py + # Task and model definition files (yaml) + bench_tasks: /path/to/circuitforge-orch/scripts/bench_tasks.yaml + bench_models: /path/to/circuitforge-orch/scripts/bench_models.yaml + # Where benchmark results are written (also used for SFT candidate discovery) + results_dir: /path/to/circuitforge-orch/scripts/bench_results + # Python interpreter with cf-orch installed + python_bin: /devl/miniconda3/envs/cf/bin/python + + # Connection config — override env vars CF_ORCH_URL / CF_LICENSE_KEY / OLLAMA_HOST + # coordinator_url: http://localhost:7700 + # license_key: CFG-AVCT-xxxx-xxxx-xxxx + # ollama_url: http://localhost:11434 + # ollama_model: llama3.2:3b diff --git a/environment.yml b/environment.yml index 73f1941..e8553f3 100644 --- a/environment.yml +++ b/environment.yml @@ -22,5 +22,8 @@ dependencies: # Optional: BGE reranker adapter # - FlagEmbedding + # CircuitForge shared core (LLM router, tier system, config) + - circuitforge-core>=0.9.0 + # Dev - pytest>=8.0 diff --git a/tests/test_cforch.py b/tests/test_cforch.py index 537cd5f..3abc8ea 100644 --- a/tests/test_cforch.py +++ b/tests/test_cforch.py @@ -280,3 +280,87 @@ def test_cancel_terminates_running_benchmark(client): mock_proc.terminate.assert_called_once() assert cforch_module._BENCH_RUNNING is False assert cforch_module._bench_proc is None + + +# ── GET /config ──────────────────────────────────────────────────────────────── + +def test_config_returns_empty_when_no_yaml_no_env(client, monkeypatch): + """No yaml, no env vars — all fields empty, license_key_set False.""" + for key in ("CF_ORCH_URL", "CF_LICENSE_KEY", "OLLAMA_HOST", "OLLAMA_MODEL"): + monkeypatch.delenv(key, raising=False) + + r = client.get("/api/cforch/config") + assert r.status_code == 200 + data = r.json() + assert data["coordinator_url"] == "" + assert data["ollama_url"] == "" + assert data["license_key_set"] is False + + +def test_config_reads_env_vars_when_no_yaml(client, monkeypatch): + """Env vars populate fields when label_tool.yaml has no cforch section.""" + monkeypatch.setenv("CF_ORCH_URL", "http://orch.example.com:7700") + monkeypatch.setenv("CF_LICENSE_KEY", "CFG-AVCT-TEST-TEST-TEST") + monkeypatch.setenv("OLLAMA_HOST", "http://ollama.local:11434") + monkeypatch.setenv("OLLAMA_MODEL", "mistral:7b") + + r = client.get("/api/cforch/config") + assert r.status_code == 200 + data = r.json() + assert data["coordinator_url"] == "http://orch.example.com:7700" + assert data["ollama_url"] == "http://ollama.local:11434" + assert data["ollama_model"] == "mistral:7b" + assert data["license_key_set"] is True # set, but value not exposed + + +def test_config_yaml_overrides_env(client, config_dir, monkeypatch): + """label_tool.yaml cforch values take priority over env vars.""" + monkeypatch.setenv("CF_ORCH_URL", "http://env-orch:7700") + monkeypatch.setenv("OLLAMA_HOST", "http://env-ollama:11434") + + _write_config(config_dir, { + "coordinator_url": "http://yaml-orch:7700", + "ollama_url": "http://yaml-ollama:11434", + }) + + r = client.get("/api/cforch/config") + assert r.status_code == 200 + data = r.json() + assert data["coordinator_url"] == "http://yaml-orch:7700" + assert data["ollama_url"] == "http://yaml-ollama:11434" + assert data["source"] == "yaml+env" + + +def test_run_passes_license_key_env_to_subprocess(client, config_dir, tmp_path, monkeypatch): + """CF_LICENSE_KEY must be forwarded to the benchmark subprocess env.""" + monkeypatch.setenv("CF_LICENSE_KEY", "CFG-AVCT-ENV-ONLY-KEY") + + bench_script = tmp_path / "benchmark.py" + bench_script.write_text("# stub", encoding="utf-8") + tasks_file = tmp_path / "bench_tasks.yaml" + tasks_file.write_text(yaml.dump({"tasks": []}), encoding="utf-8") + models_file = tmp_path / "bench_models.yaml" + models_file.write_text(yaml.dump({"models": []}), encoding="utf-8") + + _write_config(config_dir, { + "bench_script": str(bench_script), + "bench_tasks": str(tasks_file), + "bench_models": str(models_file), + "results_dir": str(tmp_path / "results"), + "python_bin": "/usr/bin/python3", + }) + + captured_env: dict = {} + + def fake_popen(cmd, **kwargs): + captured_env.update(kwargs.get("env", {})) + mock = MagicMock() + mock.stdout = iter([]) + mock.returncode = 0 + mock.wait = MagicMock() + return mock + + with patch("app.cforch._subprocess.Popen", side_effect=fake_popen): + client.get("/api/cforch/run") + + assert captured_env.get("CF_LICENSE_KEY") == "CFG-AVCT-ENV-ONLY-KEY" diff --git a/web/src/views/SettingsView.vue b/web/src/views/SettingsView.vue index 032ddc1..e93240e 100644 --- a/web/src/views/SettingsView.vue +++ b/web/src/views/SettingsView.vue @@ -115,8 +115,18 @@

cf-orch Integration

Import SFT (supervised fine-tuning) candidates from cf-orch benchmark runs. + Connection settings fall back to environment variables + (CF_ORCH_URL, CF_LICENSE_KEY, OLLAMA_HOST) + when not set here.

+ +
+ {{ orchStatusLabel }} + via env vars + via label_tool.yaml +
+