feat(#10): env var LLM config + cf-orch coordinator auth
- _load_cforch_config() falls back to CF_ORCH_URL / CF_LICENSE_KEY / OLLAMA_HOST / OLLAMA_MODEL env vars when label_tool.yaml cforch: key is absent or empty (yaml wins when both present) - CF_LICENSE_KEY forwarded to benchmark subprocess env so cf-orch agent can authenticate without it appearing in command args - GET /api/cforch/config endpoint — returns resolved connection state; redacts license key (returns license_key_set bool only) - SettingsView: connection status pill (cf-orch / Ollama / unconfigured) loaded from /api/cforch/config on mount; shows env vs yaml source - .env.example documenting all relevant vars - config/label_tool.yaml.example: full cforch: section with all keys - environment.yml: add circuitforge-core>=0.9.0 dependency - .gitignore: add .env - 4 new tests (17 total in test_cforch.py); 136 passing overall Closes #10
This commit is contained in:
parent
dffb1d0d7a
commit
a271278dc9
6 changed files with 243 additions and 13 deletions
19
.env.example
Normal file
19
.env.example
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Avocet — environment variable configuration
|
||||||
|
# Copy to .env and fill in values. All keys are optional.
|
||||||
|
# label_tool.yaml takes precedence over env vars where both exist.
|
||||||
|
|
||||||
|
# ── Local inference (Ollama) ───────────────────────────────────────────────────
|
||||||
|
# OLLAMA_HOST defaults to http://localhost:11434 if unset.
|
||||||
|
OLLAMA_HOST=http://localhost:11434
|
||||||
|
OLLAMA_MODEL=llama3.2:3b
|
||||||
|
|
||||||
|
# ── cf-orch coordinator (paid/premium tiers) ───────────────────────────────────
|
||||||
|
# Required for multi-GPU LLM benchmarking via the cf-orch benchmark harness.
|
||||||
|
# Free-tier users can leave these unset and use Ollama only.
|
||||||
|
CF_ORCH_URL=http://localhost:7700
|
||||||
|
CF_LICENSE_KEY=CFG-AVCT-xxxx-xxxx-xxxx
|
||||||
|
|
||||||
|
# ── Cloud LLM backends (optional — paid/premium) ──────────────────────────────
|
||||||
|
# Set one of these to use a cloud LLM instead of a local model.
|
||||||
|
# ANTHROPIC_API_KEY=sk-ant-...
|
||||||
|
# OPENAI_API_KEY=sk-...
|
||||||
|
|
@ -14,6 +14,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import subprocess as _subprocess
|
import subprocess as _subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
@ -49,16 +50,32 @@ def _config_file() -> Path:
|
||||||
|
|
||||||
|
|
||||||
def _load_cforch_config() -> dict:
|
def _load_cforch_config() -> dict:
|
||||||
"""Read label_tool.yaml and return the cforch sub-dict (or {} if absent/malformed)."""
|
"""Read label_tool.yaml cforch section, falling back to environment variables.
|
||||||
|
|
||||||
|
Priority (highest to lowest):
|
||||||
|
1. label_tool.yaml cforch: key
|
||||||
|
2. Environment variables (CF_ORCH_URL, CF_LICENSE_KEY, OLLAMA_HOST, OLLAMA_MODEL)
|
||||||
|
"""
|
||||||
f = _config_file()
|
f = _config_file()
|
||||||
if not f.exists():
|
file_cfg: dict = {}
|
||||||
return {}
|
if f.exists():
|
||||||
try:
|
try:
|
||||||
raw = yaml.safe_load(f.read_text(encoding="utf-8")) or {}
|
raw = yaml.safe_load(f.read_text(encoding="utf-8")) or {}
|
||||||
|
file_cfg = raw.get("cforch", {}) or {}
|
||||||
except yaml.YAMLError as exc:
|
except yaml.YAMLError as exc:
|
||||||
logger.warning("Failed to parse cforch config %s: %s", f, exc)
|
logger.warning("Failed to parse cforch config %s: %s", f, exc)
|
||||||
return {}
|
|
||||||
return raw.get("cforch", {}) or {}
|
# Env var fallbacks — only used when the yaml key is absent or empty
|
||||||
|
def _coalesce(file_val: str, env_key: str) -> str:
|
||||||
|
return file_val if file_val else os.environ.get(env_key, "")
|
||||||
|
|
||||||
|
return {
|
||||||
|
**file_cfg,
|
||||||
|
"coordinator_url": _coalesce(file_cfg.get("coordinator_url", ""), "CF_ORCH_URL"),
|
||||||
|
"license_key": _coalesce(file_cfg.get("license_key", ""), "CF_LICENSE_KEY"),
|
||||||
|
"ollama_url": _coalesce(file_cfg.get("ollama_url", ""), "OLLAMA_HOST"),
|
||||||
|
"ollama_model": _coalesce(file_cfg.get("ollama_model", ""), "OLLAMA_MODEL"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _strip_ansi(text: str) -> str:
|
def _strip_ansi(text: str) -> str:
|
||||||
|
|
@ -185,6 +202,7 @@ def run_benchmark(
|
||||||
python_bin = cfg.get("python_bin", "/devl/miniconda3/envs/cf/bin/python")
|
python_bin = cfg.get("python_bin", "/devl/miniconda3/envs/cf/bin/python")
|
||||||
cfg_coordinator = cfg.get("coordinator_url", "")
|
cfg_coordinator = cfg.get("coordinator_url", "")
|
||||||
cfg_ollama = cfg.get("ollama_url", "")
|
cfg_ollama = cfg.get("ollama_url", "")
|
||||||
|
cfg_license_key = cfg.get("license_key", "")
|
||||||
|
|
||||||
def generate():
|
def generate():
|
||||||
global _BENCH_RUNNING, _bench_proc
|
global _BENCH_RUNNING, _bench_proc
|
||||||
|
|
@ -206,6 +224,7 @@ def run_benchmark(
|
||||||
if model_tags:
|
if model_tags:
|
||||||
cmd.extend(["--filter-tags"] + model_tags.split(","))
|
cmd.extend(["--filter-tags"] + model_tags.split(","))
|
||||||
|
|
||||||
|
# query param overrides config, config overrides env var (already resolved by _load_cforch_config)
|
||||||
effective_coordinator = coordinator_url if coordinator_url else cfg_coordinator
|
effective_coordinator = coordinator_url if coordinator_url else cfg_coordinator
|
||||||
effective_ollama = ollama_url if ollama_url else cfg_ollama
|
effective_ollama = ollama_url if ollama_url else cfg_ollama
|
||||||
if effective_coordinator:
|
if effective_coordinator:
|
||||||
|
|
@ -213,6 +232,11 @@ def run_benchmark(
|
||||||
if effective_ollama:
|
if effective_ollama:
|
||||||
cmd.extend(["--ollama-url", effective_ollama])
|
cmd.extend(["--ollama-url", effective_ollama])
|
||||||
|
|
||||||
|
# Pass license key as env var so subprocess can authenticate with cf-orch
|
||||||
|
proc_env = {**os.environ}
|
||||||
|
if cfg_license_key:
|
||||||
|
proc_env["CF_LICENSE_KEY"] = cfg_license_key
|
||||||
|
|
||||||
_BENCH_RUNNING = True
|
_BENCH_RUNNING = True
|
||||||
try:
|
try:
|
||||||
proc = _subprocess.Popen(
|
proc = _subprocess.Popen(
|
||||||
|
|
@ -221,6 +245,7 @@ def run_benchmark(
|
||||||
stderr=_subprocess.STDOUT,
|
stderr=_subprocess.STDOUT,
|
||||||
text=True,
|
text=True,
|
||||||
bufsize=1,
|
bufsize=1,
|
||||||
|
env=proc_env,
|
||||||
)
|
)
|
||||||
_bench_proc = proc
|
_bench_proc = proc
|
||||||
try:
|
try:
|
||||||
|
|
@ -254,6 +279,25 @@ def run_benchmark(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── GET /config ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@router.get("/config")
|
||||||
|
def get_cforch_config() -> dict:
|
||||||
|
"""Return resolved cf-orch connection config (env vars merged with yaml).
|
||||||
|
|
||||||
|
Redacts license_key — only returns whether it is set, not the value.
|
||||||
|
Used by the Settings UI to show current connection state.
|
||||||
|
"""
|
||||||
|
cfg = _load_cforch_config()
|
||||||
|
return {
|
||||||
|
"coordinator_url": cfg.get("coordinator_url", ""),
|
||||||
|
"ollama_url": cfg.get("ollama_url", ""),
|
||||||
|
"ollama_model": cfg.get("ollama_model", ""),
|
||||||
|
"license_key_set": bool(cfg.get("license_key", "")),
|
||||||
|
"source": "env" if not _config_file().exists() else "yaml+env",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ── GET /results ───────────────────────────────────────────────────────────────
|
# ── GET /results ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@router.get("/results")
|
@router.get("/results")
|
||||||
|
|
|
||||||
|
|
@ -26,3 +26,23 @@ max_per_account: 500
|
||||||
# produced by circuitforge-orch's benchmark harness.
|
# produced by circuitforge-orch's benchmark harness.
|
||||||
sft:
|
sft:
|
||||||
bench_results_dir: /path/to/circuitforge-orch/scripts/bench_results
|
bench_results_dir: /path/to/circuitforge-orch/scripts/bench_results
|
||||||
|
|
||||||
|
# cf-orch integration — LLM benchmark harness via cf-orch coordinator.
|
||||||
|
# All keys here override the corresponding environment variables.
|
||||||
|
# Omit any key to fall back to the env var (see .env.example).
|
||||||
|
cforch:
|
||||||
|
# Path to cf-orch's benchmark.py script
|
||||||
|
bench_script: /path/to/circuitforge-orch/scripts/benchmark.py
|
||||||
|
# Task and model definition files (yaml)
|
||||||
|
bench_tasks: /path/to/circuitforge-orch/scripts/bench_tasks.yaml
|
||||||
|
bench_models: /path/to/circuitforge-orch/scripts/bench_models.yaml
|
||||||
|
# Where benchmark results are written (also used for SFT candidate discovery)
|
||||||
|
results_dir: /path/to/circuitforge-orch/scripts/bench_results
|
||||||
|
# Python interpreter with cf-orch installed
|
||||||
|
python_bin: /devl/miniconda3/envs/cf/bin/python
|
||||||
|
|
||||||
|
# Connection config — override env vars CF_ORCH_URL / CF_LICENSE_KEY / OLLAMA_HOST
|
||||||
|
# coordinator_url: http://localhost:7700
|
||||||
|
# license_key: CFG-AVCT-xxxx-xxxx-xxxx
|
||||||
|
# ollama_url: http://localhost:11434
|
||||||
|
# ollama_model: llama3.2:3b
|
||||||
|
|
|
||||||
|
|
@ -22,5 +22,8 @@ dependencies:
|
||||||
# Optional: BGE reranker adapter
|
# Optional: BGE reranker adapter
|
||||||
# - FlagEmbedding
|
# - FlagEmbedding
|
||||||
|
|
||||||
|
# CircuitForge shared core (LLM router, tier system, config)
|
||||||
|
- circuitforge-core>=0.9.0
|
||||||
|
|
||||||
# Dev
|
# Dev
|
||||||
- pytest>=8.0
|
- pytest>=8.0
|
||||||
|
|
|
||||||
|
|
@ -280,3 +280,87 @@ def test_cancel_terminates_running_benchmark(client):
|
||||||
mock_proc.terminate.assert_called_once()
|
mock_proc.terminate.assert_called_once()
|
||||||
assert cforch_module._BENCH_RUNNING is False
|
assert cforch_module._BENCH_RUNNING is False
|
||||||
assert cforch_module._bench_proc is None
|
assert cforch_module._bench_proc is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── GET /config ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_config_returns_empty_when_no_yaml_no_env(client, monkeypatch):
|
||||||
|
"""No yaml, no env vars — all fields empty, license_key_set False."""
|
||||||
|
for key in ("CF_ORCH_URL", "CF_LICENSE_KEY", "OLLAMA_HOST", "OLLAMA_MODEL"):
|
||||||
|
monkeypatch.delenv(key, raising=False)
|
||||||
|
|
||||||
|
r = client.get("/api/cforch/config")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert data["coordinator_url"] == ""
|
||||||
|
assert data["ollama_url"] == ""
|
||||||
|
assert data["license_key_set"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_config_reads_env_vars_when_no_yaml(client, monkeypatch):
|
||||||
|
"""Env vars populate fields when label_tool.yaml has no cforch section."""
|
||||||
|
monkeypatch.setenv("CF_ORCH_URL", "http://orch.example.com:7700")
|
||||||
|
monkeypatch.setenv("CF_LICENSE_KEY", "CFG-AVCT-TEST-TEST-TEST")
|
||||||
|
monkeypatch.setenv("OLLAMA_HOST", "http://ollama.local:11434")
|
||||||
|
monkeypatch.setenv("OLLAMA_MODEL", "mistral:7b")
|
||||||
|
|
||||||
|
r = client.get("/api/cforch/config")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert data["coordinator_url"] == "http://orch.example.com:7700"
|
||||||
|
assert data["ollama_url"] == "http://ollama.local:11434"
|
||||||
|
assert data["ollama_model"] == "mistral:7b"
|
||||||
|
assert data["license_key_set"] is True # set, but value not exposed
|
||||||
|
|
||||||
|
|
||||||
|
def test_config_yaml_overrides_env(client, config_dir, monkeypatch):
|
||||||
|
"""label_tool.yaml cforch values take priority over env vars."""
|
||||||
|
monkeypatch.setenv("CF_ORCH_URL", "http://env-orch:7700")
|
||||||
|
monkeypatch.setenv("OLLAMA_HOST", "http://env-ollama:11434")
|
||||||
|
|
||||||
|
_write_config(config_dir, {
|
||||||
|
"coordinator_url": "http://yaml-orch:7700",
|
||||||
|
"ollama_url": "http://yaml-ollama:11434",
|
||||||
|
})
|
||||||
|
|
||||||
|
r = client.get("/api/cforch/config")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert data["coordinator_url"] == "http://yaml-orch:7700"
|
||||||
|
assert data["ollama_url"] == "http://yaml-ollama:11434"
|
||||||
|
assert data["source"] == "yaml+env"
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_passes_license_key_env_to_subprocess(client, config_dir, tmp_path, monkeypatch):
|
||||||
|
"""CF_LICENSE_KEY must be forwarded to the benchmark subprocess env."""
|
||||||
|
monkeypatch.setenv("CF_LICENSE_KEY", "CFG-AVCT-ENV-ONLY-KEY")
|
||||||
|
|
||||||
|
bench_script = tmp_path / "benchmark.py"
|
||||||
|
bench_script.write_text("# stub", encoding="utf-8")
|
||||||
|
tasks_file = tmp_path / "bench_tasks.yaml"
|
||||||
|
tasks_file.write_text(yaml.dump({"tasks": []}), encoding="utf-8")
|
||||||
|
models_file = tmp_path / "bench_models.yaml"
|
||||||
|
models_file.write_text(yaml.dump({"models": []}), encoding="utf-8")
|
||||||
|
|
||||||
|
_write_config(config_dir, {
|
||||||
|
"bench_script": str(bench_script),
|
||||||
|
"bench_tasks": str(tasks_file),
|
||||||
|
"bench_models": str(models_file),
|
||||||
|
"results_dir": str(tmp_path / "results"),
|
||||||
|
"python_bin": "/usr/bin/python3",
|
||||||
|
})
|
||||||
|
|
||||||
|
captured_env: dict = {}
|
||||||
|
|
||||||
|
def fake_popen(cmd, **kwargs):
|
||||||
|
captured_env.update(kwargs.get("env", {}))
|
||||||
|
mock = MagicMock()
|
||||||
|
mock.stdout = iter([])
|
||||||
|
mock.returncode = 0
|
||||||
|
mock.wait = MagicMock()
|
||||||
|
return mock
|
||||||
|
|
||||||
|
with patch("app.cforch._subprocess.Popen", side_effect=fake_popen):
|
||||||
|
client.get("/api/cforch/run")
|
||||||
|
|
||||||
|
assert captured_env.get("CF_LICENSE_KEY") == "CFG-AVCT-ENV-ONLY-KEY"
|
||||||
|
|
|
||||||
|
|
@ -115,8 +115,18 @@
|
||||||
<h2 class="section-title">cf-orch Integration</h2>
|
<h2 class="section-title">cf-orch Integration</h2>
|
||||||
<p class="section-desc">
|
<p class="section-desc">
|
||||||
Import SFT (supervised fine-tuning) candidates from cf-orch benchmark runs.
|
Import SFT (supervised fine-tuning) candidates from cf-orch benchmark runs.
|
||||||
|
Connection settings fall back to environment variables
|
||||||
|
(<code>CF_ORCH_URL</code>, <code>CF_LICENSE_KEY</code>, <code>OLLAMA_HOST</code>)
|
||||||
|
when not set here.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
|
<!-- Connection status pill -->
|
||||||
|
<div v-if="orchConfig" class="orch-status-row">
|
||||||
|
<span class="orch-status-pill" :class="orchStatusClass">{{ orchStatusLabel }}</span>
|
||||||
|
<span v-if="orchConfig.source === 'env'" class="orch-source-note">via env vars</span>
|
||||||
|
<span v-else class="orch-source-note">via label_tool.yaml</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="field-row">
|
<div class="field-row">
|
||||||
<label class="field field-grow">
|
<label class="field field-grow">
|
||||||
<span>bench_results_dir</span>
|
<span>bench_results_dir</span>
|
||||||
|
|
@ -181,7 +191,7 @@
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref, onMounted } from 'vue'
|
import { ref, computed, onMounted } from 'vue'
|
||||||
import { useApiFetch } from '../composables/useApi'
|
import { useApiFetch } from '../composables/useApi'
|
||||||
|
|
||||||
interface Account {
|
interface Account {
|
||||||
|
|
@ -199,12 +209,27 @@ const saveOk = ref(true)
|
||||||
const richMotion = ref(localStorage.getItem('cf-avocet-rich-motion') !== 'false')
|
const richMotion = ref(localStorage.getItem('cf-avocet-rich-motion') !== 'false')
|
||||||
const keyHints = ref(localStorage.getItem('cf-avocet-key-hints') !== 'false')
|
const keyHints = ref(localStorage.getItem('cf-avocet-key-hints') !== 'false')
|
||||||
|
|
||||||
// SFT integration state
|
// SFT / cf-orch integration state
|
||||||
const benchResultsDir = ref('')
|
const benchResultsDir = ref('')
|
||||||
const runs = ref<Array<{ run_id: string; timestamp: string; candidate_count: number; already_imported: boolean }>>([])
|
const runs = ref<Array<{ run_id: string; timestamp: string; candidate_count: number; already_imported: boolean }>>([])
|
||||||
const importingRunId = ref<string | null>(null)
|
const importingRunId = ref<string | null>(null)
|
||||||
const importResult = ref<{ imported: number; skipped: number } | null>(null)
|
const importResult = ref<{ imported: number; skipped: number } | null>(null)
|
||||||
const saveStatus = ref('')
|
const saveStatus = ref('')
|
||||||
|
const orchConfig = ref<{ coordinator_url: string; ollama_url: string; ollama_model: string; license_key_set: boolean; source: string } | null>(null)
|
||||||
|
|
||||||
|
const orchStatusClass = computed(() => {
|
||||||
|
if (!orchConfig.value) return 'status-unknown'
|
||||||
|
if (orchConfig.value.coordinator_url) return 'status-connected'
|
||||||
|
if (orchConfig.value.ollama_url) return 'status-local'
|
||||||
|
return 'status-unconfigured'
|
||||||
|
})
|
||||||
|
|
||||||
|
const orchStatusLabel = computed(() => {
|
||||||
|
if (!orchConfig.value) return 'Unknown'
|
||||||
|
if (orchConfig.value.coordinator_url) return '● cf-orch coordinator'
|
||||||
|
if (orchConfig.value.ollama_url) return '● Ollama (local)'
|
||||||
|
return '○ Not configured'
|
||||||
|
})
|
||||||
|
|
||||||
async function loadSftConfig() {
|
async function loadSftConfig() {
|
||||||
try {
|
try {
|
||||||
|
|
@ -218,6 +243,15 @@ async function loadSftConfig() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadOrchConfig() {
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/cforch/config')
|
||||||
|
if (res.ok) orchConfig.value = await res.json()
|
||||||
|
} catch {
|
||||||
|
// non-fatal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function saveSftConfig() {
|
async function saveSftConfig() {
|
||||||
saveStatus.value = 'Saving…'
|
saveStatus.value = 'Saving…'
|
||||||
try {
|
try {
|
||||||
|
|
@ -337,6 +371,7 @@ function onKeyHintsChange() {
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
reload()
|
reload()
|
||||||
loadSftConfig()
|
loadSftConfig()
|
||||||
|
loadOrchConfig()
|
||||||
})
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|
@ -564,6 +599,31 @@ onMounted(() => {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.orch-status-row {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-2);
|
||||||
|
margin-bottom: var(--space-3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.orch-status-pill {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
font-weight: 600;
|
||||||
|
padding: var(--space-1) var(--space-3);
|
||||||
|
border-radius: var(--radius-full);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-connected { background: color-mix(in srgb, var(--color-success, #3a7a32) 12%, transparent); color: var(--color-success, #3a7a32); }
|
||||||
|
.status-local { background: color-mix(in srgb, var(--color-primary) 12%, transparent); color: var(--color-primary); }
|
||||||
|
.status-unconfigured { background: var(--color-surface-alt); color: var(--color-text-muted); }
|
||||||
|
.status-unknown { background: var(--color-surface-alt); color: var(--color-text-muted); }
|
||||||
|
|
||||||
|
.orch-source-note {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
.runs-table {
|
.runs-table {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
border-collapse: collapse;
|
border-collapse: collapse;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue