feat(container): multi-agent env vars, HF cache mount, and ML deps
podman-standalone.sh: - Add HF_CACHE_DIR=/opt/turnstone/hf-cache with mkdir guard - Mount HF_HOME=/hf-cache so model weights persist across restarts - Forward all multi-agent env vars (TURNSTONE_MULTI_AGENT_DIAGNOSE, GPU_SERVER_URL, TURNSTONE_CLASSIFIER_MODEL, TURNSTONE_EMBED_*) - Add documentation comments for Contributor/Contributor2 remote instance setup requirements.txt: - Add torch (CPU-only), transformers, sentence-transformers for the 5-stage multi-agent diagnose pipeline (classifier + suppressor stages) - Use --extra-index-url for cpu wheel to keep image ~2GB lighter - Both modules keep ImportError guards so server starts without them, but container images should ship fully capable
This commit is contained in:
parent
3cfd587d16
commit
2a4a5a5152
2 changed files with 40 additions and 0 deletions
|
|
@ -62,6 +62,7 @@ set -euo pipefail
|
||||||
REPO_DIR=/opt/turnstone
|
REPO_DIR=/opt/turnstone
|
||||||
DATA_DIR=/opt/turnstone/data
|
DATA_DIR=/opt/turnstone/data
|
||||||
PATTERNS_DIR=/opt/turnstone/patterns
|
PATTERNS_DIR=/opt/turnstone/patterns
|
||||||
|
HF_CACHE_DIR=/opt/turnstone/hf-cache # persists downloaded ML models across restarts
|
||||||
TZ=America/Los_Angeles
|
TZ=America/Los_Angeles
|
||||||
|
|
||||||
# ── Bundle push configuration ────────────────────────────────────────────────
|
# ── Bundle push configuration ────────────────────────────────────────────────
|
||||||
|
|
@ -80,6 +81,25 @@ TZ=America/Los_Angeles
|
||||||
# bash /opt/turnstone/podman-standalone.sh
|
# bash /opt/turnstone/podman-standalone.sh
|
||||||
#
|
#
|
||||||
# TURNSTONE_SOURCE_HOST is auto-detected from `hostname` — override if needed.
|
# TURNSTONE_SOURCE_HOST is auto-detected from `hostname` — override if needed.
|
||||||
|
#
|
||||||
|
# ── Multi-agent diagnose pipeline ────────────────────────────────────────────
|
||||||
|
# The 5-stage ML pipeline requires three env vars and a writable HF cache dir:
|
||||||
|
#
|
||||||
|
# TURNSTONE_MULTI_AGENT_DIAGNOSE=true — enable the pipeline
|
||||||
|
# GPU_SERVER_URL=http://<orch-host>:7700 — cf-orch coordinator or Ollama base URL
|
||||||
|
#
|
||||||
|
# ML models are downloaded on first diagnose run and cached in HF_CACHE_DIR.
|
||||||
|
# On a CPU-only host (no GPU) set TURNSTONE_EMBED_DEVICE=cpu (default).
|
||||||
|
#
|
||||||
|
# For Contributor2's instance (example-node.tv) — Heimdall's cf-orch via WireGuard:
|
||||||
|
# export GPU_SERVER_URL=http://<YOUR_HOST_IP>:7700
|
||||||
|
# export TURNSTONE_MULTI_AGENT_DIAGNOSE=true
|
||||||
|
# sudo bash /opt/turnstone/podman-standalone.sh
|
||||||
|
#
|
||||||
|
# For Contributor's instance (Huginn) — same cf-orch via WireGuard:
|
||||||
|
# export GPU_SERVER_URL=http://<YOUR_HOST_IP>:7700
|
||||||
|
# export TURNSTONE_MULTI_AGENT_DIAGNOSE=true
|
||||||
|
# bash /opt/turnstone/podman-standalone.sh (or rebuild Docker container)
|
||||||
|
|
||||||
# ── Turnstone container ───────────────────────────────────────────────────────
|
# ── Turnstone container ───────────────────────────────────────────────────────
|
||||||
# Image is built locally — no registry auto-update label.
|
# Image is built locally — no registry auto-update label.
|
||||||
|
|
@ -96,6 +116,9 @@ TZ=America/Los_Angeles
|
||||||
echo "Building Turnstone image..."
|
echo "Building Turnstone image..."
|
||||||
podman build -t localhost/turnstone:latest "${REPO_DIR}"
|
podman build -t localhost/turnstone:latest "${REPO_DIR}"
|
||||||
|
|
||||||
|
# Create HF model cache dir if not present (persists across container rebuilds)
|
||||||
|
mkdir -p "${HF_CACHE_DIR}"
|
||||||
|
|
||||||
# Remove existing container if present (safe re-run)
|
# Remove existing container if present (safe re-run)
|
||||||
podman rm -f turnstone 2>/dev/null || true
|
podman rm -f turnstone 2>/dev/null || true
|
||||||
|
|
||||||
|
|
@ -105,6 +128,7 @@ podman run -d \
|
||||||
--net=host \
|
--net=host \
|
||||||
-v "${DATA_DIR}:/data:Z" \
|
-v "${DATA_DIR}:/data:Z" \
|
||||||
-v "${PATTERNS_DIR}:/patterns:Z" \
|
-v "${PATTERNS_DIR}:/patterns:Z" \
|
||||||
|
-v "${HF_CACHE_DIR}:/hf-cache:Z" \
|
||||||
-v /opt:/opt:ro \
|
-v /opt:/opt:ro \
|
||||||
-v /var/log:/var/log:ro \
|
-v /var/log:/var/log:ro \
|
||||||
-e TURNSTONE_DB=/data/turnstone.db \
|
-e TURNSTONE_DB=/data/turnstone.db \
|
||||||
|
|
@ -113,6 +137,13 @@ podman run -d \
|
||||||
-e TURNSTONE_SUBMIT_ENDPOINT="${TURNSTONE_SUBMIT_ENDPOINT:-}" \
|
-e TURNSTONE_SUBMIT_ENDPOINT="${TURNSTONE_SUBMIT_ENDPOINT:-}" \
|
||||||
-e PYTHONUNBUFFERED=1 \
|
-e PYTHONUNBUFFERED=1 \
|
||||||
-e TZ="${TZ}" \
|
-e TZ="${TZ}" \
|
||||||
|
-e TURNSTONE_MULTI_AGENT_DIAGNOSE="${TURNSTONE_MULTI_AGENT_DIAGNOSE:-false}" \
|
||||||
|
-e GPU_SERVER_URL="${GPU_SERVER_URL:-}" \
|
||||||
|
-e HF_HOME=/hf-cache \
|
||||||
|
-e TURNSTONE_CLASSIFIER_MODEL="${TURNSTONE_CLASSIFIER_MODEL:-byviz/bylastic_classification_logs}" \
|
||||||
|
-e TURNSTONE_EMBED_BACKEND="${TURNSTONE_EMBED_BACKEND:-sentence_transformers}" \
|
||||||
|
-e TURNSTONE_EMBED_MODEL="${TURNSTONE_EMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2}" \
|
||||||
|
-e TURNSTONE_EMBED_DEVICE="${TURNSTONE_EMBED_DEVICE:-cpu}" \
|
||||||
--health-cmd="curl -f http://localhost:8534/turnstone/health || exit 1" \
|
--health-cmd="curl -f http://localhost:8534/turnstone/health || exit 1" \
|
||||||
--health-interval=30s \
|
--health-interval=30s \
|
||||||
--health-timeout=10s \
|
--health-timeout=10s \
|
||||||
|
|
|
||||||
|
|
@ -7,3 +7,12 @@ python-multipart>=0.0.9
|
||||||
dateparser>=1.2.0
|
dateparser>=1.2.0
|
||||||
httpx>=0.27.0
|
httpx>=0.27.0
|
||||||
paramiko
|
paramiko
|
||||||
|
|
||||||
|
# Multi-agent diagnose pipeline — ML deps
|
||||||
|
# classifier.py and suppressor.py have ImportError guards and fall back gracefully,
|
||||||
|
# but these are included unconditionally so container images are fully capable.
|
||||||
|
# Install CPU-only torch to avoid pulling the ~2GB CUDA wheel into the image.
|
||||||
|
--extra-index-url https://download.pytorch.org/whl/cpu
|
||||||
|
torch>=2.2.0
|
||||||
|
transformers>=4.40.0
|
||||||
|
sentence-transformers>=3.0.0
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue