turnstone/scripts/docker-cluster.sh

#!/usr/bin/env bash
# docker-cluster.sh — Turnstone cluster monitoring instance on Heimdall.
#
# Ingests logs from the full CircuitForge cluster:
#   - Heimdall (local journal + dmesg)
#   - Navi, Sif, Cass, Strahl (SSH-collected journals)
#   - Docker services: cf-orch-coordinator, cf-web, cf-directus, caddy-proxy
#   - Network syslog (router, switches, UniFi APs — UDP 5140)
#
# Logs are pre-collected to /devl/turnstone-cluster/data/ by collect_cluster_logs.sh
# before each ingest run. This script only manages the container lifecycle.
#
# ── Prerequisites ────────────────────────────────────────────────────────────
#   1. SSH key access to navi, sif, cass, strahl (test: ssh <node> hostname)
#   2. Build the image first:
#        cd /Library/Development/CircuitForge/turnstone
#        docker build -t circuitforge/turnstone:latest .
#
#   3. Run this script:
#        bash /Library/Development/CircuitForge/turnstone/scripts/docker-cluster.sh
#
# ── Ingest cron (every 15 min — add to root's crontab: sudo crontab -e) ─────
#   */15 * * * * bash /Library/Development/CircuitForge/turnstone/scripts/collect_cluster_logs.sh \
#     && docker exec turnstone-cluster python scripts/ingest_corpus.py \
#       --sources /patterns/sources-cluster.yaml --db /data/turnstone.db \
#       >> /var/log/turnstone-cluster-ingest.log 2>&1
#
# ── Caddy reverse proxy (add to /devl/caddy-proxy/Caddyfile) ─────────────────
#   turnstone.heimdall.lan {
#     reverse_proxy 127.0.0.1:8535
#   }
#   Then: docker restart caddy-proxy
#
# ── Ports ────────────────────────────────────────────────────────────────────
#   Turnstone UI  → http://heimdall:8535/turnstone/
#
set -euo pipefail

REPO_DIR=/Library/Development/CircuitForge/turnstone
DATA_DIR=/devl/turnstone-cluster/data
PATTERNS_DIR="${REPO_DIR}/patterns"
PORT=8534
TZ=America/Los_Angeles

# LLM: route to local cf-orch coordinator (same host, host network).
# Coordinator runs at 127.0.0.1 inside --net=host, so localhost works directly.
# Override LLM_URL to point at a different backend if needed.
LLM_URL="${TURNSTONE_LLM_URL:-http://127.0.0.1:7701}"
LLM_MODEL="${TURNSTONE_LLM_MODEL:-llama3.1:8b}"
LLM_API_KEY="${TURNSTONE_LLM_API_KEY:-}"

mkdir -p "${DATA_DIR}"

# ── Seed LLM preferences (only if not already configured) ────────────────────
# preferences.json lives in the data dir and persists across container restarts.
# If it doesn't exist yet, write defaults pointing at the local cf-orch coordinator
# so the first ingest gets real summarization without manual UI config.
PREFS_FILE="${DATA_DIR}/preferences.json"
if [ ! -f "${PREFS_FILE}" ]; then
  python3 -c "
import json
prefs = {
    'llm_url': '${LLM_URL}',
    'llm_model': '${LLM_MODEL}',
    'llm_api_key': '${LLM_API_KEY}',
}
print(json.dumps(prefs))
" > "${PREFS_FILE}"
  echo "Seeded ${PREFS_FILE} (llm_url=${LLM_URL}, model=${LLM_MODEL})"
else
  echo "Preferences already exist at ${PREFS_FILE} — skipping seed"
fi

# ── Build image ───────────────────────────────────────────────────────────────
echo "Building Turnstone image..."
docker build -t circuitforge/turnstone:latest "${REPO_DIR}"

# ── Deploy container ──────────────────────────────────────────────────────────
docker rm -f turnstone-cluster 2>/dev/null || true

docker run -d \
  --name=turnstone-cluster \
  --restart=unless-stopped \
  --net=host \
  -v "${DATA_DIR}:/data" \
  -v "${PATTERNS_DIR}:/patterns:ro" \
  -e TURNSTONE_DB=/data/turnstone.db \
  -e TURNSTONE_SOURCE_HOST="heimdall-cluster" \
  -e TURNSTONE_BUNDLE_ENDPOINT="${TURNSTONE_BUNDLE_ENDPOINT:-}" \
  -e PYTHONUNBUFFERED=1 \
  -e TZ="${TZ}" \
  --health-cmd="curl -f http://localhost:${PORT}/turnstone/health || exit 1" \
  --health-interval=30s \
  --health-timeout=10s \
  --health-start-period=20s \
  --health-retries=3 \
  circuitforge/turnstone:latest

echo ""
echo "Turnstone cluster is starting up."
echo "  UI: http://heimdall:${PORT}/turnstone/"
echo ""

# ── systemd service (optional) ────────────────────────────────────────────────
# To create a systemd unit that auto-starts on boot:
#   sudo tee /etc/systemd/system/turnstone-cluster.service > /dev/null << 'EOF'
#   [Unit]
#   Description=Turnstone cluster log monitor
#   After=docker.service
#   Requires=docker.service
#
#   [Service]
#   Type=oneshot
#   RemainAfterExit=yes
#   ExecStart=/usr/bin/docker start turnstone-cluster
#   ExecStop=/usr/bin/docker stop turnstone-cluster
#
#   [Install]
#   WantedBy=multi-user.target
#   EOF
#   sudo systemctl daemon-reload
#   sudo systemctl enable --now turnstone-cluster

echo "Check container health with:"
echo "  docker ps --filter name=turnstone-cluster"
echo "  docker logs turnstone-cluster"
echo ""
echo "Ingest now:"
echo "  bash ${REPO_DIR}/scripts/collect_cluster_logs.sh && \\"
echo "    docker exec turnstone-cluster python scripts/ingest_corpus.py \\"
echo "      --sources /patterns/sources-cluster.yaml --db /data/turnstone.db"
echo ""
echo "To set up the 15-minute cron, add to root's crontab (sudo crontab -e):"
echo "  */15 * * * * bash ${REPO_DIR}/scripts/collect_cluster_logs.sh && \\"
echo "    docker exec turnstone-cluster python scripts/ingest_corpus.py \\"
echo "      --sources /patterns/sources-cluster.yaml --db /data/turnstone.db \\"
echo "      >> /var/log/turnstone-cluster-ingest.log 2>&1"