pyr0ball
72679787d3
Local Heimdall sources (journal, Docker containers, network syslog) are now tailed continuously by the built-in watcher via watch.yaml — no periodic collection needed for those. SSH collection of remote node journals is now handled by a systemd timer (turnstone-cluster-collect.service/.timer) instead of cron. collect_cluster_logs.sh simplified to only SSH-collect remote nodes and trigger ingest directly. docker-cluster.sh updated to mount: - /var/run/docker.sock (so watcher can run docker logs -f) - /run/systemd/journal (so watcher can run journalctl -f) - /devl/turnstone-cluster/patterns/ (cluster-specific watch.yaml)
106 lines
4.7 KiB
Bash
106 lines
4.7 KiB
Bash
#!/usr/bin/env bash
|
|
# docker-cluster.sh — Turnstone cluster monitoring instance on Heimdall.
|
|
#
|
|
# Local sources (Heimdall journal, Docker containers, network syslog) are
|
|
# tailed live by the built-in watcher (watch.yaml) — no periodic collection needed.
|
|
#
|
|
# Remote node journals (navi, sif, cass, strahl) are collected by a
|
|
# systemd timer every 15 minutes and ingested via ingest_corpus.py.
|
|
# Install the timer:
|
|
# sudo cp scripts/turnstone-cluster-collect.{service,timer} /etc/systemd/system/
|
|
# sudo systemctl daemon-reload && sudo systemctl enable --now turnstone-cluster-collect.timer
|
|
#
|
|
# ── Prerequisites ────────────────────────────────────────────────────────────
|
|
# SSH key access to navi, sif, cass, strahl (test: ssh <node> hostname)
|
|
#
|
|
# ── Run ───────────────────────────────────────────────────────────────────────
|
|
# bash /Library/Development/CircuitForge/turnstone/scripts/docker-cluster.sh
|
|
#
|
|
# ── Caddy reverse proxy (add to /devl/caddy-proxy/Caddyfile) ─────────────────
|
|
# turnstone.heimdall.lan {
|
|
# reverse_proxy 127.0.0.1:8534
|
|
# }
|
|
# Then: docker restart caddy-proxy
|
|
#
|
|
# ── Ports ────────────────────────────────────────────────────────────────────
|
|
# Turnstone UI → http://heimdall:8534/turnstone/
|
|
#
|
|
set -euo pipefail
|
|
|
|
REPO_DIR=/Library/Development/CircuitForge/turnstone
|
|
DATA_DIR=/devl/turnstone-cluster/data
|
|
PATTERNS_DIR=/devl/turnstone-cluster/patterns
|
|
PORT=8534
|
|
TZ=America/Los_Angeles
|
|
|
|
# LLM: route to local cf-orch coordinator (same host, host network).
|
|
LLM_URL="${TURNSTONE_LLM_URL:-http://127.0.0.1:7701}"
|
|
LLM_MODEL="${TURNSTONE_LLM_MODEL:-llama3.1:8b}"
|
|
LLM_API_KEY="${TURNSTONE_LLM_API_KEY:-}"
|
|
|
|
mkdir -p "${DATA_DIR}" "${PATTERNS_DIR}"
|
|
|
|
# Keep default.yaml in cluster patterns dir up to date with the repo copy.
|
|
cp "${REPO_DIR}/patterns/default.yaml" "${PATTERNS_DIR}/default.yaml"
|
|
|
|
# ── Seed LLM preferences (only if not already configured) ────────────────────
|
|
PREFS_FILE="${DATA_DIR}/preferences.json"
|
|
if [ ! -f "${PREFS_FILE}" ]; then
|
|
python3 -c "
|
|
import json
|
|
prefs = {
|
|
'llm_url': '${LLM_URL}',
|
|
'llm_model': '${LLM_MODEL}',
|
|
'llm_api_key': '${LLM_API_KEY}',
|
|
}
|
|
print(json.dumps(prefs))
|
|
" > "${PREFS_FILE}"
|
|
echo "Seeded ${PREFS_FILE} (llm_url=${LLM_URL}, model=${LLM_MODEL})"
|
|
else
|
|
echo "Preferences already exist at ${PREFS_FILE} — skipping seed"
|
|
fi
|
|
|
|
# Touch network-syslog.txt so the file watcher has something to tail
|
|
# before the syslog receiver writes to it.
|
|
touch "${DATA_DIR}/network-syslog.txt"
|
|
|
|
# ── Build image ───────────────────────────────────────────────────────────────
|
|
echo "Building Turnstone image..."
|
|
docker build -t circuitforge/turnstone:latest "${REPO_DIR}"
|
|
|
|
# ── Deploy container ──────────────────────────────────────────────────────────
|
|
docker rm -f turnstone-cluster 2>/dev/null || true
|
|
|
|
docker run -d \
|
|
--name=turnstone-cluster \
|
|
--restart=unless-stopped \
|
|
--net=host \
|
|
-v "${DATA_DIR}:/data" \
|
|
-v "${PATTERNS_DIR}:/patterns:ro" \
|
|
-v /var/run/docker.sock:/var/run/docker.sock \
|
|
-v /run/systemd/journal:/run/systemd/journal:ro \
|
|
-e TURNSTONE_DB=/data/turnstone.db \
|
|
-e TURNSTONE_SOURCE_HOST="heimdall-cluster" \
|
|
-e TURNSTONE_BUNDLE_ENDPOINT="${TURNSTONE_BUNDLE_ENDPOINT:-}" \
|
|
-e PYTHONUNBUFFERED=1 \
|
|
-e TZ="${TZ}" \
|
|
--health-cmd="curl -f http://localhost:${PORT}/turnstone/health || exit 1" \
|
|
--health-interval=30s \
|
|
--health-timeout=10s \
|
|
--health-start-period=20s \
|
|
--health-retries=3 \
|
|
circuitforge/turnstone:latest
|
|
|
|
echo ""
|
|
echo "Turnstone cluster is starting up."
|
|
echo " UI: http://heimdall:${PORT}/turnstone/"
|
|
echo " Live watching: Heimdall journal + Docker containers + network syslog"
|
|
echo " Remote nodes: install the systemd timer for periodic SSH collection"
|
|
echo ""
|
|
echo " sudo cp ${REPO_DIR}/scripts/turnstone-cluster-collect.{service,timer} /etc/systemd/system/"
|
|
echo " sudo systemctl daemon-reload && sudo systemctl enable --now turnstone-cluster-collect.timer"
|
|
echo ""
|
|
echo "Check container:"
|
|
echo " docker ps --filter name=turnstone-cluster"
|
|
echo " docker logs turnstone-cluster"
|
|
echo " curl http://localhost:${PORT}/turnstone/api/watch/status"
|