53 lines
2.1 KiB
Bash
53 lines
2.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Collect recent journal logs from remote CircuitForge cluster nodes
|
|
# into /devl/turnstone-cluster/data/ for Turnstone to ingest.
|
|
#
|
|
# Local Heimdall sources (journal, Docker containers, network syslog) are
|
|
# handled by the Turnstone live watcher (watch.yaml) — no collection needed.
|
|
#
|
|
# Triggered by systemd timer: turnstone-cluster-collect.timer (every 15 min).
|
|
# Install: sudo cp <scripts>/turnstone-cluster-collect.* /etc/systemd/system/
|
|
# sudo systemctl daemon-reload && sudo systemctl enable --now turnstone-cluster-collect.timer
|
|
#
|
|
# Manual run:
|
|
# bash /Library/Development/CircuitForge/turnstone/scripts/collect_cluster_logs.sh
|
|
|
|
set -euo pipefail
|
|
|
|
DATA_DIR=/devl/turnstone-cluster/data
|
|
WINDOW="20 minutes ago"
|
|
SSH_OPTS="-o ConnectTimeout=5 -o BatchMode=yes -o StrictHostKeyChecking=no"
|
|
|
|
mkdir -p "${DATA_DIR}"
|
|
|
|
# ── Remote cluster nodes ──────────────────────────────────────────────────────
|
|
declare -A NODES=(
|
|
[navi]="${DATA_DIR}/navi-journal.jsonl"
|
|
[sif]="${DATA_DIR}/sif-journal.jsonl"
|
|
[cass]="${DATA_DIR}/cass-journal.jsonl"
|
|
[strahl]="${DATA_DIR}/strahl-journal.jsonl"
|
|
)
|
|
|
|
for node in "${!NODES[@]}"; do
|
|
outfile="${NODES[$node]}"
|
|
echo "${node}: collecting journal..."
|
|
if ssh ${SSH_OPTS} "${node}" true 2>/dev/null; then
|
|
ssh ${SSH_OPTS} "${node}" \
|
|
"journalctl --output=json --priority=0..5 --since '${WINDOW}' --no-pager 2>/dev/null || true" \
|
|
> "${outfile}" 2>/dev/null || { echo "${node}: ssh failed, skipping"; : > "${outfile}"; }
|
|
echo "${node}: $(wc -l < "${outfile}") entries"
|
|
else
|
|
echo "${node}: unreachable, skipping"
|
|
: > "${outfile}"
|
|
fi
|
|
done
|
|
|
|
# Ingest remote node journals directly via the cf Python environment.
|
|
TURNSTONE_DB=/devl/turnstone-cluster/data/turnstone.db \
|
|
/devl/miniconda3/envs/cf/bin/python \
|
|
/Library/Development/CircuitForge/turnstone/scripts/ingest_corpus.py \
|
|
--sources /devl/turnstone-cluster/patterns/sources-cluster.yaml \
|
|
--db /devl/turnstone-cluster/data/turnstone.db \
|
|
>> /devl/turnstone-cluster/data/ingest.log 2>&1
|
|
|
|
echo "collect_cluster_logs: done"
|