turnstone/manage.sh
pyr0ball cbdd681faf feat: plain-text and Plex log ingestors
- app/ingest/plex.py: Plex Media Server log parser
  Regex-based line parser for 'Mon DD, YYYY HH:MM:SS.mmm [pid] LEVEL - msg'
  format. Handles multi-line entries (stack traces). Detects plex_eae_failure
  and all other patterns via shared pattern library.
- app/ingest/plaintext.py: generic fallback parser for unrecognized formats
  Extracts timestamps (ISO 8601, syslog, common log) and severity via regex.
- pipeline.py: detect plex format via is_plex_log(); fall back to plaintext
  instead of skipping; process *.log files alongside *.jsonl; add ingest_file()
  for single-file ingestion.
- scripts/ingest_corpus.py: accept single file or directory as target
- manage.sh: ingest-plex command SSHes to Cass (or HOST arg), pulls
  Plex Media Server.log, and ingests it directly
2026-05-08 17:50:01 -07:00

237 lines
8.2 KiB
Bash
Executable file

#!/usr/bin/env bash
# manage.sh — Turnstone diagnostic intelligence layer
# Usage: ./manage.sh <command> [args]
set -euo pipefail
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
info() { echo -e "${BLUE}[turnstone]${NC} $*"; }
success() { echo -e "${GREEN}[turnstone]${NC} $*"; }
warn() { echo -e "${YELLOW}[turnstone]${NC} $*"; }
error() { echo -e "${RED}[turnstone]${NC} $*" >&2; exit 1; }
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
API_PORT=8534 # FastAPI: serves REST API + built Vue SPA
VITE_PORT=5174 # Vite HMR port in dev mode (proxies /api → 8534)
LOG_DIR="log"
API_PID_FILE=".turnstone-api.pid"
DB="${TURNSTONE_DB:-${SCRIPT_DIR}/data/turnstone.db}"
CONDA_BASE="${CONDA_BASE:-/devl/miniconda3}"
PYTHON="${CONDA_BASE}/envs/cf/bin/python"
# ── Helpers ───────────────────────────────────────────────────────────────────
_is_alive() {
local pid_file="$1"
[[ -f "$pid_file" ]] && kill -0 "$(<"$pid_file")" 2>/dev/null
}
_kill_pid_file() {
local pid_file="$1" label="$2"
if [[ -f "$pid_file" ]]; then
local pid
pid=$(<"$pid_file")
if kill -0 "$pid" 2>/dev/null; then
kill "$pid" && rm -f "$pid_file"
success "$label stopped (PID $pid)."
else
warn "Stale PID file for $label (PID $pid not running). Cleaning up."
rm -f "$pid_file"
fi
else
warn "$label not running."
fi
}
_wait_for_port() {
local port="$1" label="$2" pid_file="$3"
for _i in $(seq 1 20); do
sleep 0.5
(echo "" >/dev/tcp/127.0.0.1/"$port") 2>/dev/null && return 0
if ! _is_alive "$pid_file"; then
rm -f "$pid_file"
error "$label died during startup. Check ${LOG_DIR}/api.log"
fi
done
error "$label did not bind to port $port within 10 s."
}
# ── Usage ─────────────────────────────────────────────────────────────────────
usage() {
echo ""
echo -e " ${BLUE}Turnstone — Diagnostic Log Intelligence${NC}"
echo ""
echo " Usage: ./manage.sh <command> [args]"
echo ""
echo " Production-like (built SPA + uvicorn):"
echo -e " ${GREEN}start${NC} Build Vue SPA, start FastAPI + SPA on :${API_PORT}"
echo -e " ${GREEN}stop${NC} Stop the server"
echo -e " ${GREEN}restart${NC} Stop then start"
echo -e " ${GREEN}status${NC} Show running process"
echo -e " ${GREEN}logs${NC} Tail server log"
echo -e " ${GREEN}open${NC} Open UI in browser"
echo ""
echo " Development (hot-reload):"
echo -e " ${GREEN}dev${NC} uvicorn --reload (:${API_PORT}) + Vite HMR (:${VITE_PORT})"
echo ""
echo " Data:"
echo -e " ${GREEN}ingest PATH [DB]${NC} Ingest a log file or corpus directory"
echo -e " ${GREEN}ingest-plex [HOST]${NC} Pull Plex log from Cass (or HOST) and ingest"
echo -e " ${GREEN}build-fts${NC} Rebuild the FTS search index"
echo ""
echo " Tests:"
echo -e " ${GREEN}test [args]${NC} Run pytest suite"
echo ""
echo " DB: ${DB}"
echo " Conda env: cf"
echo ""
echo " Examples:"
echo " ./manage.sh start"
echo " ./manage.sh dev"
echo " ./manage.sh ingest corpus/raw/"
echo " ./manage.sh ingest corpus/raw/ data/custom.db"
echo ""
}
# ── Commands ──────────────────────────────────────────────────────────────────
CMD="${1:-help}"
shift || true
case "$CMD" in
start)
if _is_alive "$API_PID_FILE"; then
warn "Already running (PID $(<"$API_PID_FILE")) — use 'restart' to rebuild."
exit 0
fi
mkdir -p "$LOG_DIR" data
info "Building Vue SPA…"
(cd web && npm run build) 2>&1 | tee "${LOG_DIR}/build.log" | grep -E "built in|error" || true
success "SPA built → web/dist/"
info "Starting on port ${API_PORT}"
TURNSTONE_DB="$DB" nohup "$PYTHON" -m uvicorn app.rest:app \
--host 0.0.0.0 --port "$API_PORT" \
>> "${LOG_DIR}/api.log" 2>&1 &
echo $! > "$API_PID_FILE"
_wait_for_port "$API_PORT" "Turnstone" "$API_PID_FILE"
success "Running → http://localhost:${API_PORT} (PID $(<"$API_PID_FILE"))"
;;
stop)
_kill_pid_file "$API_PID_FILE" "Turnstone"
;;
restart)
bash "$0" stop
exec bash "$0" start
;;
status)
echo ""
if _is_alive "$API_PID_FILE"; then
success "Turnstone RUNNING PID $(<"$API_PID_FILE") → http://localhost:${API_PORT}"
else
echo -e " Turnstone ${RED}STOPPED${NC}"
fi
echo ""
;;
logs)
tail -f "${LOG_DIR}/api.log"
;;
open)
URL="http://localhost:${API_PORT}"
info "Opening ${URL}"
if command -v xdg-open &>/dev/null; then xdg-open "$URL"
elif command -v open &>/dev/null; then open "$URL"
else echo "$URL"
fi
;;
dev)
DEV_API_PID=".turnstone-dev-api.pid"
mkdir -p "$LOG_DIR" data
if _is_alive "$DEV_API_PID"; then
warn "Dev API already running (PID $(<"$DEV_API_PID"))"
else
info "Starting uvicorn --reload on port ${API_PORT}"
TURNSTONE_DB="$DB" nohup "$PYTHON" -m uvicorn app.rest:app \
--host 0.0.0.0 --port "$API_PORT" --reload \
>> "${LOG_DIR}/api.log" 2>&1 &
echo $! > "$DEV_API_PID"
_wait_for_port "$API_PORT" "FastAPI (dev)" "$DEV_API_PID"
success "API (hot-reload) → http://localhost:${API_PORT}"
fi
_cleanup_dev() {
local pid
pid=$(<"$DEV_API_PID" 2>/dev/null) || true
[[ -n "${pid:-}" ]] && kill "$pid" 2>/dev/null && rm -f "$DEV_API_PID"
info "Dev servers stopped."
}
trap _cleanup_dev EXIT INT TERM
info "Starting Vite HMR on port ${VITE_PORT}"
success "Frontend (HMR) → http://localhost:${VITE_PORT}"
(cd web && npm run dev -- --port "$VITE_PORT")
;;
ingest)
if [[ $# -lt 1 ]]; then
error "Usage: ./manage.sh ingest <file_or_dir> [DB_PATH]"
fi
info "Ingesting $1${2:-$DB}"
"$PYTHON" scripts/ingest_corpus.py "$1" "${2:-$DB}"
;;
ingest-plex)
PLEX_HOST="${1:-cass}"
PLEX_LOG_PATH="/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Logs/Plex Media Server.log"
TMP_LOG="/tmp/turnstone-plex-$(date +%s).log"
info "Pulling Plex log from ${PLEX_HOST}"
if ! ssh "$PLEX_HOST" "cat '${PLEX_LOG_PATH}'" > "$TMP_LOG" 2>&1; then
rm -f "$TMP_LOG"
error "SSH to ${PLEX_HOST} failed. Ensure 'ssh ${PLEX_HOST}' works without a password prompt."
fi
LINES=$(wc -l < "$TMP_LOG")
success "Pulled ${LINES} lines from ${PLEX_HOST}"
info "Ingesting into ${DB}"
"$PYTHON" scripts/ingest_corpus.py "$TMP_LOG" "$DB"
rm -f "$TMP_LOG"
success "Done. Restart the server to refresh: ./manage.sh restart"
;;
build-fts)
info "Rebuilding FTS index for ${DB}"
TURNSTONE_DB="$DB" "$PYTHON" scripts/build_fts_index.py "$DB"
success "FTS index rebuilt."
;;
test)
info "Running test suite…"
PYTEST="${CONDA_BASE}/envs/cf/bin/pytest"
[[ -x "$PYTEST" ]] || error "pytest not found in cf env at ${PYTEST}"
TURNSTONE_DB=":memory:" "$PYTEST" tests/ -v "$@"
;;
help|--help|-h)
usage
;;
*)
error "Unknown command: ${CMD}. Run './manage.sh help' for usage."
;;
esac