feat: journald export + system failure patterns

- Add scripts/export_journal.sh — dumps recent journal (priority 0-5,
  20min window) to /opt/turnstone/data/journal-export.jsonl; idempotent
  via entry_id deduplication so overlap is safe
- Add system-journal source to sources.yaml pointing at the export file
- Add 9 system-level patterns to default.yaml:
  systemd_fail, oom_kill, disk_hw_error, fs_error, kernel_error,
  ssh_brute, container_crash, smart_error, nfs_error
This commit is contained in:
pyr0ball 2026-05-11 06:54:42 -07:00
parent f46aba8165
commit 286778d6a9
3 changed files with 85 additions and 0 deletions

View file

@ -81,6 +81,53 @@ patterns:
severity: INFO severity: INFO
description: IP address change or DHCP event description: IP address change or DHCP event
# ── System / journald patterns ─────────────────────────────────────────────
- name: systemd_fail
pattern: "(Failed to start|failed with result|entered failed state|start request repeated too quickly|Main process exited)"
severity: ERROR
description: systemd service failed to start or crashed
- name: oom_kill
pattern: "(Killed process|oom.kill|oom_kill_process|Out of memory: Kill|memory cgroup out of memory)"
severity: CRITICAL
description: Kernel OOM killer terminated a process
- name: disk_hw_error
pattern: "(ata[0-9]|sd[a-z]|nvme[0-9]).*(error|failed|reset|timeout|exception|EH|FAILED COMMAND)"
severity: ERROR
description: Storage device hardware error or reset
- name: fs_error
pattern: "(EXT4-fs error|XFS.*error|BTRFS.*error|I/O error|blk_update_request.*error|buffer I/O error)"
severity: ERROR
description: Filesystem or block I/O error
- name: kernel_error
pattern: "(kernel: BUG|kernel panic|Oops:|general protection fault|Call Trace|RIP:.*[0-9a-f]{16})"
severity: CRITICAL
description: Kernel bug, panic, or oops — system may be unstable
- name: ssh_brute
pattern: "(Failed password|Invalid user|authentication failure|Connection closed by authenticating user).*(sshd|ssh)"
severity: WARN
description: SSH authentication failure — possible brute force
- name: container_crash
pattern: "(container.*exited|oci runtime.*error|podman.*error|docker.*error|container.*killed|OCI.*failed)"
severity: ERROR
description: Container runtime error or unexpected exit
- name: smart_error
pattern: "(smartd|SMART.*error|reallocated sector|pending sector|uncorrectable sector|Current_Pending_Sector)"
severity: CRITICAL
description: SMART disk health warning — potential drive failure
- name: nfs_error
pattern: "(nfs.*error|nfs.*timeout|RPC.*timed out|nfs4.*server.*not responding|mount.*nfs.*failed)"
severity: ERROR
description: NFS mount or RPC timeout
# Add device/service-specific patterns below this line: # Add device/service-specific patterns below this line:
- name: qbit_tracker_error - name: qbit_tracker_error

View file

@ -1,4 +1,6 @@
# Turnstone log sources — edit this file to add or remove services. # Turnstone log sources — edit this file to add or remove services.
# NOTE: the system-journal entry requires export_journal.sh to run on the HOST
# before the container ingest step. See crontab setup instructions in the README.
# Run ingest manually: # Run ingest manually:
# sudo podman exec turnstone python scripts/ingest_corpus.py \ # sudo podman exec turnstone python scripts/ingest_corpus.py \
# --sources /patterns/sources.yaml --db /data/turnstone.db # --sources /patterns/sources.yaml --db /data/turnstone.db
@ -8,6 +10,10 @@
# services that are temporarily down. # services that are temporarily down.
sources: sources:
# ── System journal (exported by export_journal.sh on the host) ────────────
- id: system-journal
path: /data/journal-export.jsonl
# ── Download ───────────────────────────────────────────────────────────── # ── Download ─────────────────────────────────────────────────────────────
- id: qbittorrent - id: qbittorrent
path: /opt/qbittorrent/config/data/logs/qbittorrent.log path: /opt/qbittorrent/config/data/logs/qbittorrent.log

32
scripts/export_journal.sh Normal file
View file

@ -0,0 +1,32 @@
#!/usr/bin/env bash
# Export recent journald entries to a JSONL file the Turnstone container can ingest.
#
# Run this on the HOST before the container ingest step. The output file lands in
# /opt/turnstone/data/ which is bind-mounted at /data/ inside the container.
#
# Priority filter 0-5 (emerg→notice) skips debug/info noise while keeping
# all warnings, errors, and service lifecycle events.
#
# Usage (standalone):
# sudo bash /opt/turnstone/scripts/export_journal.sh
#
# Typical cron (combined with ingest — see crontab comment below):
# */15 * * * * bash /opt/turnstone/scripts/export_journal.sh && \
# podman exec turnstone python scripts/ingest_corpus.py \
# --sources /patterns/sources.yaml --db /data/turnstone.db \
# >> /var/log/turnstone-ingest.log 2>&1
set -euo pipefail
OUT=/opt/turnstone/data/journal-export.jsonl
# 20-minute window (slightly wider than the 15-min cron interval) ensures no
# gaps between runs. Ingest is idempotent via entry_id hash, so overlap is safe.
journalctl \
--output=json \
--priority=0..5 \
--since "20 minutes ago" \
--no-pager \
> "${OUT}"
echo "Exported $(wc -l < "${OUT}") journal entries to ${OUT}"