feat: journald export + system failure patterns
- Add scripts/export_journal.sh — dumps recent journal (priority 0-5, 20min window) to /opt/turnstone/data/journal-export.jsonl; idempotent via entry_id deduplication so overlap is safe - Add system-journal source to sources.yaml pointing at the export file - Add 9 system-level patterns to default.yaml: systemd_fail, oom_kill, disk_hw_error, fs_error, kernel_error, ssh_brute, container_crash, smart_error, nfs_error
This commit is contained in:
parent
ec931598ca
commit
a37ec559aa
3 changed files with 85 additions and 0 deletions
|
|
@ -81,6 +81,53 @@ patterns:
|
||||||
severity: INFO
|
severity: INFO
|
||||||
description: IP address change or DHCP event
|
description: IP address change or DHCP event
|
||||||
|
|
||||||
|
# ── System / journald patterns ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
- name: systemd_fail
|
||||||
|
pattern: "(Failed to start|failed with result|entered failed state|start request repeated too quickly|Main process exited)"
|
||||||
|
severity: ERROR
|
||||||
|
description: systemd service failed to start or crashed
|
||||||
|
|
||||||
|
- name: oom_kill
|
||||||
|
pattern: "(Killed process|oom.kill|oom_kill_process|Out of memory: Kill|memory cgroup out of memory)"
|
||||||
|
severity: CRITICAL
|
||||||
|
description: Kernel OOM killer terminated a process
|
||||||
|
|
||||||
|
- name: disk_hw_error
|
||||||
|
pattern: "(ata[0-9]|sd[a-z]|nvme[0-9]).*(error|failed|reset|timeout|exception|EH|FAILED COMMAND)"
|
||||||
|
severity: ERROR
|
||||||
|
description: Storage device hardware error or reset
|
||||||
|
|
||||||
|
- name: fs_error
|
||||||
|
pattern: "(EXT4-fs error|XFS.*error|BTRFS.*error|I/O error|blk_update_request.*error|buffer I/O error)"
|
||||||
|
severity: ERROR
|
||||||
|
description: Filesystem or block I/O error
|
||||||
|
|
||||||
|
- name: kernel_error
|
||||||
|
pattern: "(kernel: BUG|kernel panic|Oops:|general protection fault|Call Trace|RIP:.*[0-9a-f]{16})"
|
||||||
|
severity: CRITICAL
|
||||||
|
description: Kernel bug, panic, or oops — system may be unstable
|
||||||
|
|
||||||
|
- name: ssh_brute
|
||||||
|
pattern: "(Failed password|Invalid user|authentication failure|Connection closed by authenticating user).*(sshd|ssh)"
|
||||||
|
severity: WARN
|
||||||
|
description: SSH authentication failure — possible brute force
|
||||||
|
|
||||||
|
- name: container_crash
|
||||||
|
pattern: "(container.*exited|oci runtime.*error|podman.*error|docker.*error|container.*killed|OCI.*failed)"
|
||||||
|
severity: ERROR
|
||||||
|
description: Container runtime error or unexpected exit
|
||||||
|
|
||||||
|
- name: smart_error
|
||||||
|
pattern: "(smartd|SMART.*error|reallocated sector|pending sector|uncorrectable sector|Current_Pending_Sector)"
|
||||||
|
severity: CRITICAL
|
||||||
|
description: SMART disk health warning — potential drive failure
|
||||||
|
|
||||||
|
- name: nfs_error
|
||||||
|
pattern: "(nfs.*error|nfs.*timeout|RPC.*timed out|nfs4.*server.*not responding|mount.*nfs.*failed)"
|
||||||
|
severity: ERROR
|
||||||
|
description: NFS mount or RPC timeout
|
||||||
|
|
||||||
# Add device/service-specific patterns below this line:
|
# Add device/service-specific patterns below this line:
|
||||||
|
|
||||||
- name: qbit_tracker_error
|
- name: qbit_tracker_error
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
# Turnstone log sources — edit this file to add or remove services.
|
# Turnstone log sources — edit this file to add or remove services.
|
||||||
|
# NOTE: the system-journal entry requires export_journal.sh to run on the HOST
|
||||||
|
# before the container ingest step. See crontab setup instructions in the README.
|
||||||
# Run ingest manually:
|
# Run ingest manually:
|
||||||
# sudo podman exec turnstone python scripts/ingest_corpus.py \
|
# sudo podman exec turnstone python scripts/ingest_corpus.py \
|
||||||
# --sources /patterns/sources.yaml --db /data/turnstone.db
|
# --sources /patterns/sources.yaml --db /data/turnstone.db
|
||||||
|
|
@ -8,6 +10,10 @@
|
||||||
# services that are temporarily down.
|
# services that are temporarily down.
|
||||||
|
|
||||||
sources:
|
sources:
|
||||||
|
# ── System journal (exported by export_journal.sh on the host) ────────────
|
||||||
|
- id: system-journal
|
||||||
|
path: /data/journal-export.jsonl
|
||||||
|
|
||||||
# ── Download ─────────────────────────────────────────────────────────────
|
# ── Download ─────────────────────────────────────────────────────────────
|
||||||
- id: qbittorrent
|
- id: qbittorrent
|
||||||
path: /opt/qbittorrent/config/data/logs/qbittorrent.log
|
path: /opt/qbittorrent/config/data/logs/qbittorrent.log
|
||||||
|
|
|
||||||
32
scripts/export_journal.sh
Normal file
32
scripts/export_journal.sh
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Export recent journald entries to a JSONL file the Turnstone container can ingest.
|
||||||
|
#
|
||||||
|
# Run this on the HOST before the container ingest step. The output file lands in
|
||||||
|
# /opt/turnstone/data/ which is bind-mounted at /data/ inside the container.
|
||||||
|
#
|
||||||
|
# Priority filter 0-5 (emerg→notice) skips debug/info noise while keeping
|
||||||
|
# all warnings, errors, and service lifecycle events.
|
||||||
|
#
|
||||||
|
# Usage (standalone):
|
||||||
|
# sudo bash /opt/turnstone/scripts/export_journal.sh
|
||||||
|
#
|
||||||
|
# Typical cron (combined with ingest — see crontab comment below):
|
||||||
|
# */15 * * * * bash /opt/turnstone/scripts/export_journal.sh && \
|
||||||
|
# podman exec turnstone python scripts/ingest_corpus.py \
|
||||||
|
# --sources /patterns/sources.yaml --db /data/turnstone.db \
|
||||||
|
# >> /var/log/turnstone-ingest.log 2>&1
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
OUT=/opt/turnstone/data/journal-export.jsonl
|
||||||
|
|
||||||
|
# 20-minute window (slightly wider than the 15-min cron interval) ensures no
|
||||||
|
# gaps between runs. Ingest is idempotent via entry_id hash, so overlap is safe.
|
||||||
|
journalctl \
|
||||||
|
--output=json \
|
||||||
|
--priority=0..5 \
|
||||||
|
--since "20 minutes ago" \
|
||||||
|
--no-pager \
|
||||||
|
> "${OUT}"
|
||||||
|
|
||||||
|
echo "Exported $(wc -l < "${OUT}") journal entries to ${OUT}"
|
||||||
Loading…
Reference in a new issue