pyr0ball
63c742a708
Adds asyncio-native background scheduler (TURNSTONE_INGEST_INTERVAL, default 900s) that runs batch ingest then pushes pattern-matched entries to a remote CF harvest endpoint (TURNSTONE_SUBMIT_ENDPOINT). - app/tasks/ingest_scheduler.py: IngestState, scheduler_loop, run_once, submit_matched, _query_matched_since — asyncio.Lock prevents concurrent runs - app/rest.py: POST /api/ingest/batch (pre-parsed entry receiver), GET /api/tasks/ingest/status, POST /api/tasks/ingest (manual trigger), TURNSTONE_INGEST_INTERVAL + TURNSTONE_SUBMIT_ENDPOINT env wiring in lifespan - docker-compose.submissions.yml: segregated contrib1 (8536) + contrib2 (8537) receiving instances on Heimdall, isolated DBs under /devl/docker/turnstone-submissions/<node>/ - podman-standalone.sh: pass-through for TURNSTONE_SUBMIT_ENDPOINT + TURNSTONE_SOURCE_HOST - app/ingest/mqtt_subscriber.py: MQTT log source adapter - app/ingest/wazuh.py: Wazuh alert JSON adapter - tests/test_ingest_wazuh.py: Wazuh adapter test suite
96 lines
3.7 KiB
YAML
96 lines
3.7 KiB
YAML
# Turnstone log sources — edit this file to add or remove services.
|
|
# NOTE: the system-journal entry requires export_journal.sh to run on the HOST
|
|
# before the container ingest step. See crontab setup instructions in the README.
|
|
# Run ingest manually:
|
|
# sudo podman exec turnstone python scripts/ingest_corpus.py \
|
|
# --sources /patterns/sources.yaml --db /data/turnstone.db
|
|
#
|
|
# Paths here are container-side paths under the /opt bind mount.
|
|
# Missing paths are skipped with a warning — safe to leave entries for
|
|
# services that are temporarily down.
|
|
|
|
sources:
|
|
# ── System (exported by export_journal.sh on the host) ───────────────────
|
|
# journal-export.jsonl and dmesg-export.txt are written to /opt/turnstone/data/
|
|
# by the export script before each ingest run.
|
|
- id: system-journal
|
|
path: /data/journal-export.jsonl
|
|
|
|
- id: dmesg
|
|
path: /data/dmesg-export.txt
|
|
|
|
# ── Syslog / rsyslog (direct file reads via /var/log bind mount) ──────────
|
|
# Uncomment the file(s) present on your system.
|
|
# Debian/Ubuntu:
|
|
# - id: syslog
|
|
# path: /var/log/syslog
|
|
# - id: auth-log
|
|
# path: /var/log/auth.log
|
|
# - id: kern-log
|
|
# path: /var/log/kern.log
|
|
# RHEL/Fedora/Rocky:
|
|
# - id: messages
|
|
# path: /var/log/messages
|
|
# - id: secure
|
|
# path: /var/log/secure
|
|
|
|
# ── Download ─────────────────────────────────────────────────────────────
|
|
- id: qbittorrent
|
|
path: /opt/qbittorrent/config/data/logs/qbittorrent.log
|
|
|
|
# ── Servarr stack ─────────────────────────────────────────────────────────
|
|
- id: sonarr
|
|
path: /opt/sonarr/config/logs/sonarr.0.txt
|
|
|
|
- id: radarr
|
|
path: /opt/radarr/config/logs/radarr.0.txt
|
|
|
|
- id: lidarr
|
|
path: /opt/lidarr/config/logs/Lidarr.0.txt
|
|
|
|
- id: readarr
|
|
path: /opt/readarr/config/logs/readarr.0.txt
|
|
|
|
- id: whisparr
|
|
path: /opt/whisparr/config/logs/whisparr.0.txt
|
|
|
|
- id: prowlarr
|
|
path: /opt/prowlarr/config/logs/prowlarr.0.txt
|
|
|
|
- id: bazarr
|
|
path: /opt/bazarr/config/log/bazarr.log
|
|
|
|
# ── Usenet ────────────────────────────────────────────────────────────────
|
|
- id: nzbget
|
|
path: /opt/nzbget/config/nzbget.log
|
|
|
|
# ── Media / Requests ─────────────────────────────────────────────────────
|
|
- id: tautulli
|
|
path: /opt/tautulli/config/logs/tautulli.log
|
|
|
|
- id: jellyseerr
|
|
path: /opt/jellyseerr/config/logs/jellyseerr.log
|
|
|
|
# ── MQTT / IoT (live — subscribe mode, no path needed) ───────────────────
|
|
# Requires: pip install circuitforge-core[mqtt]
|
|
# These sources are handled by the live MQTT subscriber task (not batch ingest).
|
|
# Uncomment and configure to enable.
|
|
#
|
|
# Meshtastic MQTT bridge (node must have MQTT uplink enabled):
|
|
# - id: meshtastic-home
|
|
# type: mqtt
|
|
# broker_host: 10.1.10.5 # IP of your local MQTT broker (e.g. Mosquitto on Huginn)
|
|
# broker_port: 1883
|
|
# topics:
|
|
# - msh/# # all Meshtastic regions; use msh/us-east/# to narrow
|
|
#
|
|
# Generic IoT sensors:
|
|
# - id: iot-home
|
|
# type: mqtt
|
|
# broker_host: localhost
|
|
# broker_port: 1883
|
|
# topics:
|
|
# - home/+/temperature
|
|
# - home/+/humidity
|
|
# - home/+/motion
|
|
# severity: INFO
|