pyr0ball
63c742a708
Adds asyncio-native background scheduler (TURNSTONE_INGEST_INTERVAL, default 900s) that runs batch ingest then pushes pattern-matched entries to a remote CF harvest endpoint (TURNSTONE_SUBMIT_ENDPOINT). - app/tasks/ingest_scheduler.py: IngestState, scheduler_loop, run_once, submit_matched, _query_matched_since — asyncio.Lock prevents concurrent runs - app/rest.py: POST /api/ingest/batch (pre-parsed entry receiver), GET /api/tasks/ingest/status, POST /api/tasks/ingest (manual trigger), TURNSTONE_INGEST_INTERVAL + TURNSTONE_SUBMIT_ENDPOINT env wiring in lifespan - docker-compose.submissions.yml: segregated contrib1 (8536) + contrib2 (8537) receiving instances on Heimdall, isolated DBs under /devl/docker/turnstone-submissions/<node>/ - podman-standalone.sh: pass-through for TURNSTONE_SUBMIT_ENDPOINT + TURNSTONE_SOURCE_HOST - app/ingest/mqtt_subscriber.py: MQTT log source adapter - app/ingest/wazuh.py: Wazuh alert JSON adapter - tests/test_ingest_wazuh.py: Wazuh adapter test suite
74 lines
2.3 KiB
YAML
74 lines
2.3 KiB
YAML
# Turnstone — CF receiving instances for external node submissions.
|
|
#
|
|
# These are SEPARATE instances from the main Turnstone deployment. Each node
|
|
# that has TURNSTONE_SUBMIT_ENDPOINT configured pushes pattern-matched entries
|
|
# here. Each instance has its own isolated database. Avocet reads these
|
|
# databases for training data.
|
|
#
|
|
# Ports:
|
|
# 8536 → submissions-contrib1 (harvest.circuitforge.tech/contrib1/*)
|
|
# 8537 → submissions-contrib2 (harvest.circuitforge.tech/contrib2/*)
|
|
#
|
|
# Deploy on Heimdall:
|
|
# docker compose -f docker-compose.submissions.yml up -d
|
|
#
|
|
# Database locations:
|
|
# /devl/docker/turnstone-submissions/contrib1/turnstone.db
|
|
# /devl/docker/turnstone-submissions/contrib2/turnstone.db
|
|
#
|
|
# These instances have TURNSTONE_INGEST_INTERVAL=0 — they only receive POSTs,
|
|
# they do not run their own scheduled ingest.
|
|
|
|
services:
|
|
submissions-contrib1:
|
|
image: turnstone:latest
|
|
container_name: turnstone-submissions-contrib1
|
|
restart: unless-stopped
|
|
ports:
|
|
- "8536:8534"
|
|
volumes:
|
|
- /devl/docker/turnstone-submissions/contrib1:/data:z
|
|
- /devl/docker/turnstone-submissions/contrib1/patterns:/patterns:ro
|
|
environment:
|
|
TURNSTONE_DB: /data/turnstone.db
|
|
TURNSTONE_PATTERNS: /patterns
|
|
TURNSTONE_SOURCE_HOST: submissions-contrib1
|
|
TURNSTONE_INGEST_INTERVAL: "0"
|
|
PYTHONUNBUFFERED: "1"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8534/turnstone/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 20s
|
|
retries: 3
|
|
networks:
|
|
- caddy-internal
|
|
|
|
submissions-contrib2:
|
|
image: turnstone:latest
|
|
container_name: turnstone-submissions-contrib2
|
|
restart: unless-stopped
|
|
ports:
|
|
- "8537:8534"
|
|
volumes:
|
|
- /devl/docker/turnstone-submissions/contrib2:/data:z
|
|
- /devl/docker/turnstone-submissions/contrib2/patterns:/patterns:ro
|
|
environment:
|
|
TURNSTONE_DB: /data/turnstone.db
|
|
TURNSTONE_PATTERNS: /patterns
|
|
TURNSTONE_SOURCE_HOST: submissions-contrib2
|
|
TURNSTONE_INGEST_INTERVAL: "0"
|
|
PYTHONUNBUFFERED: "1"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8534/turnstone/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 20s
|
|
retries: 3
|
|
networks:
|
|
- caddy-internal
|
|
|
|
networks:
|
|
caddy-internal:
|
|
name: caddy-proxy_caddy-internal
|
|
external: true
|