feat: add UDP syslog receiver for network device log collection

scripts/syslog_receiver.py: asyncio UDP server listening on port 5140,
appends raw syslog lines to network-syslog.txt for the Turnstone live
watcher to tail. Requires no root — port 5140 is non-privileged.

scripts/turnstone-syslog-receiver.service: systemd unit for auto-start.

app/ingest/syslog.py: strip optional RFC 3164 <PRI> prefix before
parsing so network-forwarded syslog (OpenWRT logd, Arista EOS, etc.)
is handled correctly without the PRI value breaking the regex.
This commit is contained in:
pyr0ball 2026-05-13 04:58:51 -07:00
parent d769be04d4
commit a4ec5a6951
3 changed files with 120 additions and 2 deletions

View file

@ -26,6 +26,8 @@ _MONTHS = {
# May 11 14:23:01 hostname ident[pid]: message # May 11 14:23:01 hostname ident[pid]: message
# May 1 04:00:00 hostname ident: message (no pid, day may be space-padded) # May 1 04:00:00 hostname ident: message (no pid, day may be space-padded)
# <134>May 11 14:23:01 ... (optional RFC 3164 PRI prefix from network syslog)
_PRI_RE = re.compile(r"^<\d{1,3}>")
_LINE_RE = re.compile( _LINE_RE = re.compile(
r"^(?P<month>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)" r"^(?P<month>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
r"\s+(?P<day>\d{1,2})\s+(?P<time>\d{2}:\d{2}:\d{2})" r"\s+(?P<day>\d{1,2})\s+(?P<time>\d{2}:\d{2}:\d{2})"
@ -35,7 +37,8 @@ _LINE_RE = re.compile(
def is_syslog(first_line: str) -> bool: def is_syslog(first_line: str) -> bool:
return bool(_LINE_RE.match(first_line.strip())) stripped = _PRI_RE.sub("", first_line.strip(), count=1)
return bool(_LINE_RE.match(stripped))
def _parse_ts(month_str: str, day: str, time_str: str) -> tuple[str, str]: def _parse_ts(month_str: str, day: str, time_str: str) -> tuple[str, str]:
@ -80,7 +83,7 @@ def parse(
) )
for raw_line in lines: for raw_line in lines:
line = raw_line.rstrip("\n") line = _PRI_RE.sub("", raw_line.rstrip("\n"), count=1)
m = _LINE_RE.match(line) m = _LINE_RE.match(line)
if m: if m:
if pending_text is not None: if pending_text is not None:

View file

@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""UDP syslog receiver for Turnstone cluster monitoring.
Listens on UDP port 5140 (non-privileged) and appends received messages
to /devl/turnstone-cluster/data/network-syslog.txt for the Turnstone
live watcher to tail.
Each line written is:
<source_ip> <raw_syslog_message>
This preserves the original syslog content while adding the sender IP so
Turnstone's syslog ingestor can tag entries by device.
Usage:
python3 syslog_receiver.py [--port 5140] [--output /path/to/network-syslog.txt]
Installed as: turnstone-syslog-receiver.service (see adjacent .service file)
"""
from __future__ import annotations
import argparse
import asyncio
import logging
import signal
import sys
from pathlib import Path
logger = logging.getLogger("syslog-receiver")
DEFAULT_PORT = 5140
DEFAULT_OUTPUT = "/devl/turnstone-cluster/data/network-syslog.txt"
class SyslogReceiverProtocol(asyncio.DatagramProtocol):
def __init__(self, output_path: Path) -> None:
self._output_path = output_path
self._fh = output_path.open("a", buffering=1) # line-buffered
def datagram_received(self, data: bytes, addr: tuple[str, int]) -> None:
try:
message = data.decode("utf-8", errors="replace").rstrip("\r\n")
except Exception:
return
if not message:
return
# RFC 3164 messages already include the sending hostname — write raw.
try:
self._fh.write(f"{message}\n")
except OSError as exc:
logger.error("Write failed: %s", exc)
def error_received(self, exc: Exception) -> None:
logger.warning("Socket error: %s", exc)
def connection_lost(self, exc: Exception | None) -> None:
self._fh.flush()
self._fh.close()
async def run(port: int, output_path: Path) -> None:
output_path.parent.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_running_loop()
transport, protocol = await loop.create_datagram_endpoint(
lambda: SyslogReceiverProtocol(output_path),
local_addr=("0.0.0.0", port),
)
logger.info("Listening on UDP :%d%s", port, output_path)
stop = loop.create_future()
for sig in (signal.SIGINT, signal.SIGTERM):
loop.add_signal_handler(sig, stop.set_result, None)
await stop
transport.close()
logger.info("Syslog receiver stopped.")
def main() -> None:
parser = argparse.ArgumentParser(description="UDP syslog receiver for Turnstone")
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
parser.add_argument("--output", default=DEFAULT_OUTPUT)
args = parser.parse_args()
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
stream=sys.stdout,
)
asyncio.run(run(args.port, Path(args.output)))
if __name__ == "__main__":
main()

View file

@ -0,0 +1,19 @@
[Unit]
Description=Turnstone — UDP syslog receiver (network devices → network-syslog.txt)
Documentation=https://git.opensourcesolarpunk.com/Circuit-Forge/turnstone
After=network.target
[Service]
Type=simple
ExecStart=/devl/miniconda3/envs/cf/bin/python \
/Library/Development/CircuitForge/turnstone/scripts/syslog_receiver.py \
--port 5140 \
--output /devl/turnstone-cluster/data/network-syslog.txt
Restart=on-failure
RestartSec=5s
StandardOutput=journal
StandardError=journal
SyslogIdentifier=turnstone-syslog-receiver
[Install]
WantedBy=multi-user.target