feat: add UDP syslog receiver for network device log collection
scripts/syslog_receiver.py: asyncio UDP server listening on port 5140, appends raw syslog lines to network-syslog.txt for the Turnstone live watcher to tail. Requires no root — port 5140 is non-privileged. scripts/turnstone-syslog-receiver.service: systemd unit for auto-start. app/ingest/syslog.py: strip optional RFC 3164 <PRI> prefix before parsing so network-forwarded syslog (OpenWRT logd, Arista EOS, etc.) is handled correctly without the PRI value breaking the regex.
This commit is contained in:
parent
07e151b01f
commit
f8e86254bb
3 changed files with 120 additions and 2 deletions
|
|
@ -26,6 +26,8 @@ _MONTHS = {
|
||||||
|
|
||||||
# May 11 14:23:01 hostname ident[pid]: message
|
# May 11 14:23:01 hostname ident[pid]: message
|
||||||
# May 1 04:00:00 hostname ident: message (no pid, day may be space-padded)
|
# May 1 04:00:00 hostname ident: message (no pid, day may be space-padded)
|
||||||
|
# <134>May 11 14:23:01 ... (optional RFC 3164 PRI prefix from network syslog)
|
||||||
|
_PRI_RE = re.compile(r"^<\d{1,3}>")
|
||||||
_LINE_RE = re.compile(
|
_LINE_RE = re.compile(
|
||||||
r"^(?P<month>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
|
r"^(?P<month>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
|
||||||
r"\s+(?P<day>\d{1,2})\s+(?P<time>\d{2}:\d{2}:\d{2})"
|
r"\s+(?P<day>\d{1,2})\s+(?P<time>\d{2}:\d{2}:\d{2})"
|
||||||
|
|
@ -35,7 +37,8 @@ _LINE_RE = re.compile(
|
||||||
|
|
||||||
|
|
||||||
def is_syslog(first_line: str) -> bool:
|
def is_syslog(first_line: str) -> bool:
|
||||||
return bool(_LINE_RE.match(first_line.strip()))
|
stripped = _PRI_RE.sub("", first_line.strip(), count=1)
|
||||||
|
return bool(_LINE_RE.match(stripped))
|
||||||
|
|
||||||
|
|
||||||
def _parse_ts(month_str: str, day: str, time_str: str) -> tuple[str, str]:
|
def _parse_ts(month_str: str, day: str, time_str: str) -> tuple[str, str]:
|
||||||
|
|
@ -80,7 +83,7 @@ def parse(
|
||||||
)
|
)
|
||||||
|
|
||||||
for raw_line in lines:
|
for raw_line in lines:
|
||||||
line = raw_line.rstrip("\n")
|
line = _PRI_RE.sub("", raw_line.rstrip("\n"), count=1)
|
||||||
m = _LINE_RE.match(line)
|
m = _LINE_RE.match(line)
|
||||||
if m:
|
if m:
|
||||||
if pending_text is not None:
|
if pending_text is not None:
|
||||||
|
|
|
||||||
96
scripts/syslog_receiver.py
Normal file
96
scripts/syslog_receiver.py
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""UDP syslog receiver for Turnstone cluster monitoring.
|
||||||
|
|
||||||
|
Listens on UDP port 5140 (non-privileged) and appends received messages
|
||||||
|
to /devl/turnstone-cluster/data/network-syslog.txt for the Turnstone
|
||||||
|
live watcher to tail.
|
||||||
|
|
||||||
|
Each line written is:
|
||||||
|
<source_ip> <raw_syslog_message>
|
||||||
|
|
||||||
|
This preserves the original syslog content while adding the sender IP so
|
||||||
|
Turnstone's syslog ingestor can tag entries by device.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 syslog_receiver.py [--port 5140] [--output /path/to/network-syslog.txt]
|
||||||
|
|
||||||
|
Installed as: turnstone-syslog-receiver.service (see adjacent .service file)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger("syslog-receiver")
|
||||||
|
|
||||||
|
DEFAULT_PORT = 5140
|
||||||
|
DEFAULT_OUTPUT = "/devl/turnstone-cluster/data/network-syslog.txt"
|
||||||
|
|
||||||
|
|
||||||
|
class SyslogReceiverProtocol(asyncio.DatagramProtocol):
|
||||||
|
def __init__(self, output_path: Path) -> None:
|
||||||
|
self._output_path = output_path
|
||||||
|
self._fh = output_path.open("a", buffering=1) # line-buffered
|
||||||
|
|
||||||
|
def datagram_received(self, data: bytes, addr: tuple[str, int]) -> None:
|
||||||
|
try:
|
||||||
|
message = data.decode("utf-8", errors="replace").rstrip("\r\n")
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
if not message:
|
||||||
|
return
|
||||||
|
# RFC 3164 messages already include the sending hostname — write raw.
|
||||||
|
try:
|
||||||
|
self._fh.write(f"{message}\n")
|
||||||
|
except OSError as exc:
|
||||||
|
logger.error("Write failed: %s", exc)
|
||||||
|
|
||||||
|
def error_received(self, exc: Exception) -> None:
|
||||||
|
logger.warning("Socket error: %s", exc)
|
||||||
|
|
||||||
|
def connection_lost(self, exc: Exception | None) -> None:
|
||||||
|
self._fh.flush()
|
||||||
|
self._fh.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def run(port: int, output_path: Path) -> None:
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
transport, protocol = await loop.create_datagram_endpoint(
|
||||||
|
lambda: SyslogReceiverProtocol(output_path),
|
||||||
|
local_addr=("0.0.0.0", port),
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Listening on UDP :%d → %s", port, output_path)
|
||||||
|
|
||||||
|
stop = loop.create_future()
|
||||||
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||||
|
loop.add_signal_handler(sig, stop.set_result, None)
|
||||||
|
|
||||||
|
await stop
|
||||||
|
transport.close()
|
||||||
|
logger.info("Syslog receiver stopped.")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="UDP syslog receiver for Turnstone")
|
||||||
|
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||||
|
parser.add_argument("--output", default=DEFAULT_OUTPUT)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(message)s",
|
||||||
|
stream=sys.stdout,
|
||||||
|
)
|
||||||
|
|
||||||
|
asyncio.run(run(args.port, Path(args.output)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
19
scripts/turnstone-syslog-receiver.service
Normal file
19
scripts/turnstone-syslog-receiver.service
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Turnstone — UDP syslog receiver (network devices → network-syslog.txt)
|
||||||
|
Documentation=https://git.opensourcesolarpunk.com/Circuit-Forge/turnstone
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/devl/miniconda3/envs/cf/bin/python \
|
||||||
|
/Library/Development/CircuitForge/turnstone/scripts/syslog_receiver.py \
|
||||||
|
--port 5140 \
|
||||||
|
--output /devl/turnstone-cluster/data/network-syslog.txt
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5s
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=turnstone-syslog-receiver
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
Loading…
Reference in a new issue