feat: add UDP syslog receiver for network device log collection
scripts/syslog_receiver.py: asyncio UDP server listening on port 5140, appends raw syslog lines to network-syslog.txt for the Turnstone live watcher to tail. Requires no root — port 5140 is non-privileged. scripts/turnstone-syslog-receiver.service: systemd unit for auto-start. app/ingest/syslog.py: strip optional RFC 3164 <PRI> prefix before parsing so network-forwarded syslog (OpenWRT logd, Arista EOS, etc.) is handled correctly without the PRI value breaking the regex.
This commit is contained in:
parent
72679787d3
commit
d80d4875db
3 changed files with 120 additions and 2 deletions
|
|
@ -26,6 +26,8 @@ _MONTHS = {
|
|||
|
||||
# May 11 14:23:01 hostname ident[pid]: message
|
||||
# May 1 04:00:00 hostname ident: message (no pid, day may be space-padded)
|
||||
# <134>May 11 14:23:01 ... (optional RFC 3164 PRI prefix from network syslog)
|
||||
_PRI_RE = re.compile(r"^<\d{1,3}>")
|
||||
_LINE_RE = re.compile(
|
||||
r"^(?P<month>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
|
||||
r"\s+(?P<day>\d{1,2})\s+(?P<time>\d{2}:\d{2}:\d{2})"
|
||||
|
|
@ -35,7 +37,8 @@ _LINE_RE = re.compile(
|
|||
|
||||
|
||||
def is_syslog(first_line: str) -> bool:
|
||||
return bool(_LINE_RE.match(first_line.strip()))
|
||||
stripped = _PRI_RE.sub("", first_line.strip(), count=1)
|
||||
return bool(_LINE_RE.match(stripped))
|
||||
|
||||
|
||||
def _parse_ts(month_str: str, day: str, time_str: str) -> tuple[str, str]:
|
||||
|
|
@ -80,7 +83,7 @@ def parse(
|
|||
)
|
||||
|
||||
for raw_line in lines:
|
||||
line = raw_line.rstrip("\n")
|
||||
line = _PRI_RE.sub("", raw_line.rstrip("\n"), count=1)
|
||||
m = _LINE_RE.match(line)
|
||||
if m:
|
||||
if pending_text is not None:
|
||||
|
|
|
|||
96
scripts/syslog_receiver.py
Normal file
96
scripts/syslog_receiver.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env python3
|
||||
"""UDP syslog receiver for Turnstone cluster monitoring.
|
||||
|
||||
Listens on UDP port 5140 (non-privileged) and appends received messages
|
||||
to /devl/turnstone-cluster/data/network-syslog.txt for the Turnstone
|
||||
live watcher to tail.
|
||||
|
||||
Each line written is:
|
||||
<source_ip> <raw_syslog_message>
|
||||
|
||||
This preserves the original syslog content while adding the sender IP so
|
||||
Turnstone's syslog ingestor can tag entries by device.
|
||||
|
||||
Usage:
|
||||
python3 syslog_receiver.py [--port 5140] [--output /path/to/network-syslog.txt]
|
||||
|
||||
Installed as: turnstone-syslog-receiver.service (see adjacent .service file)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("syslog-receiver")
|
||||
|
||||
DEFAULT_PORT = 5140
|
||||
DEFAULT_OUTPUT = "/devl/turnstone-cluster/data/network-syslog.txt"
|
||||
|
||||
|
||||
class SyslogReceiverProtocol(asyncio.DatagramProtocol):
|
||||
def __init__(self, output_path: Path) -> None:
|
||||
self._output_path = output_path
|
||||
self._fh = output_path.open("a", buffering=1) # line-buffered
|
||||
|
||||
def datagram_received(self, data: bytes, addr: tuple[str, int]) -> None:
|
||||
try:
|
||||
message = data.decode("utf-8", errors="replace").rstrip("\r\n")
|
||||
except Exception:
|
||||
return
|
||||
if not message:
|
||||
return
|
||||
# RFC 3164 messages already include the sending hostname — write raw.
|
||||
try:
|
||||
self._fh.write(f"{message}\n")
|
||||
except OSError as exc:
|
||||
logger.error("Write failed: %s", exc)
|
||||
|
||||
def error_received(self, exc: Exception) -> None:
|
||||
logger.warning("Socket error: %s", exc)
|
||||
|
||||
def connection_lost(self, exc: Exception | None) -> None:
|
||||
self._fh.flush()
|
||||
self._fh.close()
|
||||
|
||||
|
||||
async def run(port: int, output_path: Path) -> None:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
transport, protocol = await loop.create_datagram_endpoint(
|
||||
lambda: SyslogReceiverProtocol(output_path),
|
||||
local_addr=("0.0.0.0", port),
|
||||
)
|
||||
|
||||
logger.info("Listening on UDP :%d → %s", port, output_path)
|
||||
|
||||
stop = loop.create_future()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.add_signal_handler(sig, stop.set_result, None)
|
||||
|
||||
await stop
|
||||
transport.close()
|
||||
logger.info("Syslog receiver stopped.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="UDP syslog receiver for Turnstone")
|
||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
parser.add_argument("--output", default=DEFAULT_OUTPUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
|
||||
asyncio.run(run(args.port, Path(args.output)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
19
scripts/turnstone-syslog-receiver.service
Normal file
19
scripts/turnstone-syslog-receiver.service
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
[Unit]
|
||||
Description=Turnstone — UDP syslog receiver (network devices → network-syslog.txt)
|
||||
Documentation=https://git.opensourcesolarpunk.com/Circuit-Forge/turnstone
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/devl/miniconda3/envs/cf/bin/python \
|
||||
/Library/Development/CircuitForge/turnstone/scripts/syslog_receiver.py \
|
||||
--port 5140 \
|
||||
--output /devl/turnstone-cluster/data/network-syslog.txt
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=turnstone-syslog-receiver
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Reference in a new issue