pyr0ball
9ec60ea7ff
- Add syslog.py — RFC 3164 parser for /var/log/syslog, /var/log/messages, auth.log, kern.log; ident prepended to message text for searchability - Add dmesg_log.py — handles both relative [secs.usecs] and human-readable [Dow Mon DD HH:MM:SS YYYY] formats; relative timestamps preserved as raw - Wire both into pipeline.py auto-detection (before plaintext fallback) - Update export_journal.sh: checks for journalctl availability, falls back gracefully on non-systemd systems; adds dmesg -T export (falls back to plain dmesg on older kernels) - Add syslog entries (commented) + dmesg source to sources.yaml - 30 tests covering both parsers (detection + parse correctness)
102 lines
3.1 KiB
Python
102 lines
3.1 KiB
Python
"""dmesg log parser.
|
|
|
|
Handles two formats:
|
|
|
|
Relative (always available):
|
|
[ 0.000000] Linux version 6.8.0-65-generic
|
|
[12345.678901] usb 1-1: USB disconnect, device number 2
|
|
|
|
Human-readable (dmesg -T, util-linux >= 2.21):
|
|
[Mon May 11 14:23:01 2026] usb 1-1: USB disconnect, device number 2
|
|
|
|
The export_journal.sh script exports with -T when available, falling back
|
|
to plain dmesg. Relative-timestamp entries get no timestamp_iso.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from typing import Iterator
|
|
|
|
from app.ingest.base import (
|
|
SourceState, apply_patterns, detect_severity, make_entry_id, now_iso,
|
|
)
|
|
from app.services.models import LogPattern, RetrievedEntry
|
|
|
|
_DAYS = {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"}
|
|
_MONTHS_ABBR = {
|
|
"Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6,
|
|
"Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12,
|
|
}
|
|
|
|
# [ 0.000000] or [12345.678901]
|
|
_RELATIVE_RE = re.compile(r"^\[\s*(?P<secs>\d+\.\d+)\]\s+(?P<msg>.+)$")
|
|
# [Mon May 11 14:23:01 2026]
|
|
_HUMAN_RE = re.compile(
|
|
r"^\[(?P<dow>\w{3})\s+(?P<month>\w{3})\s+(?P<day>\d{1,2})"
|
|
r"\s+(?P<time>\d{2}:\d{2}:\d{2})\s+(?P<year>\d{4})\]\s+(?P<msg>.+)$"
|
|
)
|
|
|
|
|
|
def is_dmesg_log(first_line: str) -> bool:
|
|
line = first_line.strip()
|
|
return bool(_RELATIVE_RE.match(line) or _HUMAN_RE.match(line))
|
|
|
|
|
|
def _parse_human_ts(m: re.Match) -> tuple[str, str]:
|
|
ts_raw = f"{m.group('dow')} {m.group('month')} {m.group('day')} {m.group('time')} {m.group('year')}"
|
|
month = _MONTHS_ABBR.get(m.group("month"), 1)
|
|
try:
|
|
dt = datetime(
|
|
int(m.group("year")), month, int(m.group("day")),
|
|
*[int(p) for p in m.group("time").split(":")],
|
|
tzinfo=timezone.utc,
|
|
)
|
|
return ts_raw, dt.isoformat()
|
|
except ValueError:
|
|
return ts_raw, ""
|
|
|
|
|
|
def parse(
|
|
lines: Iterator[str],
|
|
source_id: str,
|
|
compiled_patterns: list[tuple[LogPattern, object]],
|
|
ingest_time: str | None = None,
|
|
) -> Iterator[RetrievedEntry]:
|
|
ingest_time = ingest_time or now_iso()
|
|
state = SourceState()
|
|
|
|
for raw_line in lines:
|
|
line = raw_line.rstrip("\n").strip()
|
|
if not line:
|
|
continue
|
|
|
|
m = _HUMAN_RE.match(line)
|
|
if m:
|
|
ts_raw, ts_iso = _parse_human_ts(m)
|
|
msg = m.group("msg")
|
|
else:
|
|
m = _RELATIVE_RE.match(line)
|
|
if not m:
|
|
continue
|
|
ts_raw = f"[{m.group('secs')}]"
|
|
ts_iso = ""
|
|
msg = m.group("msg")
|
|
|
|
severity = detect_severity(msg)
|
|
repeat, out_of_order = state.observe(msg, ts_iso or None)
|
|
matched = apply_patterns(msg, compiled_patterns)
|
|
|
|
yield RetrievedEntry(
|
|
entry_id=make_entry_id(source_id, state.sequence, msg),
|
|
source_id=source_id,
|
|
sequence=state.sequence,
|
|
timestamp_raw=ts_raw,
|
|
timestamp_iso=ts_iso,
|
|
ingest_time=ingest_time,
|
|
severity=severity,
|
|
repeat_count=repeat,
|
|
out_of_order=out_of_order,
|
|
matched_patterns=matched,
|
|
text=msg,
|
|
)
|