diff --git a/app/glean/pipeline.py b/app/glean/pipeline.py index a4b749b..ddef194 100644 --- a/app/glean/pipeline.py +++ b/app/glean/pipeline.py @@ -165,7 +165,7 @@ CREATE INDEX IF NOT EXISTS idx_chunks_doc ON context_chunks(document_id); def ensure_schema(db_path: Path) -> None: """Create all tables and apply additive migrations. Safe to call on every startup.""" - conn = sqlite3.connect(str(db_path)) + conn = sqlite3.connect(str(db_path), timeout=30.0) conn.execute("PRAGMA journal_mode=WAL") conn.executescript(_SCHEMA) # Additive column migrations — ALTER TABLE silently skips if column exists @@ -338,7 +338,7 @@ def _glean_files( ingest_time = now_iso() source_id_map = source_id_map or {} - conn = sqlite3.connect(str(db_path)) + conn = sqlite3.connect(str(db_path), timeout=30.0) conn.execute("PRAGMA journal_mode=WAL") conn.executescript(_SCHEMA) conn.commit() @@ -521,7 +521,7 @@ def glean_ssh_source( compiled = _compile(load_patterns(effective_pattern_file)) ingest_time = now_iso() - conn = sqlite3.connect(str(db_path)) + conn = sqlite3.connect(str(db_path), timeout=30.0) conn.execute("PRAGMA journal_mode=WAL") conn.executescript(_SCHEMA) conn.commit() @@ -643,7 +643,7 @@ def glean_sources( compiled = _compile(load_patterns(effective_pattern_file)) ingest_time = now_iso() - conn = sqlite3.connect(str(db_path)) + conn = sqlite3.connect(str(db_path), timeout=30.0) conn.execute("PRAGMA journal_mode=WAL") conn.executescript(_SCHEMA) conn.commit() diff --git a/patterns/default.yaml b/patterns/default.yaml index 227d671..b308895 100644 --- a/patterns/default.yaml +++ b/patterns/default.yaml @@ -1,4 +1,4 @@ -# Turnstone pattern library — named regex patterns for log tagging at glean time. +# Turnstone pattern library — named regex patterns for log tagging at ingest time. # Each matched pattern name is stored on RetrievedEntry.matched_patterns and # used to boost retrieval relevance for diagnostic queries. # @@ -128,21 +128,6 @@ patterns: severity: ERROR description: NFS mount or RPC timeout - - name: service_crash_loop - pattern: "(restart counter is at [0-9]|start request repeated too quickly|Restart limit hit)" - severity: WARN - description: systemd service crash-looping — restart counter incrementing or rate-limit hit; check for DNS resolution failures, missing dependencies, or bad config - - - name: pkg_daemon_restart - pattern: "(invoke-rc\\.d|Unit process.*(apt-get|dpkg|preinst).*remains running after unit stopped|Stopped.*service.*openssh|Restarting.*OpenBSD Secure Shell)" - severity: WARN - description: Package manager restarted a system daemon — active SSH or service sessions may have been interrupted - - - name: ssh_forward_conflict - pattern: "(channel_setup_fwd_listener_tcpip: cannot listen to port|error: bind.*Address already in use)" - severity: WARN - description: SSH port-forward conflict — previous session port still bound; stale sessions accumulating or rapid reconnects - # Add device/service-specific patterns below this line: - name: qbit_tracker_error @@ -194,3 +179,54 @@ patterns: # pattern: "ERR-\d{4}" # severity: ERROR # description: EXT_DEVICE device error code + + # ── VPN / tunnel patterns ────────────────────────────────────────────────── + + - name: vpn_tunnel_fail + pattern: "(wg-quick@|wireguard|spirit-city-tunnel|cf-orch-tunnel|cf-tunnel|openvpn|vpn).*(failed|error|exit.code|timeout|connection reset)" + severity: ERROR + description: VPN or WireGuard tunnel service failed — remote node may be unreachable + + - name: vpn_handshake + pattern: "(handshake|peer.*allowed|WireGuard|wg-quick).*(initiating|complete|timeout|fail|retrying)" + severity: WARN + description: WireGuard peer handshake event — track for timeout/retry patterns + + - name: dns_degraded + pattern: "(degraded feature set|DNS.*fall.?back|resolver.*fail|NXDOMAIN|DNS.*timeout|SERVFAIL)" + severity: WARN + description: DNS resolver degradation or fallback — often precedes connectivity failures + + # ── GPU / NVIDIA driver patterns ─────────────────────────────────────────── + + - name: nvidia_api_mismatch + pattern: "(NVRM: API mismatch|nvidia.*version mismatch|driver.*mismatch|kernel module.*mismatch)" + severity: ERROR + description: NVIDIA kernel module version does not match userspace driver — GPU ops will fail until driver reinstalled + + - name: nvidia_xid + pattern: "(NVRM: Xid|Xid.*(error|critical)|GPU.*Xid)" + severity: CRITICAL + description: NVIDIA Xid error — GPU hardware fault or driver crash (check nvidia-smi error code) + + - name: nvidia_gpu_reset + pattern: "(nvidia.*reset|GPU.*reset|NVRM.*reset|nvml.*error|NVLink.*fail)" + severity: ERROR + description: NVIDIA GPU reset or NVLink fault — possible hardware instability + + # ── Power / thermal patterns ─────────────────────────────────────────────── + + - name: acpi_error + pattern: "(ACPI.*failed|ACPI.*error|ACPI.*_DSM|acpi.*_PPC|ACPI BIOS Error)" + severity: WARN + description: ACPI firmware evaluation failure — often harmless but can indicate BIOS/power management issues + + - name: thermal_throttle + pattern: "(CPU.*throttl|thermal throttl|Package temp|TjMax|temperature.*critical|No RAPL|RAPL.*not available)" + severity: WARN + description: CPU/GPU thermal throttling or thermal management subsystem unavailable + + - name: undervoltage + pattern: "(under.?voltage|brownout|voltage.*(low|critical)|power supply.*insufficient)" + severity: ERROR + description: Undervoltage event — instability risk, check PSU and cable connections diff --git a/scripts/glean_corpus.py b/scripts/glean_corpus.py index e3d14db..8da992c 100644 --- a/scripts/glean_corpus.py +++ b/scripts/glean_corpus.py @@ -2,10 +2,14 @@ Usage: # Single file or directory (legacy) - python scripts/glean_corpus.py [db_path] + python scripts/glean_corpus.py [db_path] [--force] # Sources config (multi-service) - python scripts/glean_corpus.py --sources [--db ] + python scripts/glean_corpus.py --sources [--db ] [--force] + +Options: + --force Bypass fingerprint checks and re-glean all files, re-applying + all patterns. Use after updating patterns/default.yaml. """ from __future__ import annotations @@ -33,15 +37,18 @@ if __name__ == "__main__": if not args: print( "Usage:\n" - " glean_corpus.py [db_path]\n" - " glean_corpus.py --sources [--db ]", + " glean_corpus.py [db_path] [--force]\n" + " glean_corpus.py --sources [--db ] [--force]", file=sys.stderr, ) sys.exit(1) + force = "--force" in args + args = [a for a in args if a != "--force"] + if args[0] == "--sources": if len(args) < 2: - print("Usage: glean_corpus.py --sources [--db ]", file=sys.stderr) + print("Usage: glean_corpus.py --sources [--db ] [--force]", file=sys.stderr) sys.exit(1) sources_file = Path(args[1]) db_path = Path("data/turnstone.db") @@ -49,7 +56,7 @@ if __name__ == "__main__": db_path = Path(args[args.index("--db") + 1]) db_path.parent.mkdir(parents=True, exist_ok=True) print(f"Gleaning sources from {sources_file} → {db_path}") - stats = glean_sources(sources_file, db_path) + stats = glean_sources(sources_file, db_path, force=force) _print_stats(stats) else: target = Path(args[0]) @@ -57,9 +64,9 @@ if __name__ == "__main__": db_path.parent.mkdir(parents=True, exist_ok=True) print(f"Gleaning {target} → {db_path}") if target.is_file(): - stats = glean_file(target, db_path) + stats = glean_file(target, db_path, force=force) elif target.is_dir(): - stats = glean_dir(target, db_path) + stats = glean_dir(target, db_path, force=force) else: print(f"Error: {target} is not a file or directory", file=sys.stderr) sys.exit(1)