pyr0ball
828b69768a
Renames the app/ingest/ package to app/glean/ and updates all references across Python modules, shell scripts, Vue components, tests, and documentation. Intentionally preserved: - SQLite column name ingest_time (avoids schema migration) - RetrievedEntry.ingest_time field (maps to the column above) - Any public-facing JSON keys that reference ingest_time Changes by category: - app/ingest/ → app/glean/ (full package move, all parsers) - app/tasks/ingest_scheduler.py → app/tasks/glean_scheduler.py - scripts/ingest_corpus.py → scripts/glean_corpus.py - tests/test_ingest_*.py → tests/test_glean_*.py - Docstrings, log messages, comments: ingest → glean - Env var: TURNSTONE_INGEST_INTERVAL → TURNSTONE_GLEAN_INTERVAL - Shell scripts: glean.log, glean_corpus.py references - README.md: multi-source ingest → multi-source glean - .env.example: updated env var name - patterns/: new diagnostic patterns from 2026-05-20 SSH incident (service_crash_loop, pkg_daemon_restart, ssh_forward_conflict) - SourcesView.vue: pipeline label updated - All test import paths updated to app.glean.* 285 tests passing.
66 lines
2.1 KiB
Python
66 lines
2.1 KiB
Python
"""CLI: glean a log file or corpus directory into the Turnstone SQLite database.
|
|
|
|
Usage:
|
|
# Single file or directory (legacy)
|
|
python scripts/glean_corpus.py <file_or_dir> [db_path]
|
|
|
|
# Sources config (multi-service)
|
|
python scripts/glean_corpus.py --sources <sources.yaml> [--db <db_path>]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from app.glean.pipeline import glean_dir, glean_file, glean_sources
|
|
|
|
|
|
def _print_stats(stats: dict[str, int]) -> None:
|
|
total = sum(stats.values())
|
|
for source, count in sorted(stats.items()):
|
|
print(f" {source}: {count:,}")
|
|
print(f" TOTAL: {total:,} entries")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = sys.argv[1:]
|
|
|
|
if not args:
|
|
print(
|
|
"Usage:\n"
|
|
" glean_corpus.py <file_or_dir> [db_path]\n"
|
|
" glean_corpus.py --sources <sources.yaml> [--db <db_path>]",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
if args[0] == "--sources":
|
|
if len(args) < 2:
|
|
print("Usage: glean_corpus.py --sources <sources.yaml> [--db <db_path>]", file=sys.stderr)
|
|
sys.exit(1)
|
|
sources_file = Path(args[1])
|
|
db_path = Path("data/turnstone.db")
|
|
if "--db" in args:
|
|
db_path = Path(args[args.index("--db") + 1])
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
print(f"Gleaning sources from {sources_file} → {db_path}")
|
|
stats = glean_sources(sources_file, db_path)
|
|
_print_stats(stats)
|
|
else:
|
|
target = Path(args[0])
|
|
db_path = Path(args[1]) if len(args) > 1 else Path("data/turnstone.db")
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
print(f"Gleaning {target} → {db_path}")
|
|
if target.is_file():
|
|
stats = glean_file(target, db_path)
|
|
elif target.is_dir():
|
|
stats = glean_dir(target, db_path)
|
|
else:
|
|
print(f"Error: {target} is not a file or directory", file=sys.stderr)
|
|
sys.exit(1)
|
|
_print_stats(stats)
|