feat(#7,#10): signal crawler -- Reddit + Lemmy community monitoring

Implements the full signal detection pipeline:

Backend:
- app/services/lemmy/client.py: async Lemmy API v3 client, community@instance
  addressing, integer cursor dedup, normalised post dicts
- app/services/scraper.py: platform-agnostic scraper; Reddit (.json API,
  fullname cursor) + Lemmy (integer ID cursor); keyword/regex/all match modes,
  min_score gate, NormalizedPost shape, upsert dedup via UNIQUE post_id
- app/api/endpoints/signals.py: CRUD for signal_rules + signals queue;
  POST /signals/scrape manual trigger; scrape-state viewer
- migrations 010-012: signal_rules, signals, signal_scrape_state tables
- scheduler: interval job every 30 min (scraper_enabled=True in config)
- Fixed migration collision: 007_signal_rules.sql → 010, 008 → 011, 009 → 012

Frontend:
- SignalsView.vue: signal feed with status filter (new/saved/dismissed),
  keyword chips, score/comment counts, save/dismiss actions, rules editor panel
- api.ts: SignalRule, Signal types + signalRules/signals API methods
- Nav: Signals as default landing route (replaces /campaigns default)

Closes #7 (signal extraction), closes #10 (Lemmy JSON crawler)
This commit is contained in:
Alan Weinstock 2026-04-22 11:00:14 -07:00
parent fb036ae064
commit 80718e206c
25 changed files with 2228 additions and 161 deletions

1
.gitignore vendored
View file

@ -29,3 +29,4 @@ data/
# CLAUDE.md — gitignored per BSL 1.1 + docs-location policy # CLAUDE.md — gitignored per BSL 1.1 + docs-location policy
CLAUDE.md CLAUDE.md
frontend/.vite

View file

@ -0,0 +1,159 @@
from __future__ import annotations
import json
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from app.core.config import get_settings
from app.db.store import Store
router = APIRouter()
def get_store() -> Store:
s = get_settings()
return Store(s.db_path)
# ------------------------------------------------------------------ #
# Schemas
# ------------------------------------------------------------------ #
class SignalRuleCreate(BaseModel):
name: str
platform: str = "reddit"
sub: str | None = None
keywords: list[str] = Field(default_factory=list)
match_mode: str = "any" # any | all | regex
min_score: int = 0
label: str | None = None # pain-point | feedback | mention | trust
notes: str | None = None
class SignalRuleUpdate(BaseModel):
name: str | None = None
sub: str | None = None
keywords: list[str] | None = None
match_mode: str | None = None
min_score: int | None = None
label: str | None = None
active: bool | None = None
notes: str | None = None
class SignalStatusUpdate(BaseModel):
status: str # new | saved | dismissed
notes: str | None = None
def _decode_json_fields(row: dict[str, Any]) -> dict[str, Any]:
"""Decode JSON-encoded list fields returned as strings from SQLite."""
for col in ("keywords", "matched_keywords"):
if col in row and isinstance(row[col], str):
try:
row[col] = json.loads(row[col])
except (json.JSONDecodeError, TypeError):
row[col] = []
return row
# ------------------------------------------------------------------ #
# Signal rules
# ------------------------------------------------------------------ #
@router.get("/signal-rules")
def list_signal_rules(active_only: bool = False, store: Store = Depends(get_store)):
return [_decode_json_fields(r) for r in store.list_signal_rules(active_only=active_only)]
@router.post("/signal-rules", status_code=201)
def create_signal_rule(body: SignalRuleCreate, store: Store = Depends(get_store)):
rule = store.create_signal_rule(
name=body.name,
platform=body.platform,
sub=body.sub,
keywords=body.keywords,
match_mode=body.match_mode,
min_score=body.min_score,
label=body.label,
notes=body.notes,
)
return _decode_json_fields(rule)
@router.get("/signal-rules/{rule_id}")
def get_signal_rule(rule_id: int, store: Store = Depends(get_store)):
rule = store.get_signal_rule(rule_id)
if not rule:
raise HTTPException(status_code=404, detail="Signal rule not found")
return _decode_json_fields(rule)
@router.patch("/signal-rules/{rule_id}")
def update_signal_rule(rule_id: int, body: SignalRuleUpdate, store: Store = Depends(get_store)):
rule = store.get_signal_rule(rule_id)
if not rule:
raise HTTPException(status_code=404, detail="Signal rule not found")
updates = body.model_dump(exclude_none=True)
updated = store.update_signal_rule(rule_id, **updates)
return _decode_json_fields(updated)
@router.delete("/signal-rules/{rule_id}", status_code=204)
def delete_signal_rule(rule_id: int, store: Store = Depends(get_store)):
if not store.delete_signal_rule(rule_id):
raise HTTPException(status_code=404, detail="Signal rule not found")
# ------------------------------------------------------------------ #
# Signals queue — fixed-path routes MUST precede /{signal_id} routes
# (FastAPI matches in registration order; literal paths win over params)
# ------------------------------------------------------------------ #
@router.get("/signals")
def list_signals(
status: str | None = None,
platform: str | None = None,
sub: str | None = None,
limit: int = 100,
store: Store = Depends(get_store),
):
rows = store.list_signals(status=status, platform=platform, sub=sub, limit=limit)
return [_decode_json_fields(r) for r in rows]
@router.post("/signals/scrape")
async def trigger_scrape():
"""Manually trigger a full signal scrape pass. Useful for testing rules."""
from app.services.scraper import scrape_signals
summary = await scrape_signals()
return summary
@router.get("/signals/scrape-state")
def get_scrape_state(store: Store = Depends(get_store)):
"""Return per-sub cursor state (last scraped, posts seen, signals found)."""
return store.list_scrape_states()
@router.get("/signals/{signal_id}")
def get_signal(signal_id: int, store: Store = Depends(get_store)):
signal = store.get_signal(signal_id)
if not signal:
raise HTTPException(status_code=404, detail="Signal not found")
result = _decode_json_fields(signal)
result["matched_rules"] = store.get_signal_rule_matches(signal_id)
return result
@router.patch("/signals/{signal_id}/status")
def update_signal_status(signal_id: int, body: SignalStatusUpdate, store: Store = Depends(get_store)):
allowed = {"new", "saved", "dismissed"}
if body.status not in allowed:
raise HTTPException(status_code=422, detail=f"status must be one of {allowed}")
signal = store.update_signal_status(signal_id, body.status, body.notes)
if not signal:
raise HTTPException(status_code=404, detail="Signal not found")
return _decode_json_fields(signal)

View file

@ -1,6 +1,6 @@
from fastapi import FastAPI from fastapi import FastAPI
from app.api.endpoints import campaigns, opportunities, posts, scheduler, subs from app.api.endpoints import campaigns, opportunities, posts, scheduler, signals, subs
def register_routes(app: FastAPI) -> None: def register_routes(app: FastAPI) -> None:
@ -9,3 +9,4 @@ def register_routes(app: FastAPI) -> None:
app.include_router(subs.router, prefix="/api/v1") app.include_router(subs.router, prefix="/api/v1")
app.include_router(scheduler.router, prefix="/api/v1") app.include_router(scheduler.router, prefix="/api/v1")
app.include_router(opportunities.router, prefix="/api/v1") app.include_router(opportunities.router, prefix="/api/v1")
app.include_router(signals.router, prefix="/api/v1")

View file

@ -21,6 +21,13 @@ class Settings(BaseSettings):
# Scheduler # Scheduler
scheduler_enabled: bool = True scheduler_enabled: bool = True
# Signal scraper
scraper_enabled: bool = True
scraper_interval_mins: int = 30 # how often to poll (per full pass of all subs)
scraper_request_delay_secs: float = 2.0 # pause between sub requests to respect rate limits
scraper_fetch_limit: int = 25 # posts to fetch per sub per run (max 100)
scraper_user_agent: str = "Magpie/0.1 signal-monitor (by CircuitForge)"
def get_settings() -> Settings: def get_settings() -> Settings:
return Settings() return Settings()

View file

@ -0,0 +1,17 @@
-- Signal monitoring rules: what to watch for across communities
CREATE TABLE IF NOT EXISTS signal_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL, -- human label ("CF pain points", "Kiwi mentions")
platform TEXT NOT NULL DEFAULT 'reddit',
sub TEXT, -- NULL = apply to all monitored subs
keywords TEXT NOT NULL DEFAULT '[]', -- JSON array of strings
match_mode TEXT NOT NULL DEFAULT 'any', -- 'any' | 'all' | 'regex'
min_score INTEGER NOT NULL DEFAULT 0, -- ignore posts below this karma score
label TEXT, -- signal category: pain-point|feedback|mention|trust
active INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
notes TEXT
);
CREATE INDEX IF NOT EXISTS idx_signal_rules_platform_sub ON signal_rules(platform, sub);
CREATE INDEX IF NOT EXISTS idx_signal_rules_active ON signal_rules(active);

View file

@ -0,0 +1,32 @@
-- Surfaced content instances from signal monitoring
CREATE TABLE IF NOT EXISTS signals (
id INTEGER PRIMARY KEY AUTOINCREMENT,
platform TEXT NOT NULL DEFAULT 'reddit',
sub TEXT NOT NULL,
post_id TEXT NOT NULL, -- platform-native ID (reddit: t3_xxxxx)
title TEXT NOT NULL,
body_snippet TEXT, -- first ~500 chars
score INTEGER,
comment_count INTEGER,
author TEXT,
url TEXT,
posted_at TEXT, -- original post timestamp
surfaced_at TEXT NOT NULL DEFAULT (datetime('now')),
matched_keywords TEXT NOT NULL DEFAULT '[]', -- JSON array of matched terms
status TEXT NOT NULL DEFAULT 'new',-- new|saved|dismissed
notes TEXT,
UNIQUE(platform, post_id) -- deduplicate across rule runs
);
-- Junction table: which rules matched each signal (many-to-many)
CREATE TABLE IF NOT EXISTS signal_rule_matches (
signal_id INTEGER NOT NULL REFERENCES signals(id) ON DELETE CASCADE,
rule_id INTEGER NOT NULL REFERENCES signal_rules(id) ON DELETE CASCADE,
matched_at TEXT NOT NULL DEFAULT (datetime('now')),
PRIMARY KEY (signal_id, rule_id)
);
CREATE INDEX IF NOT EXISTS idx_signals_status ON signals(status);
CREATE INDEX IF NOT EXISTS idx_signals_platform_sub ON signals(platform, sub);
CREATE INDEX IF NOT EXISTS idx_signals_surfaced_at ON signals(surfaced_at DESC);
CREATE INDEX IF NOT EXISTS idx_signal_rule_matches_rule ON signal_rule_matches(rule_id);

View file

@ -0,0 +1,11 @@
-- Per-sub cursor state for the signal scraper.
-- Stores the newest post fullname seen so subsequent runs only fetch newer posts.
CREATE TABLE IF NOT EXISTS signal_scrape_state (
platform TEXT NOT NULL DEFAULT 'reddit',
sub TEXT NOT NULL,
last_fullname TEXT, -- e.g. t3_abc123; NULL = never scraped
last_scraped_at TEXT,
posts_seen INTEGER NOT NULL DEFAULT 0,
signals_found INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (platform, sub)
);

View file

@ -353,3 +353,175 @@ class Store:
def dismiss_opportunity(self, opportunity_id: int, note: str | None = None) -> dict | None: def dismiss_opportunity(self, opportunity_id: int, note: str | None = None) -> dict | None:
return self.update_opportunity(opportunity_id, status="dismissed", dismiss_note=note) return self.update_opportunity(opportunity_id, status="dismissed", dismiss_note=note)
# ------------------------------------------------------------------ #
# Signal rules
# ------------------------------------------------------------------ #
def list_signal_rules(self, active_only: bool = False) -> list[dict]:
where = "WHERE active = 1" if active_only else ""
return self._fetchall(f"SELECT * FROM signal_rules {where} ORDER BY platform, name")
def get_signal_rule(self, rule_id: int) -> dict | None:
return self._fetchone("SELECT * FROM signal_rules WHERE id = ?", (rule_id,))
def create_signal_rule(
self,
name: str,
platform: str = "reddit",
sub: str | None = None,
keywords: list[str] | None = None,
match_mode: str = "any",
min_score: int = 0,
label: str | None = None,
notes: str | None = None,
) -> dict:
return self._insert_returning(
"INSERT INTO signal_rules (name, platform, sub, keywords, match_mode, min_score, label, notes)"
" VALUES (?,?,?,?,?,?,?,?) RETURNING *",
(name, platform, sub, json.dumps(keywords or []), match_mode, min_score, label, notes),
)
def update_signal_rule(self, rule_id: int, **fields) -> dict | None:
allowed = {"name", "sub", "keywords", "match_mode", "min_score", "label", "active", "notes"}
updates = {}
for k, v in fields.items():
if k not in allowed:
continue
updates[k] = json.dumps(v) if k == "keywords" else v
if not updates:
return self.get_signal_rule(rule_id)
set_clause = ", ".join(f"{k} = ?" for k in updates)
params = tuple(updates.values()) + (rule_id,)
self.conn.execute(f"UPDATE signal_rules SET {set_clause} WHERE id = ?", params)
self.conn.commit()
return self.get_signal_rule(rule_id)
def delete_signal_rule(self, rule_id: int) -> bool:
cur = self.conn.execute("DELETE FROM signal_rules WHERE id = ?", (rule_id,))
self.conn.commit()
return cur.rowcount > 0
# ------------------------------------------------------------------ #
# Signals
# ------------------------------------------------------------------ #
def list_signals(
self,
status: str | None = None,
platform: str | None = None,
sub: str | None = None,
limit: int = 100,
) -> list[dict]:
clauses, params = [], []
if status:
clauses.append("status = ?")
params.append(status)
if platform:
clauses.append("platform = ?")
params.append(platform)
if sub:
clauses.append("sub = ?")
params.append(sub)
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
params.append(limit)
return self._fetchall(
f"SELECT * FROM signals {where} ORDER BY surfaced_at DESC LIMIT ?", tuple(params)
)
def get_signal(self, signal_id: int) -> dict | None:
return self._fetchone("SELECT * FROM signals WHERE id = ?", (signal_id,))
def upsert_signal(
self,
platform: str,
sub: str,
post_id: str,
title: str,
body_snippet: str | None = None,
score: int | None = None,
comment_count: int | None = None,
author: str | None = None,
url: str | None = None,
posted_at: str | None = None,
matched_keywords: list[str] | None = None,
) -> dict:
"""Insert or ignore (dedup on platform+post_id). Returns the row either way."""
self.conn.execute(
"INSERT OR IGNORE INTO signals"
" (platform, sub, post_id, title, body_snippet, score, comment_count,"
" author, url, posted_at, matched_keywords)"
" VALUES (?,?,?,?,?,?,?,?,?,?,?)",
(platform, sub, post_id, title, body_snippet, score, comment_count,
author, url, posted_at, json.dumps(matched_keywords or [])),
)
self.conn.commit()
return self._fetchone(
"SELECT * FROM signals WHERE platform = ? AND post_id = ?", (platform, post_id)
)
def record_signal_rule_match(self, signal_id: int, rule_id: int) -> None:
"""Record that a rule matched a signal (many-to-many, ignore duplicates)."""
self.conn.execute(
"INSERT OR IGNORE INTO signal_rule_matches (signal_id, rule_id) VALUES (?,?)",
(signal_id, rule_id),
)
self.conn.commit()
def update_signal_status(
self, signal_id: int, status: str, notes: str | None = None
) -> dict | None:
self.conn.execute(
"UPDATE signals SET status = ?, notes = COALESCE(?, notes) WHERE id = ?",
(status, notes, signal_id),
)
self.conn.commit()
return self.get_signal(signal_id)
def get_signal_rule_matches(self, signal_id: int) -> list[dict]:
"""Return all rules that matched a given signal."""
return self._fetchall(
"SELECT sr.* FROM signal_rules sr"
" JOIN signal_rule_matches srm ON sr.id = srm.rule_id"
" WHERE srm.signal_id = ?",
(signal_id,),
)
# ------------------------------------------------------------------ #
# Signal scrape state (per-sub cursor tracking)
# ------------------------------------------------------------------ #
def get_scrape_state(self, sub: str, platform: str = "reddit") -> dict | None:
return self._fetchone(
"SELECT * FROM signal_scrape_state WHERE platform = ? AND sub = ?",
(platform, sub),
)
def update_scrape_state(
self,
sub: str,
platform: str = "reddit",
last_fullname: str | None = None,
posts_seen_delta: int = 0,
signals_found_delta: int = 0,
) -> None:
self.conn.execute(
"""
INSERT INTO signal_scrape_state
(platform, sub, last_fullname, last_scraped_at, posts_seen, signals_found)
VALUES (?, ?, ?, datetime('now'), ?, ?)
ON CONFLICT(platform, sub) DO UPDATE SET
last_fullname = excluded.last_fullname,
last_scraped_at = excluded.last_scraped_at,
posts_seen = signal_scrape_state.posts_seen + excluded.posts_seen,
signals_found = signal_scrape_state.signals_found + excluded.signals_found
""",
(platform, sub, last_fullname, posts_seen_delta, signals_found_delta),
)
self.conn.commit()
def list_scrape_states(self, platform: str = "reddit") -> list[dict]:
return self._fetchall(
"SELECT * FROM signal_scrape_state WHERE platform = ? ORDER BY sub",
(platform,),
)

View file

@ -9,7 +9,10 @@ from fastapi.middleware.cors import CORSMiddleware
from app.api.routes import register_routes from app.api.routes import register_routes
from app.core.config import get_settings from app.core.config import get_settings
from app.db.store import Store from app.db.store import Store
from app.services.scheduler import start_scheduler, stop_scheduler, sync_all_campaigns from app.services.scheduler import (
start_scheduler, stop_scheduler, sync_all_campaigns,
start_scraper_job,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -33,6 +36,14 @@ async def lifespan(app: FastAPI):
app.state.scheduler = None app.state.scheduler = None
logger.info("Magpie started — scheduler disabled") logger.info("Magpie started — scheduler disabled")
# Start signal scraper job
if settings.scraper_enabled:
if not settings.scheduler_enabled:
# Scraper needs the scheduler even if campaign scheduling is off
start_scheduler()
start_scraper_job(interval_mins=settings.scraper_interval_mins)
logger.info("Signal scraper scheduled every %d min", settings.scraper_interval_mins)
store.close() store.close()
yield yield

View file

View file

@ -0,0 +1,122 @@
"""
Lemmy API v3 client read-only, no auth required for public communities.
Community addressing convention: <community>@<instance>
e.g. "selfhosted@lemmy.world", "technology@reddthat.com"
API reference: https://{instance}/api/v3/
Key endpoints used:
GET /api/v3/post/list?community_name=<name>&sort=New&limit=<n>&page=<p>
GET /api/v3/community?name=<name> (validation / metadata)
"""
from __future__ import annotations
import logging
from typing import Any
import httpx
logger = logging.getLogger(__name__)
# Instances known to be reliably federated and publicly accessible
KNOWN_INSTANCES = frozenset([
"lemmy.world",
"lemmy.ml",
"beehaw.org",
"reddthat.com",
"sh.itjust.works",
"feddit.de",
"aussie.zone",
"lemmy.ca",
])
_DEFAULT_TIMEOUT = 15.0
_DEFAULT_USER_AGENT = "Magpie/0.1 signal-monitor (by CircuitForge)"
def parse_community_target(sub: str) -> tuple[str, str]:
"""
Parse a 'community@instance' string into (community_name, instance_host).
Raises ValueError if the format is invalid.
"""
if "@" not in sub:
raise ValueError(
f"Lemmy community must be in 'community@instance' format, got: {sub!r}"
)
community, instance = sub.rsplit("@", 1)
if not community or not instance:
raise ValueError(f"Invalid Lemmy community target: {sub!r}")
return community.strip(), instance.strip().lower()
async def fetch_new_posts(
community: str,
instance: str,
last_seen_id: int | None,
limit: int = 50,
user_agent: str = _DEFAULT_USER_AGENT,
) -> tuple[list[dict[str, Any]], int | None]:
"""
Fetch new posts from a Lemmy community.
Fetches the first page sorted by New, then filters to posts with
id > last_seen_id (client-side cursor Lemmy has no 'before' param).
Returns:
posts filtered list of post dicts, newest first
new_max_id highest post ID seen this run (use as next cursor), or None
"""
url = f"https://{instance}/api/v3/post/list"
params = {
"community_name": community,
"sort": "New",
"limit": min(limit, 50), # Lemmy max is 50 per page
"type_": "Local", # stay within the instance's local community
}
async with httpx.AsyncClient(
headers={"User-Agent": user_agent},
follow_redirects=True,
timeout=_DEFAULT_TIMEOUT,
) as client:
resp = await client.get(url, params=params)
resp.raise_for_status()
data = resp.json()
post_views = data.get("posts", [])
# Normalize to flat dicts and apply cursor filter
posts = []
new_max_id: int | None = None
for pv in post_views:
post = pv.get("post", {})
counts = pv.get("counts", {})
creator = pv.get("creator", {})
post_id: int = post.get("id", 0)
# Track max ID regardless of cursor (always advance)
if new_max_id is None or post_id > new_max_id:
new_max_id = post_id
# Apply cursor: skip posts we've already seen
if last_seen_id is not None and post_id <= last_seen_id:
continue
posts.append({
"id": post_id,
"ap_id": post.get("ap_id", f"https://{instance}/post/{post_id}"),
"title": post.get("name", ""),
"body": post.get("body", "") or "",
"url": post.get("ap_id", ""), # ap_id is the canonical URL
"author": creator.get("name", ""),
"score": counts.get("score", 0),
"comment_count": counts.get("comments", 0),
"published": post.get("published", ""),
"community": community,
"instance": instance,
})
return posts, new_max_id

View file

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
from app.services.poster import run_campaign from app.services.poster import run_campaign
@ -119,3 +120,54 @@ def sync_all_campaigns(campaigns: list[dict]) -> None:
for campaign in campaigns: for campaign in campaigns:
sync_campaign(campaign) sync_campaign(campaign)
logger.info("Synced %d campaign(s) to scheduler", len(campaigns)) logger.info("Synced %d campaign(s) to scheduler", len(campaigns))
# ------------------------------------------------------------------ #
# Signal scraper job
# ------------------------------------------------------------------ #
_SCRAPER_JOB_ID = "signal_scraper"
async def _run_scraper_job() -> None:
"""APScheduler coroutine target: run one pass of signal scraping."""
from app.services.scraper import scrape_signals # deferred to avoid circular imports
logger.info("Signal scraper job starting")
try:
summary = await scrape_signals()
logger.info(
"Signal scraper done: %d sub(s), %d post(s) seen, %d signal(s) found",
summary["subs_scraped"], summary["posts_seen"], summary["signals_found"],
)
except Exception:
logger.exception("Unhandled error in signal scraper job")
def start_scraper_job(interval_mins: int = 30) -> None:
"""
Register (or replace) the signal scraper interval job.
Call this at startup when scraper_enabled=True.
"""
sched = get_scheduler()
existing = sched.get_job(_SCRAPER_JOB_ID)
if existing:
existing.remove()
sched.add_job(
_run_scraper_job,
trigger=IntervalTrigger(minutes=interval_mins, timezone="UTC"),
id=_SCRAPER_JOB_ID,
replace_existing=True,
misfire_grace_time=600,
)
logger.info("Signal scraper scheduled every %d minute(s)", interval_mins)
def stop_scraper_job() -> None:
"""Remove the signal scraper job (e.g. when disabling via settings)."""
sched = get_scheduler()
existing = sched.get_job(_SCRAPER_JOB_ID)
if existing:
existing.remove()
logger.info("Signal scraper job removed")

364
app/services/scraper.py Normal file
View file

@ -0,0 +1,364 @@
"""
Signal scraper: polls Reddit and Lemmy for posts matching signal_rules.
Architecture:
- One global APScheduler interval job dispatches to per-platform fetchers.
- Reddit: public JSON API, cursor via fullname (t3_xxxxx) + `before` param.
- Lemmy: public v3 API, cursor via max integer post ID (client-side filter).
- Correctness dedup lives in the DB (UNIQUE platform+post_id + INSERT OR IGNORE).
Cursors are a performance optimization, not a correctness mechanism.
- Keyword matching (_matches_rule) is platform-agnostic all rules evaluated
in one pass per fetched post to avoid redundant work.
Adding a new platform:
1. Add a fetch function returning (posts: list[NormalizedPost], cursor: str | None).
2. Add a branch in _process_platform_sub().
3. Add the platform string to signal_rules rows.
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
from datetime import datetime, timezone
from typing import Any
import httpx
from app.core.config import get_settings
from app.db.store import Store
logger = logging.getLogger(__name__)
# ------------------------------------------------------------------ #
# Reddit fetch
# ------------------------------------------------------------------ #
async def _fetch_reddit_posts(
sub: str,
before: str | None,
limit: int,
user_agent: str,
) -> tuple[list[dict[str, Any]], str | None]:
"""
Fetch new posts from r/{sub}.
Returns:
posts list of raw post data dicts (newest first)
new_cursor fullname of the newest post fetched, or None if empty
"""
params: dict[str, Any] = {"limit": limit, "raw_json": 1}
if before:
# 'before' in Reddit listing API = posts ABOVE this fullname = newer posts
params["before"] = before
url = f"https://www.reddit.com/r/{sub}/new.json"
async with httpx.AsyncClient(
headers={"User-Agent": user_agent},
follow_redirects=True,
timeout=15.0,
) as client:
resp = await client.get(url, params=params)
resp.raise_for_status()
data = resp.json()
children = data.get("data", {}).get("children", [])
posts = [c["data"] for c in children if c.get("kind") == "t3"]
# Newest post is first in the list; its fullname becomes the next cursor
new_cursor = posts[0]["name"] if posts else None
return posts, new_cursor
# ------------------------------------------------------------------ #
# Keyword matching
# ------------------------------------------------------------------ #
def _matches_rule(post: dict[str, Any], rule: dict[str, Any]) -> list[str]:
"""
Check if a post matches a signal rule.
Returns a list of matched keywords (non-empty = match), or [] (no match).
"""
haystack = f"{post.get('title', '')} {post.get('selftext', '')}".lower()
keywords: list[str] = json.loads(rule["keywords"]) if isinstance(rule["keywords"], str) else rule["keywords"]
match_mode: str = rule.get("match_mode", "any")
min_score: int = rule.get("min_score", 0)
# Score gate: skip posts below the minimum
post_score = post.get("score", 0) or 0
if post_score < min_score:
return []
# No keywords = match everything (useful for "watch this sub broadly")
if not keywords:
return ["*"]
if match_mode == "regex":
matched = []
for pattern in keywords:
try:
if re.search(pattern, haystack, re.IGNORECASE):
matched.append(pattern)
except re.error:
logger.warning("Invalid regex in rule %d: %r", rule["id"], pattern)
return matched if matched else []
if match_mode == "all":
matched = [kw for kw in keywords if kw.lower() in haystack]
return matched if len(matched) == len(keywords) else []
# Default: any
return [kw for kw in keywords if kw.lower() in haystack]
# ------------------------------------------------------------------ #
# Normalized post type
# ------------------------------------------------------------------ #
# Both platform fetchers normalize their output to this shape before
# the common matching + upsert pipeline runs.
#
# Fields mirror Reddit's naming where possible so _matches_rule() which
# uses "title" and "selftext" works without platform-specific branches.
class NormalizedPost(dict):
"""
Keys guaranteed present:
post_id str unique within platform (Reddit fullname, Lemmy ap_id URL)
title str
selftext str body text (empty string if none)
score int
num_comments int
author str
url str
created_utc float | None
"""
def _normalize_reddit(post: dict[str, Any]) -> NormalizedPost:
return NormalizedPost({
"post_id": post["name"],
"title": post.get("title", ""),
"selftext": post.get("selftext", "") or "",
"score": post.get("score", 0),
"num_comments": post.get("num_comments", 0),
"author": post.get("author", ""),
"url": post.get("url", ""),
"created_utc": post.get("created_utc"),
})
def _normalize_lemmy(post: dict[str, Any]) -> NormalizedPost:
published = post.get("published", "")
try:
ts = datetime.fromisoformat(published.rstrip("Z")).replace(tzinfo=timezone.utc).timestamp()
except (ValueError, AttributeError):
ts = None
return NormalizedPost({
"post_id": post["ap_id"],
"title": post.get("title", ""),
"selftext": post.get("body", "") or "",
"score": post.get("score", 0),
"num_comments": post.get("comment_count", 0),
"author": post.get("author", ""),
"url": post.get("url", "") or post.get("ap_id", ""),
"created_utc": ts,
})
# ------------------------------------------------------------------ #
# Common match + upsert pipeline
# ------------------------------------------------------------------ #
def _run_matching(
post: NormalizedPost,
rules: list[dict],
) -> tuple[list[int], list[str]]:
"""Apply all rules to a post. Returns (matched_rule_ids, matched_keywords)."""
matched_kws: set[str] = set()
matched_rule_ids: list[int] = []
for rule in rules:
kws = _matches_rule(post, rule)
if kws:
matched_kws.update(k for k in kws if k != "*")
matched_rule_ids.append(rule["id"])
return matched_rule_ids, sorted(matched_kws)
def _upsert_post(
post: NormalizedPost,
platform: str,
sub: str,
matched_rule_ids: list[int],
matched_kws: list[str],
store: Store,
) -> None:
posted_at = (
datetime.fromtimestamp(post["created_utc"], tz=timezone.utc).isoformat()
if post.get("created_utc")
else None
)
signal = store.upsert_signal(
platform=platform,
sub=sub,
post_id=post["post_id"],
title=post["title"],
body_snippet=(post["selftext"][:500] or None),
score=post["score"],
comment_count=post["num_comments"],
author=post["author"],
url=post["url"],
posted_at=posted_at,
matched_keywords=matched_kws,
)
for rule_id in matched_rule_ids:
store.record_signal_rule_match(signal["id"], rule_id)
# ------------------------------------------------------------------ #
# Core scraper
# ------------------------------------------------------------------ #
def _build_target_map(
all_rules: list[dict],
) -> dict[tuple[str, str], list[dict]]:
"""
Build a (platform, sub) [rules] map from active rules.
Rules with sub=None are global and appended to every explicit target.
Returns an empty dict if no sub-specific targets exist.
"""
global_rules_by_platform: dict[str, list[dict]] = {}
target_map: dict[tuple[str, str], list[dict]] = {}
for rule in all_rules:
platform = rule.get("platform", "reddit")
if not rule["sub"]:
global_rules_by_platform.setdefault(platform, []).append(rule)
else:
key = (platform, rule["sub"])
target_map.setdefault(key, []).append(rule)
# Attach global rules to each explicit target of the same platform
for (platform, _sub), rules in target_map.items():
rules.extend(global_rules_by_platform.get(platform, []))
return target_map
async def scrape_signals() -> dict[str, int]:
"""
Run one full scrape pass across all platforms and monitored communities.
Returns: {"subs_scraped": N, "posts_seen": N, "signals_found": N}
"""
from app.services.lemmy.client import fetch_new_posts as lemmy_fetch, parse_community_target
settings = get_settings()
store = Store(settings.db_path)
try:
all_rules = store.list_signal_rules(active_only=True)
if not all_rules:
logger.info("Signal scraper: no active rules, skipping")
return {"subs_scraped": 0, "posts_seen": 0, "signals_found": 0}
target_map = _build_target_map(all_rules)
if not target_map:
logger.info("Signal scraper: global rules but no sub targets — add sub-specific rules")
return {"subs_scraped": 0, "posts_seen": 0, "signals_found": 0}
total_posts = 0
total_signals = 0
for (platform, sub), rules in target_map.items():
state = store.get_scrape_state(sub, platform)
cursor = state["last_fullname"] if state else None
label = f"{platform}:{sub}"
logger.info("Scraping %s (cursor=%s, rules=%d)", label, cursor, len(rules))
# ---- Fetch -----------------------------------------------
raw_posts: list[dict[str, Any]] = []
new_cursor: str | None = None
try:
if platform == "reddit":
raw_posts, new_cursor = await _fetch_reddit_posts(
sub=sub,
before=cursor,
limit=settings.scraper_fetch_limit,
user_agent=settings.scraper_user_agent,
)
normalize = _normalize_reddit
elif platform == "lemmy":
community, instance = parse_community_target(sub)
last_id = int(cursor) if cursor and cursor.isdigit() else None
lemmy_posts, max_id = await lemmy_fetch(
community=community,
instance=instance,
last_seen_id=last_id,
limit=settings.scraper_fetch_limit,
user_agent=settings.scraper_user_agent,
)
raw_posts = lemmy_posts
new_cursor = str(max_id) if max_id is not None else None
normalize = _normalize_lemmy
else:
logger.warning("Unknown platform %r for sub %r — skipping", platform, sub)
continue
except httpx.HTTPStatusError as exc:
if exc.response.status_code in (403, 404):
logger.warning("%s returned %d — community may be private", label, exc.response.status_code)
else:
logger.error("HTTP error scraping %s: %s", label, exc)
await asyncio.sleep(settings.scraper_request_delay_secs)
continue
except ValueError as exc:
logger.error("Config error for %s: %s", label, exc)
continue
except Exception:
logger.exception("Unexpected error scraping %s", label)
await asyncio.sleep(settings.scraper_request_delay_secs)
continue
# ---- Match + upsert --------------------------------------
sub_signals = 0
for raw in raw_posts:
total_posts += 1
post = normalize(raw)
matched_rule_ids, matched_kws = _run_matching(post, rules)
if not matched_rule_ids:
continue
_upsert_post(post, platform, sub, matched_rule_ids, matched_kws, store)
sub_signals += 1
logger.debug("Signal: %s%s", label, post["title"][:60])
total_signals += sub_signals
if new_cursor:
store.update_scrape_state(
sub=sub,
platform=platform,
last_fullname=new_cursor,
posts_seen_delta=len(raw_posts),
signals_found_delta=sub_signals,
)
logger.info("%s: %d posts checked, %d signal(s) found", label, len(raw_posts), sub_signals)
await asyncio.sleep(settings.scraper_request_delay_secs)
return {
"subs_scraped": len(target_map),
"posts_seen": total_posts,
"signals_found": total_signals,
}
finally:
store.close()

View file

@ -4,6 +4,11 @@
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Magpie — Campaign Dashboard</title> <title>Magpie — Campaign Dashboard</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Space+Mono:wght@400;700&family=DM+Sans:ital,opsz,wght@0,9..40,300;0,9..40,400;0,9..40,500;0,9..40,600;1,9..40,300&display=swap" rel="stylesheet">
<meta name="theme-color" content="#0f1117">
<meta name="apple-mobile-web-app-capable" content="yes">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>

View file

@ -1,14 +1,52 @@
<template> <template>
<div class="app-shell"> <div class="app-shell">
<!-- Sidebar (desktop 768px) -->
<nav class="sidebar"> <nav class="sidebar">
<div class="nav-brand">🪶 Magpie</div> <div class="nav-brand">🪶 Magpie</div>
<router-link class="nav-link" to="/opportunities" active-class="active">Opportunities</router-link> <router-link class="nav-link" to="/signals" active-class="active">
<router-link class="nav-link" to="/campaigns" active-class="active">Campaigns</router-link> <span>🔍</span> Signals
<router-link class="nav-link" to="/posts" active-class="active">Post History</router-link> </router-link>
<router-link class="nav-link" to="/subs" active-class="active">Sub Rules</router-link> <router-link class="nav-link" to="/opportunities" active-class="active">
<span>📥</span> Opportunities
</router-link>
<router-link class="nav-link" to="/campaigns" active-class="active">
<span>📡</span> Campaigns
</router-link>
<router-link class="nav-link" to="/posts" active-class="active">
<span>📋</span> Post History
</router-link>
<router-link class="nav-link" to="/subs" active-class="active">
<span>📖</span> Sub Rules
</router-link>
</nav> </nav>
<!-- Main -->
<main class="main-content"> <main class="main-content">
<router-view /> <router-view />
</main> </main>
<!-- Bottom Nav (mobile <768px) -->
<nav class="bottom-nav" aria-label="Main navigation">
<router-link class="bottom-nav-item" to="/signals" active-class="active">
<span class="nav-icon">🔍</span>
<span>Signals</span>
</router-link>
<router-link class="bottom-nav-item" to="/opportunities" active-class="active">
<span class="nav-icon">📥</span>
<span>Queue</span>
</router-link>
<router-link class="bottom-nav-item" to="/campaigns" active-class="active">
<span class="nav-icon">📡</span>
<span>Campaigns</span>
</router-link>
<router-link class="bottom-nav-item" to="/posts" active-class="active">
<span class="nav-icon">📋</span>
<span>Posts</span>
</router-link>
<router-link class="bottom-nav-item" to="/subs" active-class="active">
<span class="nav-icon">📖</span>
<span>Rules</span>
</router-link>
</nav>
</div> </div>
</template> </template>

View file

@ -12,7 +12,7 @@
</button> </button>
</div> </div>
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: var(--spacing-lg);"> <div class="detail-grid">
<!-- Left: variants + subs --> <!-- Left: variants + subs -->
<div> <div>
<!-- Variants --> <!-- Variants -->
@ -52,7 +52,7 @@
<!-- Right: recent posts --> <!-- Right: recent posts -->
<div class="card" style="padding: 0; overflow: hidden; align-self: start;"> <div class="card" style="padding: 0; overflow: hidden; align-self: start;">
<div class="card-title" style="padding: var(--spacing-md) var(--spacing-md) 0;">Recent Posts</div> <div class="card-title" style="padding: var(--spacing-md) var(--spacing-md) 0;">Recent Posts</div>
<table class="table"> <table class="table table-responsive">
<thead> <thead>
<tr> <tr>
<th>Sub</th> <th>Sub</th>
@ -62,13 +62,13 @@
</thead> </thead>
<tbody> <tbody>
<tr v-for="p in recentPosts" :key="p.id"> <tr v-for="p in recentPosts" :key="p.id">
<td>r/{{ p.target }}</td> <td data-label="Sub">r/{{ p.target }}</td>
<td> <td data-label="Status">
<span :class="['status-dot', p.status]"></span> <span :class="['status-dot', p.status]"></span>
<a v-if="p.url" :href="p.url" target="_blank" style="color: var(--color-primary); text-decoration: none;">{{ p.status }}</a> <a v-if="p.url" :href="p.url" target="_blank" style="color: var(--color-primary); text-decoration: none;">{{ p.status }}</a>
<span v-else>{{ p.status }}</span> <span v-else>{{ p.status }}</span>
</td> </td>
<td style="color: var(--color-text-muted); font-size: 12px;">{{ formatDate(p.posted_at) }}</td> <td data-label="When" style="color: var(--color-text-muted); font-size: 12px;">{{ formatDate(p.posted_at) }}</td>
</tr> </tr>
<tr v-if="recentPosts.length === 0"> <tr v-if="recentPosts.length === 0">
<td colspan="3" style="color: var(--color-text-muted); text-align: center; padding: var(--spacing-lg);">No posts yet.</td> <td colspan="3" style="color: var(--color-text-muted); text-align: center; padding: var(--spacing-lg);">No posts yet.</td>
@ -210,4 +210,12 @@ function formatDate(iso: string) {
.modal { padding: var(--spacing-lg); } .modal { padding: var(--spacing-lg); }
.variant-row { padding: var(--spacing-sm); border-bottom: 1px solid var(--color-border); } .variant-row { padding: var(--spacing-sm); border-bottom: 1px solid var(--color-border); }
.variant-row:last-child { border-bottom: none; } .variant-row:last-child { border-bottom: none; }
.detail-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: var(--spacing-lg);
}
@media (max-width: 768px) {
.detail-grid { grid-template-columns: 1fr; }
}
</style> </style>

View file

@ -12,7 +12,7 @@
</div> </div>
<div v-else class="card" style="padding: 0; overflow: hidden;"> <div v-else class="card" style="padding: 0; overflow: hidden;">
<table class="table"> <table class="table table-responsive">
<thead> <thead>
<tr> <tr>
<th>Name</th> <th>Name</th>
@ -24,21 +24,21 @@
</thead> </thead>
<tbody> <tbody>
<tr v-for="c in store.campaigns" :key="c.id"> <tr v-for="c in store.campaigns" :key="c.id">
<td> <td data-label="Name">
<router-link :to="`/campaigns/${c.id}`" style="color: var(--color-primary); text-decoration: none; font-weight: 500;"> <router-link :to="`/campaigns/${c.id}`" style="color: var(--color-primary); text-decoration: none; font-weight: 500;">
{{ c.name }} {{ c.name }}
</router-link> </router-link>
</td> </td>
<td><span class="badge badge-info">{{ c.product }}</span></td> <td data-label="Product"><span class="badge badge-info">{{ c.product }}</span></td>
<td style="font-family: var(--font-mono); font-size: 12px; color: var(--color-text-muted);"> <td data-label="Schedule" class="text-mono text-sm text-muted">
{{ c.cron_schedule ?? '— manual' }} {{ c.cron_schedule ?? '— manual' }}
</td> </td>
<td> <td data-label="Status">
<span :class="['badge', c.active ? 'badge-success' : 'badge-muted']"> <span :class="['badge', c.active ? 'badge-success' : 'badge-muted']">
{{ c.active ? 'active' : 'paused' }} {{ c.active ? 'active' : 'paused' }}
</span> </span>
</td> </td>
<td> <td data-label="">
<div style="display: flex; gap: 6px; justify-content: flex-end;"> <div style="display: flex; gap: 6px; justify-content: flex-end;">
<button class="btn btn-ghost btn-sm" @click="toggle(c)" :title="c.active ? 'Pause' : 'Resume'"> <button class="btn btn-ghost btn-sm" @click="toggle(c)" :title="c.active ? 'Pause' : 'Resume'">
{{ c.active ? '⏸' : '▶' }} {{ c.active ? '⏸' : '▶' }}

View file

@ -16,6 +16,9 @@
</div> </div>
<div v-if="loading" class="state-empty">Loading...</div> <div v-if="loading" class="state-empty">Loading...</div>
<div v-else-if="loadError" class="state-empty" style="color: var(--color-danger)">
Could not load opportunities: {{ loadError }}
</div>
<div v-else-if="filtered.length === 0" class="state-empty"> <div v-else-if="filtered.length === 0" class="state-empty">
No opportunities{{ filterStatus ? ` with status "${filterStatus}"` : '' }}. No opportunities{{ filterStatus ? ` with status "${filterStatus}"` : '' }}.
</div> </div>
@ -199,6 +202,7 @@ const selected = ref<Opportunity | null>(null)
const showAddModal = ref(false) const showAddModal = ref(false)
const copied = ref(false) const copied = ref(false)
const filterStatus = ref<OpportunityStatus | ''>('') const filterStatus = ref<OpportunityStatus | ''>('')
const loadError = ref<string | null>(null)
const editBody = ref('') const editBody = ref('')
const editTitle = ref('') const editTitle = ref('')
@ -222,8 +226,11 @@ const filtered = computed(() =>
async function load() { async function load() {
loading.value = true loading.value = true
loadError.value = null
try { try {
opportunities.value = await api.opportunities.list() opportunities.value = await api.opportunities.list()
} catch (e: unknown) {
loadError.value = e instanceof Error ? e.message : 'Failed to load opportunities'
} finally { } finally {
loading.value = false loading.value = false
} }

View file

@ -5,7 +5,7 @@
</div> </div>
<div class="card" style="padding: 0; overflow: hidden;"> <div class="card" style="padding: 0; overflow: hidden;">
<table class="table"> <table class="table table-responsive">
<thead> <thead>
<tr> <tr>
<th>Campaign</th> <th>Campaign</th>
@ -18,15 +18,15 @@
</thead> </thead>
<tbody> <tbody>
<tr v-for="p in posts" :key="p.id"> <tr v-for="p in posts" :key="p.id">
<td>{{ campaignName(p.campaign_id) }}</td> <td data-label="Campaign">{{ campaignName(p.campaign_id) }}</td>
<td>{{ p.target }}</td> <td data-label="Target">{{ p.target }}</td>
<td> <td data-label="Status">
<span :class="['status-dot', p.status]"></span>{{ p.status }} <span :class="['status-dot', p.status]"></span>{{ p.status }}
<span v-if="p.error_msg" :title="p.error_msg" style="color: var(--color-danger); cursor: help;"> </span> <span v-if="p.error_msg" :title="p.error_msg" style="color: var(--color-danger); cursor: help;"> </span>
</td> </td>
<td><span class="badge badge-muted">{{ p.triggered_by }}</span></td> <td data-label="Triggered by"><span class="badge badge-muted">{{ p.triggered_by }}</span></td>
<td style="color: var(--color-text-muted); font-size: 12px;">{{ formatDate(p.posted_at) }}</td> <td data-label="When" style="color: var(--color-text-muted); font-size: 12px;">{{ formatDate(p.posted_at) }}</td>
<td> <td data-label="Link">
<a v-if="p.url" :href="p.url" target="_blank" style="color: var(--color-primary); font-size: 12px;">view </a> <a v-if="p.url" :href="p.url" target="_blank" style="color: var(--color-primary); font-size: 12px;">view </a>
<span v-else style="color: var(--color-text-muted);"></span> <span v-else style="color: var(--color-text-muted);"></span>
</td> </td>

View file

@ -0,0 +1,351 @@
<template>
<div>
<div class="page-header">
<h1 class="page-title">Signal Queue</h1>
<div style="display: flex; gap: var(--spacing-sm);">
<select class="form-select" v-model="filterStatus" style="width: auto; font-size: 13px;">
<option value="">All</option>
<option value="new">New</option>
<option value="saved">Saved</option>
<option value="dismissed">Dismissed</option>
</select>
<button class="btn btn-ghost btn-sm" @click="showRules = true"> Rules</button>
</div>
</div>
<div v-if="loading" class="empty-state">Loading...</div>
<div v-else-if="error" class="empty-state" style="color: var(--color-danger)">{{ error }}</div>
<div v-else-if="filtered.length === 0" class="empty-state">
{{ filterStatus ? `No ${filterStatus} signals.` : 'No signals yet — add monitoring rules and run the scraper.' }}
</div>
<div v-else class="signal-feed">
<div
v-for="s in filtered"
:key="s.id"
class="signal-card"
:class="{ 'is-dismissed': s.status === 'dismissed', 'is-saved': s.status === 'saved' }"
>
<div class="signal-header">
<div class="signal-chips">
<span class="chip chip-sub">{{ s.sub }}</span>
<span v-if="s.score !== null" class="chip chip-score">{{ s.score }}</span>
<span v-if="s.comment_count !== null" class="chip chip-comments">{{ s.comment_count }}c</span>
<span v-for="kw in s.matched_keywords" :key="kw" class="chip chip-kw">{{ kw }}</span>
</div>
<div class="signal-actions">
<button
v-if="s.status !== 'saved'"
class="btn btn-ghost btn-xs"
title="Save"
@click.stop="updateStatus(s, 'saved')"
></button>
<button
v-if="s.status !== 'dismissed'"
class="btn btn-ghost btn-xs"
title="Dismiss"
@click.stop="updateStatus(s, 'dismissed')"
></button>
<button
v-if="s.status === 'dismissed'"
class="btn btn-ghost btn-xs"
title="Restore"
@click.stop="updateStatus(s, 'new')"
></button>
</div>
</div>
<a
v-if="s.url"
:href="s.url"
target="_blank"
class="signal-title"
>{{ s.title }}</a>
<div v-else class="signal-title" style="cursor: default;">{{ s.title }}</div>
<p v-if="s.body_snippet" class="signal-snippet">{{ s.body_snippet }}</p>
<div class="signal-footer">
<span class="signal-author">u/{{ s.author ?? '?' }}</span>
<span class="signal-date">{{ formatDate(s.surfaced_at) }}</span>
</div>
</div>
</div>
<!-- Signal rules panel -->
<div v-if="showRules" class="modal-backdrop" @click.self="showRules = false">
<div class="modal card" style="width: 560px; max-height: 85vh; overflow-y: auto;">
<h2 style="margin-bottom: var(--spacing-md); font-size: 16px;">Signal Monitoring Rules</h2>
<div v-if="rules.length === 0" class="empty-state" style="padding: var(--spacing-md);">
No rules. Add one to start monitoring communities.
</div>
<div v-for="r in rules" :key="r.id" class="rule-row">
<div style="display: flex; align-items: center; gap: var(--spacing-sm);">
<span style="font-weight: 500; font-size: 13px;">{{ r.name }}</span>
<span v-if="r.sub" class="chip chip-sub" style="font-size: 10px;">{{ r.sub }}</span>
<span v-if="r.label" class="chip" :class="`chip-label-${r.label}`" style="font-size: 10px;">{{ r.label }}</span>
<span v-if="!r.active" class="badge badge-muted" style="margin-left: auto; font-size: 10px;">paused</span>
<div v-else style="margin-left: auto; display: flex; gap: 4px;">
<button class="btn btn-ghost btn-xs" @click="toggleRule(r)"></button>
<button class="btn btn-ghost btn-xs" style="color: var(--color-danger);" @click="deleteRule(r.id)"></button>
</div>
</div>
<div class="rule-keywords">
<code v-for="kw in r.keywords" :key="kw" class="kw-tag">{{ kw }}</code>
<span v-if="r.keywords.length === 0" style="color: var(--color-text-muted); font-size: 11px;">no keywords matches all posts</span>
</div>
<div style="font-size: 11px; color: var(--color-text-muted);">
{{ r.match_mode }} match · min score {{ r.min_score }}
</div>
</div>
<div style="border-top: 1px solid var(--color-border); margin-top: var(--spacing-md); padding-top: var(--spacing-md);">
<div style="font-size: 13px; font-weight: 500; margin-bottom: var(--spacing-sm);">New rule</div>
<div class="form-group">
<label class="form-label">Name</label>
<input class="form-input" v-model="ruleForm.name" placeholder="Kiwi pain points" />
</div>
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: var(--spacing-sm);">
<div class="form-group">
<label class="form-label">Sub (blank = all)</label>
<input class="form-input" v-model="ruleForm.sub" placeholder="selfhosted" />
</div>
<div class="form-group">
<label class="form-label">Label</label>
<select class="form-select" v-model="ruleForm.label">
<option value=""> none </option>
<option value="pain-point">pain-point</option>
<option value="feedback">feedback</option>
<option value="mention">mention</option>
<option value="trust">trust</option>
</select>
</div>
</div>
<div class="form-group">
<label class="form-label">Keywords (comma-separated)</label>
<input class="form-input" v-model="ruleForm.keywordsRaw" placeholder="food waste, expiry, pantry" style="font-family: var(--font-mono);" />
</div>
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: var(--spacing-sm);">
<div class="form-group">
<label class="form-label">Match mode</label>
<select class="form-select" v-model="ruleForm.match_mode">
<option value="any">any keyword</option>
<option value="all">all keywords</option>
</select>
</div>
<div class="form-group">
<label class="form-label">Min score</label>
<input class="form-input" type="number" v-model.number="ruleForm.min_score" min="0" />
</div>
</div>
</div>
<div style="display: flex; gap: var(--spacing-sm); justify-content: flex-end; margin-top: var(--spacing-sm);">
<button class="btn btn-ghost" @click="showRules = false">Close</button>
<button class="btn btn-primary" @click="addRule" :disabled="!ruleForm.name">Add Rule</button>
</div>
</div>
</div>
</div>
</template>
<script setup lang="ts">
import { ref, computed, onMounted, reactive } from 'vue'
import { api, type Signal, type SignalStatus, type SignalRule } from '@/services/api'
const signals = ref<Signal[]>([])
const rules = ref<SignalRule[]>([])
const loading = ref(false)
const error = ref<string | null>(null)
const filterStatus = ref<SignalStatus | ''>('')
const showRules = ref(false)
const ruleForm = reactive({
name: '',
sub: '',
label: '' as '' | 'pain-point' | 'feedback' | 'mention' | 'trust',
keywordsRaw: '',
match_mode: 'any' as 'any' | 'all',
min_score: 0,
})
const filtered = computed(() =>
filterStatus.value
? signals.value.filter(s => s.status === filterStatus.value)
: signals.value
)
onMounted(async () => {
loading.value = true
error.value = null
try {
const [sigs, rls] = await Promise.all([
api.signals.list(),
api.signalRules.list(),
])
signals.value = sigs
rules.value = rls
} catch (e: unknown) {
error.value = e instanceof Error ? e.message : 'Failed to load signals'
} finally {
loading.value = false
}
})
async function updateStatus(s: Signal, status: SignalStatus) {
const updated = await api.signals.updateStatus(s.id, status)
const idx = signals.value.findIndex(x => x.id === s.id)
if (idx >= 0) signals.value[idx] = updated
}
async function addRule() {
const keywords = ruleForm.keywordsRaw
.split(',')
.map(k => k.trim())
.filter(Boolean)
const rule = await api.signalRules.create({
name: ruleForm.name,
sub: ruleForm.sub || null,
label: ruleForm.label || null,
keywords,
match_mode: ruleForm.match_mode,
min_score: ruleForm.min_score,
})
rules.value = [...rules.value, rule]
Object.assign(ruleForm, { name: '', sub: '', label: '', keywordsRaw: '', match_mode: 'any', min_score: 0 })
}
async function toggleRule(r: SignalRule) {
const updated = await api.signalRules.update(r.id, { active: !r.active })
const idx = rules.value.findIndex(x => x.id === r.id)
if (idx >= 0) rules.value[idx] = updated
}
async function deleteRule(id: number) {
await api.signalRules.delete(id)
rules.value = rules.value.filter(r => r.id !== id)
}
function formatDate(iso: string) {
return new Date(iso).toLocaleDateString(undefined, { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' })
}
</script>
<style scoped>
.signal-feed {
display: flex;
flex-direction: column;
gap: var(--spacing-sm);
}
.signal-card {
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: var(--radius-md);
padding: var(--spacing-md);
transition: border-color 0.15s;
}
.signal-card:hover { border-color: var(--color-primary); }
.signal-card.is-dismissed { opacity: 0.45; }
.signal-card.is-saved { border-left: 3px solid var(--color-primary); }
.signal-header {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: var(--spacing-sm);
margin-bottom: var(--spacing-xs);
}
.signal-chips { display: flex; flex-wrap: wrap; gap: 4px; }
.chip {
font-size: 10px;
padding: 1px 6px;
border-radius: var(--radius-sm);
border: 1px solid var(--color-border);
background: var(--color-bg-secondary);
color: var(--color-text-muted);
font-family: var(--font-mono);
}
.chip-sub { color: var(--color-primary); border-color: var(--color-primary); }
.chip-score { color: var(--color-success); border-color: var(--color-success); }
.chip-comments { color: var(--color-text-muted); }
.chip-kw { color: var(--color-warning); border-color: var(--color-warning); }
.chip-label-pain-point { color: var(--color-danger); border-color: var(--color-danger); }
.chip-label-feedback { color: var(--color-info); border-color: var(--color-info); }
.chip-label-mention { color: var(--color-primary); border-color: var(--color-primary); }
.chip-label-trust { color: var(--color-success); border-color: var(--color-success); }
.signal-actions { display: flex; gap: 2px; flex-shrink: 0; }
.btn-xs {
padding: 2px 6px;
font-size: 11px;
min-height: 24px;
}
.signal-title {
display: block;
font-size: 14px;
font-weight: 500;
color: var(--color-text);
text-decoration: none;
margin-bottom: var(--spacing-xs);
line-height: 1.4;
}
a.signal-title:hover { color: var(--color-primary); }
.signal-snippet {
font-size: 12px;
color: var(--color-text-muted);
margin: 0 0 var(--spacing-xs);
line-height: 1.5;
display: -webkit-box;
-webkit-line-clamp: 3;
-webkit-box-orient: vertical;
overflow: hidden;
}
.signal-footer {
display: flex;
align-items: center;
gap: var(--spacing-sm);
font-size: 11px;
color: var(--color-text-muted);
font-family: var(--font-mono);
}
/* Rules panel */
.modal-backdrop {
position: fixed; inset: 0; background: rgba(0,0,0,0.6);
display: flex; align-items: center; justify-content: center; z-index: 100;
}
.modal { padding: var(--spacing-lg); }
.rule-row {
padding: var(--spacing-sm) 0;
border-bottom: 1px solid var(--color-border);
display: flex;
flex-direction: column;
gap: 4px;
}
.rule-row:last-of-type { border-bottom: none; }
.rule-keywords { display: flex; flex-wrap: wrap; gap: 4px; }
.kw-tag {
font-size: 10px;
background: var(--color-primary-dim);
color: var(--color-primary);
border-radius: var(--radius-sm);
padding: 1px 6px;
font-family: var(--font-mono);
}
@media (max-width: 640px) {
.modal { width: calc(100vw - 2 * var(--spacing-md)) !important; }
}
</style>

View file

@ -6,7 +6,7 @@
</div> </div>
<div class="card" style="padding: 0; overflow: hidden;"> <div class="card" style="padding: 0; overflow: hidden;">
<table class="table"> <table class="table table-responsive">
<thead> <thead>
<tr> <tr>
<th>Sub / Channel</th> <th>Sub / Channel</th>
@ -20,25 +20,25 @@
</thead> </thead>
<tbody> <tbody>
<tr v-for="r in rules" :key="r.id"> <tr v-for="r in rules" :key="r.id">
<td style="font-weight: 500;">{{ r.sub }}</td> <td data-label="Sub" style="font-weight: 500;">{{ r.sub }}</td>
<td><span class="badge badge-muted">{{ r.platform }}</span></td> <td data-label="Platform"><span class="badge badge-muted">{{ r.platform }}</span></td>
<td> <td data-label="Flair">
<span v-if="r.flair_required">{{ r.flair_to_use ?? '(required, unknown)' }}</span> <span v-if="r.flair_required">{{ r.flair_to_use ?? '(required, unknown)' }}</span>
<span v-else style="color: var(--color-text-muted);"></span> <span v-else style="color: var(--color-text-muted);"></span>
</td> </td>
<td> <td data-label="Promo">
<span v-if="r.promo_allowed === null" class="badge badge-muted">unknown</span> <span v-if="r.promo_allowed === null" class="badge badge-muted">unknown</span>
<span v-else-if="r.promo_allowed" class="badge badge-success">allowed</span> <span v-else-if="r.promo_allowed" class="badge badge-success">allowed</span>
<span v-else class="badge badge-danger">banned</span> <span v-else class="badge badge-danger">banned</span>
</td> </td>
<td> <td data-label="Rule Warning">
<span v-if="r.rule_warning" class="badge badge-warning">yes</span> <span v-if="r.rule_warning" class="badge badge-warning">yes</span>
<span v-else style="color: var(--color-text-muted);"></span> <span v-else style="color: var(--color-text-muted);"></span>
</td> </td>
<td style="color: var(--color-text-muted); max-width: 200px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;"> <td data-label="Notes" style="color: var(--color-text-muted); max-width: 200px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">
{{ r.notes ?? '—' }} {{ r.notes ?? '—' }}
</td> </td>
<td> <td data-label="">
<button class="btn btn-ghost btn-sm" @click="startEdit(r)">Edit</button> <button class="btn btn-ghost btn-sm" @click="startEdit(r)">Edit</button>
</td> </td>
</tr> </tr>

View file

@ -9,11 +9,13 @@ import CampaignDetail from './components/CampaignDetail.vue'
import SubRulesView from './components/SubRulesView.vue' import SubRulesView from './components/SubRulesView.vue'
import PostsView from './components/PostsView.vue' import PostsView from './components/PostsView.vue'
import OpportunitiesView from './components/OpportunitiesView.vue' import OpportunitiesView from './components/OpportunitiesView.vue'
import SignalsView from './components/SignalsView.vue'
const router = createRouter({ const router = createRouter({
history: createWebHistory(), history: createWebHistory(),
routes: [ routes: [
{ path: '/', redirect: '/opportunities' }, { path: '/', redirect: '/signals' },
{ path: '/signals', component: SignalsView },
{ path: '/opportunities', component: OpportunitiesView }, { path: '/opportunities', component: OpportunitiesView },
{ path: '/campaigns', component: CampaignList }, { path: '/campaigns', component: CampaignList },
{ path: '/campaigns/:id', component: CampaignDetail }, { path: '/campaigns/:id', component: CampaignDetail },

View file

@ -147,6 +147,54 @@ export interface ApproveResult {
instructions: string instructions: string
} }
export type SignalMatchMode = 'any' | 'all' | 'regex'
export type SignalLabel = 'pain-point' | 'feedback' | 'mention' | 'trust'
export type SignalStatus = 'new' | 'saved' | 'dismissed'
export interface SignalRule {
id: number
name: string
platform: string
sub: string | null
keywords: string[]
match_mode: SignalMatchMode
min_score: number
label: SignalLabel | null
active: number
created_at: string
notes: string | null
}
export interface SignalRuleCreate {
name: string
platform?: string
sub?: string | null
keywords?: string[]
match_mode?: SignalMatchMode
min_score?: number
label?: SignalLabel | null
notes?: string | null
}
export interface Signal {
id: number
platform: string
sub: string
post_id: string
title: string
body_snippet: string | null
score: number | null
comment_count: number | null
author: string | null
url: string | null
posted_at: string | null
surfaced_at: string
matched_keywords: string[]
status: SignalStatus
notes: string | null
matched_rules?: SignalRule[]
}
// ------------------------------------------------------------------ // // ------------------------------------------------------------------ //
// Campaigns // Campaigns
// ------------------------------------------------------------------ // // ------------------------------------------------------------------ //
@ -233,4 +281,32 @@ export const api = {
dismiss: (id: number, note?: string) => dismiss: (id: number, note?: string) =>
http.post<Opportunity>(`/opportunities/${id}/dismiss`, { note: note ?? null }).then(r => r.data), http.post<Opportunity>(`/opportunities/${id}/dismiss`, { note: note ?? null }).then(r => r.data),
}, },
signalRules: {
list: (activeOnly = false) =>
http.get<SignalRule[]>('/signal-rules', { params: { active_only: activeOnly } }).then(r => r.data),
create: (data: SignalRuleCreate) =>
http.post<SignalRule>('/signal-rules', data).then(r => r.data),
get: (id: number) =>
http.get<SignalRule>(`/signal-rules/${id}`).then(r => r.data),
update: (id: number, data: Partial<SignalRuleCreate> & { active?: boolean }) =>
http.patch<SignalRule>(`/signal-rules/${id}`, data).then(r => r.data),
delete: (id: number) =>
http.delete(`/signal-rules/${id}`),
},
signals: {
list: (params?: { status?: SignalStatus; platform?: string; sub?: string; limit?: number }) =>
http.get<Signal[]>('/signals', { params }).then(r => r.data),
get: (id: number) =>
http.get<Signal>(`/signals/${id}`).then(r => r.data),
updateStatus: (id: number, status: SignalStatus, notes?: string) =>
http.patch<Signal>(`/signals/${id}/status`, { status, notes: notes ?? null }).then(r => r.data),
},
} }

View file

@ -1,130 +1,537 @@
/** /**
* Magpie Central Theme * Magpie Central Theme
* Theme-aware, responsive CSS classes. All components use these. * Mobile-first. Bottom nav on small screens, sidebar on 768px.
*/ */
:root { /* ---- Fonts ---- */
--color-bg: #0f1117; @import url('https://fonts.googleapis.com/css2?family=Space+Mono:wght@400;700&family=DM+Sans:ital,opsz,wght@0,9..40,300;0,9..40,400;0,9..40,500;0,9..40,600;1,9..40,300&display=swap');
--color-bg-secondary: #1a1d27;
--color-bg-card: #1e2130;
--color-border: #2d3148;
--color-text: #e2e8f0;
--color-text-muted: #8892a4;
--color-primary: #7c6af7;
--color-primary-dim: #3d3578;
--color-success: #34d399;
--color-warning: #fbbf24;
--color-danger: #f87171;
--color-info: #60a5fa;
:root {
/* Colors */
--color-bg: #0b0d14;
--color-bg-secondary: #13161f;
--color-bg-card: #181c28;
--color-bg-lift: #1e2235;
--color-border: #252a3d;
--color-border-soft: #1c2030;
--color-text: #dde4f0;
--color-text-muted: #6b7694;
--color-text-dim: #3d455e;
/* Brand accent — amber-gold for the "field ops" feel */
--color-primary: #e8a93a;
--color-primary-dim: rgba(232,169,58,0.12);
--color-primary-glow: rgba(232,169,58,0.25);
/* Semantic */
--color-success: #3ecf8e;
--color-warning: #f0b429;
--color-danger: #f26b6b;
--color-info: #5ba4f5;
/* Spacing */
--spacing-xs: 4px; --spacing-xs: 4px;
--spacing-sm: 8px; --spacing-sm: 8px;
--spacing-md: 16px; --spacing-md: 16px;
--spacing-lg: 24px; --spacing-lg: 24px;
--spacing-xl: 40px; --spacing-xl: 40px;
/* Radii */
--radius-sm: 4px; --radius-sm: 4px;
--radius-md: 8px; --radius-md: 8px;
--radius-lg: 12px; --radius-lg: 14px;
--radius-pill: 999px;
--font-mono: 'JetBrains Mono', 'Fira Code', monospace; /* Typography */
--font-sans: system-ui, -apple-system, sans-serif; --font-sans: 'DM Sans', system-ui, sans-serif;
--font-mono: 'Space Mono', 'JetBrains Mono', monospace;
/* Layout */
--sidebar-w: 220px;
--bottom-nav-h: 60px;
--touch-min: 44px;
} }
@media (prefers-color-scheme: light) { @media (prefers-color-scheme: light) {
:root { :root {
--color-bg: #f8f9fc; --color-bg: #f2f4f8;
--color-bg-secondary: #ffffff; --color-bg-secondary: #ffffff;
--color-bg-card: #ffffff; --color-bg-card: #ffffff;
--color-bg-lift: #f8f9fc;
--color-border: #dde1ec; --color-border: #dde1ec;
--color-text: #1a1d27; --color-border-soft: #eaecf4;
--color-text: #1a1d2e;
--color-text-muted: #6b7280; --color-text-muted: #6b7280;
--color-primary: #5b4fcf; --color-text-dim: #c0c5d4;
--color-primary-dim: #ede9ff; --color-primary: #c47b0a;
--color-primary-dim: rgba(196,123,10,0.10);
--color-primary-glow: rgba(196,123,10,0.20);
} }
} }
* { box-sizing: border-box; margin: 0; padding: 0; } /* ================================================================== */
/* Reset & Base */
/* ================================================================== */
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
html { -webkit-text-size-adjust: 100%; }
body { body {
font-family: var(--font-sans); font-family: var(--font-sans);
background: var(--color-bg); background: var(--color-bg);
color: var(--color-text); color: var(--color-text);
font-size: 14px; font-size: 14px;
line-height: 1.5; line-height: 1.55;
/* Subtle noise texture */
background-image:
url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='300' height='300'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.75' numOctaves='4' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='300' height='300' filter='url(%23n)' opacity='0.03'/%3E%3C/svg%3E");
} }
/* ---- Layout ---- */ :focus-visible {
.app-shell { display: flex; min-height: 100vh; } outline: 2px solid var(--color-primary);
.sidebar { width: 220px; background: var(--color-bg-secondary); border-right: 1px solid var(--color-border); padding: var(--spacing-lg) var(--spacing-md); flex-shrink: 0; } outline-offset: 2px;
.main-content { flex: 1; padding: var(--spacing-lg); overflow-y: auto; } border-radius: var(--radius-sm);
}
/* ---- Nav ---- */ /* ================================================================== */
.nav-brand { font-size: 18px; font-weight: 700; color: var(--color-primary); margin-bottom: var(--spacing-lg); display: flex; align-items: center; gap: var(--spacing-sm); } /* App Shell — mobile-first */
.nav-link { display: block; padding: var(--spacing-sm) var(--spacing-md); border-radius: var(--radius-md); color: var(--color-text-muted); text-decoration: none; margin-bottom: 2px; transition: background 0.15s, color 0.15s; } /* ================================================================== */
.nav-link:hover, .nav-link.active { background: var(--color-primary-dim); color: var(--color-primary); }
/* ---- Cards ---- */ .app-shell {
.card { background: var(--color-bg-card); border: 1px solid var(--color-border); border-radius: var(--radius-lg); padding: var(--spacing-md); } display: flex;
.card-title { font-size: 13px; font-weight: 600; color: var(--color-text-muted); text-transform: uppercase; letter-spacing: 0.06em; margin-bottom: var(--spacing-md); } flex-direction: column;
min-height: 100dvh; /* dvh accounts for mobile browser chrome */
min-height: 100vh; /* fallback */
}
/* ---- Buttons ---- */ /* Main content area — leaves room for bottom nav on mobile */
.btn { display: inline-flex; align-items: center; gap: var(--spacing-xs); padding: 6px var(--spacing-md); border-radius: var(--radius-md); border: none; cursor: pointer; font-size: 13px; font-weight: 500; transition: opacity 0.15s; } .main-content {
.btn:hover { opacity: 0.85; } flex: 1;
.btn:disabled { opacity: 0.4; cursor: not-allowed; } overflow-y: auto;
.btn-primary { background: var(--color-primary); color: #fff; } padding: var(--spacing-md);
padding-bottom: calc(var(--bottom-nav-h) + var(--spacing-md));
}
/* ---- Bottom Nav (mobile) ---- */
.bottom-nav {
position: fixed;
bottom: 0; left: 0; right: 0;
height: var(--bottom-nav-h);
background: var(--color-bg-secondary);
border-top: 1px solid var(--color-border);
display: flex;
align-items: stretch;
z-index: 50;
/* Safe area for notched phones */
padding-bottom: env(safe-area-inset-bottom);
}
.bottom-nav-item {
flex: 1;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 3px;
text-decoration: none;
color: var(--color-text-muted);
font-size: 10px;
font-family: var(--font-mono);
letter-spacing: 0.04em;
text-transform: uppercase;
transition: color 0.15s;
min-height: var(--touch-min);
-webkit-tap-highlight-color: transparent;
}
.bottom-nav-item .nav-icon {
font-size: 18px;
line-height: 1;
}
.bottom-nav-item.active,
.bottom-nav-item:active {
color: var(--color-primary);
}
.bottom-nav-item.active .nav-icon {
filter: drop-shadow(0 0 6px var(--color-primary-glow));
}
/* ---- Sidebar (desktop ≥768px) ---- */
.sidebar {
display: none;
}
@media (min-width: 768px) {
.app-shell {
flex-direction: row;
}
.main-content {
padding-bottom: var(--spacing-lg);
}
.bottom-nav {
display: none;
}
.sidebar {
display: flex;
flex-direction: column;
width: var(--sidebar-w);
background: var(--color-bg-secondary);
border-right: 1px solid var(--color-border);
padding: var(--spacing-lg) var(--spacing-md);
flex-shrink: 0;
position: sticky;
top: 0;
height: 100vh;
overflow-y: auto;
}
.nav-brand {
font-family: var(--font-mono);
font-size: 15px;
font-weight: 700;
color: var(--color-primary);
margin-bottom: var(--spacing-lg);
display: flex;
align-items: center;
gap: var(--spacing-sm);
letter-spacing: -0.01em;
}
.nav-link {
display: flex;
align-items: center;
gap: var(--spacing-sm);
padding: var(--spacing-sm) var(--spacing-md);
border-radius: var(--radius-md);
color: var(--color-text-muted);
text-decoration: none;
margin-bottom: 2px;
font-size: 13px;
transition: background 0.15s, color 0.15s;
}
.nav-link:hover { background: var(--color-primary-dim); color: var(--color-text); }
.nav-link.active { background: var(--color-primary-dim); color: var(--color-primary); font-weight: 500; }
}
/* ================================================================== */
/* Cards */
/* ================================================================== */
.card {
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: var(--radius-lg);
padding: var(--spacing-md);
}
.card-title {
font-family: var(--font-mono);
font-size: 10px;
font-weight: 700;
color: var(--color-text-dim);
text-transform: uppercase;
letter-spacing: 0.1em;
margin-bottom: var(--spacing-md);
}
/* ================================================================== */
/* Buttons — touch-safe */
/* ================================================================== */
.btn {
display: inline-flex;
align-items: center;
justify-content: center;
gap: var(--spacing-xs);
padding: 0 var(--spacing-md);
height: var(--touch-min);
border-radius: var(--radius-md);
border: none;
cursor: pointer;
font-family: var(--font-sans);
font-size: 13px;
font-weight: 500;
text-decoration: none;
transition: opacity 0.15s, transform 0.1s;
-webkit-tap-highlight-color: transparent;
white-space: nowrap;
}
.btn:active { transform: scale(0.97); }
.btn:disabled { opacity: 0.38; cursor: not-allowed; }
.btn-primary { background: var(--color-primary); color: #111; }
.btn-ghost { background: transparent; color: var(--color-text-muted); border: 1px solid var(--color-border); } .btn-ghost { background: transparent; color: var(--color-text-muted); border: 1px solid var(--color-border); }
.btn-danger { background: var(--color-danger); color: #fff; } .btn-danger { background: var(--color-danger); color: #fff; }
.btn-success { background: var(--color-success); color: #111; } .btn-success { background: var(--color-success); color: #111; }
.btn-sm { padding: 4px var(--spacing-sm); font-size: 12px; }
/* ---- Badges ---- */ .btn-sm {
.badge { display: inline-flex; align-items: center; padding: 2px 8px; border-radius: 999px; font-size: 11px; font-weight: 600; } height: 32px;
.badge-success { background: color-mix(in srgb, var(--color-success) 20%, transparent); color: var(--color-success); } padding: 0 var(--spacing-sm);
.badge-warning { background: color-mix(in srgb, var(--color-warning) 20%, transparent); color: var(--color-warning); } font-size: 12px;
.badge-danger { background: color-mix(in srgb, var(--color-danger) 20%, transparent); color: var(--color-danger); } border-radius: var(--radius-sm);
.badge-info { background: color-mix(in srgb, var(--color-info) 20%, transparent); color: var(--color-info); } }
.badge-muted { background: color-mix(in srgb, var(--color-text-muted) 15%, transparent); color: var(--color-text-muted); }
/* ================================================================== */
/* Badges */
/* ================================================================== */
.badge {
display: inline-flex;
align-items: center;
padding: 2px 8px;
border-radius: var(--radius-pill);
font-family: var(--font-mono);
font-size: 10px;
font-weight: 700;
letter-spacing: 0.04em;
text-transform: uppercase;
}
.badge-success { background: color-mix(in srgb, var(--color-success) 18%, transparent); color: var(--color-success); }
.badge-warning { background: color-mix(in srgb, var(--color-warning) 18%, transparent); color: var(--color-warning); }
.badge-danger { background: color-mix(in srgb, var(--color-danger) 18%, transparent); color: var(--color-danger); }
.badge-info { background: color-mix(in srgb, var(--color-info) 18%, transparent); color: var(--color-info); }
.badge-muted { background: color-mix(in srgb, var(--color-text-muted) 12%, transparent); color: var(--color-text-muted); }
/* ================================================================== */
/* Forms */
/* ================================================================== */
.form-label {
display: block;
font-size: 11px;
font-family: var(--font-mono);
font-weight: 700;
color: var(--color-text-dim);
text-transform: uppercase;
letter-spacing: 0.08em;
margin-bottom: 6px;
}
/* ---- Forms ---- */
.form-label { display: block; font-size: 12px; font-weight: 500; color: var(--color-text-muted); margin-bottom: var(--spacing-xs); }
.form-input, .form-select, .form-textarea { .form-input, .form-select, .form-textarea {
width: 100%; padding: 8px var(--spacing-sm); background: var(--color-bg-secondary); width: 100%;
border: 1px solid var(--color-border); border-radius: var(--radius-md); padding: 10px var(--spacing-sm);
color: var(--color-text); font-size: 13px; font-family: inherit; background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
border-radius: var(--radius-md);
color: var(--color-text);
font-size: 14px;
font-family: var(--font-sans);
/* Touch-safe height */
min-height: var(--touch-min);
-webkit-appearance: none;
appearance: none;
} }
.form-input:focus, .form-select:focus, .form-textarea:focus { .form-input:focus, .form-select:focus, .form-textarea:focus {
outline: none; border-color: var(--color-primary); outline: none;
box-shadow: 0 0 0 2px color-mix(in srgb, var(--color-primary) 25%, transparent); border-color: var(--color-primary);
box-shadow: 0 0 0 2px var(--color-primary-glow);
} }
.form-textarea { resize: vertical; min-height: 100px; font-family: var(--font-mono); font-size: 12px; }
.form-textarea {
resize: vertical;
min-height: 100px;
font-family: var(--font-mono);
font-size: 12px;
line-height: 1.6;
}
.form-group { margin-bottom: var(--spacing-md); } .form-group { margin-bottom: var(--spacing-md); }
/* ---- Table ---- */ /* ================================================================== */
.table { width: 100%; border-collapse: collapse; font-size: 13px; } /* Responsive Table → Card stack */
.table th { text-align: left; padding: var(--spacing-sm) var(--spacing-md); font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; color: var(--color-text-muted); border-bottom: 1px solid var(--color-border); } /* ================================================================== */
.table td { padding: var(--spacing-sm) var(--spacing-md); border-bottom: 1px solid var(--color-border); }
.table tr:last-child td { border-bottom: none; }
.table tr:hover td { background: color-mix(in srgb, var(--color-primary) 5%, transparent); }
/* ---- Status dots ---- */ .table {
.status-dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; } width: 100%;
.status-dot.success { background: var(--color-success); } border-collapse: collapse;
font-size: 13px;
}
.table th {
text-align: left;
padding: var(--spacing-sm) var(--spacing-md);
font-family: var(--font-mono);
font-size: 10px;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.08em;
color: var(--color-text-dim);
border-bottom: 1px solid var(--color-border);
}
.table td {
padding: var(--spacing-sm) var(--spacing-md);
border-bottom: 1px solid var(--color-border-soft);
vertical-align: middle;
}
.table tr:last-child td { border-bottom: none; }
.table tr:hover td {
background: color-mix(in srgb, var(--color-primary) 4%, transparent);
}
/* Mobile: collapse table rows into stacked cards */
@media (max-width: 640px) {
.table-responsive thead { display: none; }
.table-responsive tbody tr {
display: block;
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: var(--radius-lg);
margin-bottom: var(--spacing-sm);
padding: var(--spacing-sm) 0;
}
.table-responsive tbody tr:hover { background: var(--color-bg-lift); }
.table-responsive td {
display: flex;
align-items: center;
justify-content: space-between;
padding: 6px var(--spacing-md);
border-bottom: none;
}
.table-responsive td::before {
content: attr(data-label);
font-family: var(--font-mono);
font-size: 10px;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.08em;
color: var(--color-text-dim);
flex-shrink: 0;
margin-right: var(--spacing-sm);
}
/* Hide empty/action cells label */
.table-responsive td[data-label=""] { justify-content: flex-end; }
.table-responsive td[data-label=""]::before { display: none; }
}
/* ================================================================== */
/* Status indicators */
/* ================================================================== */
.status-dot {
display: inline-block;
width: 7px; height: 7px;
border-radius: 50%;
margin-right: 6px;
flex-shrink: 0;
}
.status-dot.success { background: var(--color-success); box-shadow: 0 0 6px color-mix(in srgb, var(--color-success) 60%, transparent); }
.status-dot.failed { background: var(--color-danger); } .status-dot.failed { background: var(--color-danger); }
.status-dot.pending { background: var(--color-warning); } .status-dot.pending { background: var(--color-warning); }
.status-dot.skipped { background: var(--color-text-muted); } .status-dot.skipped { background: var(--color-text-muted); }
.status-dot.running { background: var(--color-info); animation: pulse 1s infinite; } .status-dot.running { background: var(--color-info); animation: pulse 1s ease-in-out infinite; }
@keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } } @keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.3; } }
/* ---- Page header ---- */ /* ================================================================== */
.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: var(--spacing-lg); } /* Page header */
.page-title { font-size: 20px; font-weight: 700; } /* ================================================================== */
/* ---- Empty state ---- */ .page-header {
.empty-state { text-align: center; padding: var(--spacing-xl); color: var(--color-text-muted); } display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: var(--spacing-lg);
gap: var(--spacing-sm);
}
/* ---- SR only ---- */ .page-title {
.sr-only { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0,0,0,0); white-space: nowrap; } font-family: var(--font-mono);
font-size: 16px;
font-weight: 700;
letter-spacing: -0.02em;
}
:focus-visible { outline: 2px solid var(--color-primary); outline-offset: 2px; border-radius: var(--radius-sm); } /* ================================================================== */
/* Empty state */
/* ================================================================== */
.empty-state {
text-align: center;
padding: var(--spacing-xl);
color: var(--color-text-muted);
font-size: 13px;
}
/* ================================================================== */
/* Modal / bottom sheet */
/* ================================================================== */
.modal-backdrop {
position: fixed; inset: 0;
background: rgba(0, 0, 0, 0.7);
display: flex;
align-items: flex-end; /* bottom sheet on mobile */
justify-content: center;
z-index: 100;
padding: 0;
}
.modal {
background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
border-radius: var(--radius-lg) var(--radius-lg) 0 0;
padding: var(--spacing-lg);
width: 100%;
max-height: 92dvh;
max-height: 92vh;
overflow-y: auto;
}
/* Desktop: centered dialog */
@media (min-width: 640px) {
.modal-backdrop { align-items: center; padding: var(--spacing-md); }
.modal { border-radius: var(--radius-lg); max-width: 520px; width: 100%; }
.modal.modal-wide { max-width: 680px; }
}
/* Drag handle visual cue on mobile */
.modal::before {
content: '';
display: block;
width: 36px; height: 4px;
background: var(--color-border);
border-radius: var(--radius-pill);
margin: 0 auto var(--spacing-md);
}
@media (min-width: 640px) {
.modal::before { display: none; }
}
/* ================================================================== */
/* Utilities */
/* ================================================================== */
.sr-only {
position: absolute; width: 1px; height: 1px;
overflow: hidden; clip: rect(0,0,0,0); white-space: nowrap;
}
.text-mono { font-family: var(--font-mono); }
.text-muted { color: var(--color-text-muted); }
.text-sm { font-size: 12px; }
.flex { display: flex; }
.flex-center { display: flex; align-items: center; }
.gap-sm { gap: var(--spacing-sm); }
.gap-md { gap: var(--spacing-md); }
.ml-auto { margin-left: auto; }

326
manage.sh
View file

@ -1,74 +1,298 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Magpie management script # Magpie management script — native dev process manager
set -euo pipefail set -euo pipefail
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$REPO_DIR" cd "$REPO_DIR"
# ------------------------------------------------------------------ #
# Config
# ------------------------------------------------------------------ #
API_PORT=8532 API_PORT=8532
WEB_PORT=8531 WEB_PORT=8531
COMPOSE="docker compose" CONDA_ENV=cf
DATA_DIR="${HOME}/.local/share/magpie"
LOG_DIR="${DATA_DIR}/logs"
PID_API="${DATA_DIR}/api.pid"
PID_WEB="${DATA_DIR}/web.pid"
LOG_API="${LOG_DIR}/api.log"
LOG_WEB="${LOG_DIR}/web.log"
mkdir -p "$DATA_DIR" "$LOG_DIR"
# ------------------------------------------------------------------ #
# Colors
# ------------------------------------------------------------------ #
GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; CYAN='\033[0;36m'; RESET='\033[0m'
ok() { echo -e "${GREEN}${RESET} $*"; }
warn() { echo -e "${YELLOW}${RESET} $*"; }
err() { echo -e "${RED}${RESET} $*"; }
info() { echo -e "${CYAN}${RESET} $*"; }
# ------------------------------------------------------------------ #
# Process helpers
# ------------------------------------------------------------------ #
_pid_alive() {
local pid_file="$1"
[[ -f "$pid_file" ]] && kill -0 "$(cat "$pid_file")" 2>/dev/null
}
_port_open() {
ss -tlnp 2>/dev/null | grep -q ":$1 " || \
lsof -ti:"$1" >/dev/null 2>&1
}
_start_api() {
if _pid_alive "$PID_API"; then
warn "API already running (PID $(cat "$PID_API"))"
return
fi
if _port_open "$API_PORT"; then
warn "Port :${API_PORT} already in use by another process — stop it first or change API_PORT"
return
fi
info "Starting API on :${API_PORT}..."
conda run --no-capture-output -n "$CONDA_ENV" \
uvicorn app.main:app \
--host 0.0.0.0 --port "$API_PORT" \
--reload --reload-dir app \
>> "$LOG_API" 2>&1 &
echo $! > "$PID_API"
# Wait for port to open (up to 10s)
local i=0
while ! _port_open "$API_PORT" && (( i++ < 20 )); do sleep 0.5; done
if _port_open "$API_PORT"; then
ok "API up → http://localhost:${API_PORT}/docs"
else
warn "API started (PID $(cat "$PID_API")) but port not open yet — check: ./manage.sh logs api"
fi
}
_start_web() {
if _pid_alive "$PID_WEB"; then
warn "Web already running (PID $(cat "$PID_WEB"))"
return
fi
if _port_open "$WEB_PORT"; then
warn "Port :${WEB_PORT} already in use by another process — stop it first or change WEB_PORT"
return
fi
info "Starting web on :${WEB_PORT}..."
cd frontend
npm run dev >> "$LOG_WEB" 2>&1 &
echo $! > "$PID_WEB"
cd "$REPO_DIR"
local i=0
while ! _port_open "$WEB_PORT" && (( i++ < 20 )); do sleep 0.5; done
if _port_open "$WEB_PORT"; then
ok "Web up → http://localhost:${WEB_PORT}"
else
warn "Web started (PID $(cat "$PID_WEB")) but port not open yet — check: ./manage.sh logs web"
fi
}
_stop_service() {
local name="$1" pid_file="$2"
if _pid_alive "$pid_file"; then
local pid
pid=$(cat "$pid_file")
info "Stopping ${name} (PID ${pid})..."
kill "$pid" 2>/dev/null || true
# Give it up to 5s to exit
local i=0
while _pid_alive "$pid_file" && (( i++ < 10 )); do sleep 0.5; done
if _pid_alive "$pid_file"; then
warn "${name} didn't stop cleanly — sending SIGKILL"
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
ok "${name} stopped"
else
rm -f "$pid_file"
info "${name} was not running"
fi
}
# ------------------------------------------------------------------ #
# Commands
# ------------------------------------------------------------------ #
cmd="${1:-help}" cmd="${1:-help}"
shift || true
case "$cmd" in case "$cmd" in
start) start)
echo "Starting Magpie (API :$API_PORT, Web :$WEB_PORT)..." _start_api
$COMPOSE up -d _start_web
;; ;;
stop) stop)
$COMPOSE stop _stop_service "API" "$PID_API"
_stop_service "Web" "$PID_WEB"
;; ;;
restart) restart)
$COMPOSE restart _stop_service "API" "$PID_API"
_stop_service "Web" "$PID_WEB"
_start_api
_start_web
;; ;;
restart-api)
_stop_service "API" "$PID_API"
_start_api
;;
restart-web)
_stop_service "Web" "$PID_WEB"
_start_web
;;
status) status)
$COMPOSE ps
;;
logs)
service="${2:-}"
if [[ -n "$service" ]]; then
$COMPOSE logs -f "$service"
else
$COMPOSE logs -f
fi
;;
build)
$COMPOSE build
;;
open)
xdg-open "http://localhost:$WEB_PORT" 2>/dev/null || open "http://localhost:$WEB_PORT"
;;
dev-api)
echo "Starting API in dev mode (host network, port $API_PORT)..."
conda run -n cf uvicorn app.main:app --host 0.0.0.0 --port $API_PORT --reload
;;
dev-web)
echo "Starting frontend dev server (port $WEB_PORT)..."
cd frontend && npm run dev
;;
login)
echo "Refreshing Reddit session..."
conda run -n cf xvfb-run --auto-servernum python -m app.services.reddit.post --login
;;
migrate)
echo "Running DB migrations..."
conda run -n cf python -c "from app.db.store import Store; from app.core.config import get_settings; s=Store(get_settings().db_path); s.run_migrations(); s.close(); print('Done.')"
;;
help|*)
echo "Usage: ./manage.sh <command>"
echo "" echo ""
echo "Commands:" echo -e " ${CYAN}Magpie — Service Status${RESET}"
echo " start Start all services via Docker Compose" echo " ─────────────────────────────────────"
echo " stop Stop all services"
echo " restart Restart all services" # API
echo " status Show service status" if _pid_alive "$PID_API"; then
echo " logs [svc] Tail logs (optionally for one service)" local_pid=$(cat "$PID_API")
echo " build Build Docker images" if _port_open "$API_PORT"; then
echo " open Open browser to dashboard" ok " API :${API_PORT} (PID ${local_pid})"
echo " dev-api Run API in dev mode (conda, hot-reload)" else
echo " dev-web Run frontend dev server" warn " API PID ${local_pid} alive but port :${API_PORT} not open"
echo " login Refresh Reddit Playwright session" fi
else
err " API stopped"
fi
# Web
if _pid_alive "$PID_WEB"; then
local_pid=$(cat "$PID_WEB")
if _port_open "$WEB_PORT"; then
ok " Web :${WEB_PORT} (PID ${local_pid})"
else
warn " Web PID ${local_pid} alive but port :${WEB_PORT} not open"
fi
else
err " Web stopped"
fi
# Scheduler (query API)
if _port_open "$API_PORT"; then
sched=$(curl -sf "http://localhost:${API_PORT}/api/v1/scheduler/status" \
| python3 -c "import json,sys; d=json.load(sys.stdin); print(f\"{d.get('job_count',0)} job(s)\")" 2>/dev/null || echo "?")
info " Scheduler ${sched} scheduled"
fi
# Session age
SESSION_FILE="${DATA_DIR}/session.json"
if [[ -f "$SESSION_FILE" ]]; then
age_h=$(python3 -c "import time,os; print(f\"{(time.time()-os.path.getmtime('$SESSION_FILE'))/3600:.1f}h\")" 2>/dev/null || echo "?")
if python3 -c "import time,os; exit(0 if (time.time()-os.path.getmtime('$SESSION_FILE'))/3600 < 12 else 1)" 2>/dev/null; then
ok " Session ${age_h} old (valid)"
else
warn " Session ${age_h} old (stale — run: ./manage.sh login)"
fi
else
warn " Session no session.json — run: ./manage.sh login"
fi
echo ""
;;
logs)
target="${1:-all}"
case "$target" in
api) tail -f "$LOG_API" ;;
web) tail -f "$LOG_WEB" ;;
all|*)
echo "=== API log ===" && tail -f "$LOG_API" &
echo "=== Web log ===" && tail -f "$LOG_WEB" &
wait
;;
esac
;;
open)
xdg-open "http://localhost:${WEB_PORT}" 2>/dev/null \
|| open "http://localhost:${WEB_PORT}" 2>/dev/null \
|| info "Open http://localhost:${WEB_PORT} in your browser"
;;
update)
info "Pulling latest changes..."
git pull
info "Installing Python deps..."
conda run -n "$CONDA_ENV" pip install -q -e .
info "Installing frontend deps..."
cd frontend && npm install --silent && cd "$REPO_DIR"
# Restart API to pick up code changes; web hot-reloads itself
if _pid_alive "$PID_API"; then
info "Restarting API to pick up changes..."
_stop_service "API" "$PID_API"
_start_api
else
info "API not running — start with: ./manage.sh start"
fi
ok "Update complete"
;;
login)
info "Refreshing Reddit session (opens browser via Xvfb)..."
REDDIT_SESSION_FILE="${DATA_DIR}/session.json" \
conda run --no-capture-output -n "$CONDA_ENV" \
xvfb-run --auto-servernum \
python -m app.services.reddit.post --login
;;
migrate)
info "Running DB migrations..."
conda run -n "$CONDA_ENV" python - <<'EOF'
from app.db.store import Store
from app.core.config import get_settings
s = Store(get_settings().db_path)
s.run_migrations()
s.close()
print("Migrations applied.")
EOF
;;
seed)
info "Seeding campaigns from legacy reddit-poster scripts..."
conda run -n "$CONDA_ENV" python scripts/seed_campaigns.py
;;
help|*)
echo ""
echo -e " ${CYAN}Usage: ./manage.sh <command> [args]${RESET}"
echo ""
echo " Process control:"
echo " start Start API and web dev server in background"
echo " stop Stop both"
echo " restart Full restart"
echo " restart-api Restart API only (e.g. after code changes)"
echo " restart-web Restart web dev server only"
echo " status Show process status, scheduler jobs, session age"
echo ""
echo " Logs:"
echo " logs Tail all logs"
echo " logs api Tail API log only"
echo " logs web Tail web log only"
echo " Logs at: ${LOG_DIR}/"
echo ""
echo " Maintenance:"
echo " update git pull + pip/npm install + API restart"
echo " migrate Run DB migrations standalone" echo " migrate Run DB migrations standalone"
echo " seed Seed campaigns from legacy scripts"
echo " login Refresh Reddit Playwright session"
echo " open Open dashboard in browser"
echo ""
;; ;;
esac esac