diff --git a/.gitignore b/.gitignore index 3018155..c770bb9 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ data/ # CLAUDE.md — gitignored per BSL 1.1 + docs-location policy CLAUDE.md +frontend/.vite diff --git a/app/api/endpoints/signals.py b/app/api/endpoints/signals.py new file mode 100644 index 0000000..f7e350e --- /dev/null +++ b/app/api/endpoints/signals.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field + +from app.core.config import get_settings +from app.db.store import Store + +router = APIRouter() + + +def get_store() -> Store: + s = get_settings() + return Store(s.db_path) + + +# ------------------------------------------------------------------ # +# Schemas +# ------------------------------------------------------------------ # + +class SignalRuleCreate(BaseModel): + name: str + platform: str = "reddit" + sub: str | None = None + keywords: list[str] = Field(default_factory=list) + match_mode: str = "any" # any | all | regex + min_score: int = 0 + label: str | None = None # pain-point | feedback | mention | trust + notes: str | None = None + + +class SignalRuleUpdate(BaseModel): + name: str | None = None + sub: str | None = None + keywords: list[str] | None = None + match_mode: str | None = None + min_score: int | None = None + label: str | None = None + active: bool | None = None + notes: str | None = None + + +class SignalStatusUpdate(BaseModel): + status: str # new | saved | dismissed + notes: str | None = None + + +def _decode_json_fields(row: dict[str, Any]) -> dict[str, Any]: + """Decode JSON-encoded list fields returned as strings from SQLite.""" + for col in ("keywords", "matched_keywords"): + if col in row and isinstance(row[col], str): + try: + row[col] = json.loads(row[col]) + except (json.JSONDecodeError, TypeError): + row[col] = [] + return row + + +# ------------------------------------------------------------------ # +# Signal rules +# ------------------------------------------------------------------ # + +@router.get("/signal-rules") +def list_signal_rules(active_only: bool = False, store: Store = Depends(get_store)): + return [_decode_json_fields(r) for r in store.list_signal_rules(active_only=active_only)] + + +@router.post("/signal-rules", status_code=201) +def create_signal_rule(body: SignalRuleCreate, store: Store = Depends(get_store)): + rule = store.create_signal_rule( + name=body.name, + platform=body.platform, + sub=body.sub, + keywords=body.keywords, + match_mode=body.match_mode, + min_score=body.min_score, + label=body.label, + notes=body.notes, + ) + return _decode_json_fields(rule) + + +@router.get("/signal-rules/{rule_id}") +def get_signal_rule(rule_id: int, store: Store = Depends(get_store)): + rule = store.get_signal_rule(rule_id) + if not rule: + raise HTTPException(status_code=404, detail="Signal rule not found") + return _decode_json_fields(rule) + + +@router.patch("/signal-rules/{rule_id}") +def update_signal_rule(rule_id: int, body: SignalRuleUpdate, store: Store = Depends(get_store)): + rule = store.get_signal_rule(rule_id) + if not rule: + raise HTTPException(status_code=404, detail="Signal rule not found") + updates = body.model_dump(exclude_none=True) + updated = store.update_signal_rule(rule_id, **updates) + return _decode_json_fields(updated) + + +@router.delete("/signal-rules/{rule_id}", status_code=204) +def delete_signal_rule(rule_id: int, store: Store = Depends(get_store)): + if not store.delete_signal_rule(rule_id): + raise HTTPException(status_code=404, detail="Signal rule not found") + + +# ------------------------------------------------------------------ # +# Signals queue — fixed-path routes MUST precede /{signal_id} routes +# (FastAPI matches in registration order; literal paths win over params) +# ------------------------------------------------------------------ # + +@router.get("/signals") +def list_signals( + status: str | None = None, + platform: str | None = None, + sub: str | None = None, + limit: int = 100, + store: Store = Depends(get_store), +): + rows = store.list_signals(status=status, platform=platform, sub=sub, limit=limit) + return [_decode_json_fields(r) for r in rows] + + +@router.post("/signals/scrape") +async def trigger_scrape(): + """Manually trigger a full signal scrape pass. Useful for testing rules.""" + from app.services.scraper import scrape_signals + summary = await scrape_signals() + return summary + + +@router.get("/signals/scrape-state") +def get_scrape_state(store: Store = Depends(get_store)): + """Return per-sub cursor state (last scraped, posts seen, signals found).""" + return store.list_scrape_states() + + +@router.get("/signals/{signal_id}") +def get_signal(signal_id: int, store: Store = Depends(get_store)): + signal = store.get_signal(signal_id) + if not signal: + raise HTTPException(status_code=404, detail="Signal not found") + result = _decode_json_fields(signal) + result["matched_rules"] = store.get_signal_rule_matches(signal_id) + return result + + +@router.patch("/signals/{signal_id}/status") +def update_signal_status(signal_id: int, body: SignalStatusUpdate, store: Store = Depends(get_store)): + allowed = {"new", "saved", "dismissed"} + if body.status not in allowed: + raise HTTPException(status_code=422, detail=f"status must be one of {allowed}") + signal = store.update_signal_status(signal_id, body.status, body.notes) + if not signal: + raise HTTPException(status_code=404, detail="Signal not found") + return _decode_json_fields(signal) diff --git a/app/api/routes.py b/app/api/routes.py index 53de211..5de40f1 100644 --- a/app/api/routes.py +++ b/app/api/routes.py @@ -1,6 +1,6 @@ from fastapi import FastAPI -from app.api.endpoints import campaigns, opportunities, posts, scheduler, subs +from app.api.endpoints import campaigns, opportunities, posts, scheduler, signals, subs def register_routes(app: FastAPI) -> None: @@ -9,3 +9,4 @@ def register_routes(app: FastAPI) -> None: app.include_router(subs.router, prefix="/api/v1") app.include_router(scheduler.router, prefix="/api/v1") app.include_router(opportunities.router, prefix="/api/v1") + app.include_router(signals.router, prefix="/api/v1") diff --git a/app/core/config.py b/app/core/config.py index 52d9cd9..2586e11 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -21,6 +21,13 @@ class Settings(BaseSettings): # Scheduler scheduler_enabled: bool = True + # Signal scraper + scraper_enabled: bool = True + scraper_interval_mins: int = 30 # how often to poll (per full pass of all subs) + scraper_request_delay_secs: float = 2.0 # pause between sub requests to respect rate limits + scraper_fetch_limit: int = 25 # posts to fetch per sub per run (max 100) + scraper_user_agent: str = "Magpie/0.1 signal-monitor (by CircuitForge)" + def get_settings() -> Settings: return Settings() diff --git a/app/db/migrations/010_signal_rules.sql b/app/db/migrations/010_signal_rules.sql new file mode 100644 index 0000000..cfe6aeb --- /dev/null +++ b/app/db/migrations/010_signal_rules.sql @@ -0,0 +1,17 @@ +-- Signal monitoring rules: what to watch for across communities +CREATE TABLE IF NOT EXISTS signal_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, -- human label ("CF pain points", "Kiwi mentions") + platform TEXT NOT NULL DEFAULT 'reddit', + sub TEXT, -- NULL = apply to all monitored subs + keywords TEXT NOT NULL DEFAULT '[]', -- JSON array of strings + match_mode TEXT NOT NULL DEFAULT 'any', -- 'any' | 'all' | 'regex' + min_score INTEGER NOT NULL DEFAULT 0, -- ignore posts below this karma score + label TEXT, -- signal category: pain-point|feedback|mention|trust + active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + notes TEXT +); + +CREATE INDEX IF NOT EXISTS idx_signal_rules_platform_sub ON signal_rules(platform, sub); +CREATE INDEX IF NOT EXISTS idx_signal_rules_active ON signal_rules(active); diff --git a/app/db/migrations/011_signals.sql b/app/db/migrations/011_signals.sql new file mode 100644 index 0000000..7e85865 --- /dev/null +++ b/app/db/migrations/011_signals.sql @@ -0,0 +1,32 @@ +-- Surfaced content instances from signal monitoring +CREATE TABLE IF NOT EXISTS signals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + platform TEXT NOT NULL DEFAULT 'reddit', + sub TEXT NOT NULL, + post_id TEXT NOT NULL, -- platform-native ID (reddit: t3_xxxxx) + title TEXT NOT NULL, + body_snippet TEXT, -- first ~500 chars + score INTEGER, + comment_count INTEGER, + author TEXT, + url TEXT, + posted_at TEXT, -- original post timestamp + surfaced_at TEXT NOT NULL DEFAULT (datetime('now')), + matched_keywords TEXT NOT NULL DEFAULT '[]', -- JSON array of matched terms + status TEXT NOT NULL DEFAULT 'new',-- new|saved|dismissed + notes TEXT, + UNIQUE(platform, post_id) -- deduplicate across rule runs +); + +-- Junction table: which rules matched each signal (many-to-many) +CREATE TABLE IF NOT EXISTS signal_rule_matches ( + signal_id INTEGER NOT NULL REFERENCES signals(id) ON DELETE CASCADE, + rule_id INTEGER NOT NULL REFERENCES signal_rules(id) ON DELETE CASCADE, + matched_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (signal_id, rule_id) +); + +CREATE INDEX IF NOT EXISTS idx_signals_status ON signals(status); +CREATE INDEX IF NOT EXISTS idx_signals_platform_sub ON signals(platform, sub); +CREATE INDEX IF NOT EXISTS idx_signals_surfaced_at ON signals(surfaced_at DESC); +CREATE INDEX IF NOT EXISTS idx_signal_rule_matches_rule ON signal_rule_matches(rule_id); diff --git a/app/db/migrations/012_signal_scrape_state.sql b/app/db/migrations/012_signal_scrape_state.sql new file mode 100644 index 0000000..5dc9075 --- /dev/null +++ b/app/db/migrations/012_signal_scrape_state.sql @@ -0,0 +1,11 @@ +-- Per-sub cursor state for the signal scraper. +-- Stores the newest post fullname seen so subsequent runs only fetch newer posts. +CREATE TABLE IF NOT EXISTS signal_scrape_state ( + platform TEXT NOT NULL DEFAULT 'reddit', + sub TEXT NOT NULL, + last_fullname TEXT, -- e.g. t3_abc123; NULL = never scraped + last_scraped_at TEXT, + posts_seen INTEGER NOT NULL DEFAULT 0, + signals_found INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (platform, sub) +); diff --git a/app/db/store.py b/app/db/store.py index 09e6c0b..f5c0b1a 100644 --- a/app/db/store.py +++ b/app/db/store.py @@ -353,3 +353,175 @@ class Store: def dismiss_opportunity(self, opportunity_id: int, note: str | None = None) -> dict | None: return self.update_opportunity(opportunity_id, status="dismissed", dismiss_note=note) + + # ------------------------------------------------------------------ # + # Signal rules + # ------------------------------------------------------------------ # + + def list_signal_rules(self, active_only: bool = False) -> list[dict]: + where = "WHERE active = 1" if active_only else "" + return self._fetchall(f"SELECT * FROM signal_rules {where} ORDER BY platform, name") + + def get_signal_rule(self, rule_id: int) -> dict | None: + return self._fetchone("SELECT * FROM signal_rules WHERE id = ?", (rule_id,)) + + def create_signal_rule( + self, + name: str, + platform: str = "reddit", + sub: str | None = None, + keywords: list[str] | None = None, + match_mode: str = "any", + min_score: int = 0, + label: str | None = None, + notes: str | None = None, + ) -> dict: + return self._insert_returning( + "INSERT INTO signal_rules (name, platform, sub, keywords, match_mode, min_score, label, notes)" + " VALUES (?,?,?,?,?,?,?,?) RETURNING *", + (name, platform, sub, json.dumps(keywords or []), match_mode, min_score, label, notes), + ) + + def update_signal_rule(self, rule_id: int, **fields) -> dict | None: + allowed = {"name", "sub", "keywords", "match_mode", "min_score", "label", "active", "notes"} + updates = {} + for k, v in fields.items(): + if k not in allowed: + continue + updates[k] = json.dumps(v) if k == "keywords" else v + if not updates: + return self.get_signal_rule(rule_id) + set_clause = ", ".join(f"{k} = ?" for k in updates) + params = tuple(updates.values()) + (rule_id,) + self.conn.execute(f"UPDATE signal_rules SET {set_clause} WHERE id = ?", params) + self.conn.commit() + return self.get_signal_rule(rule_id) + + def delete_signal_rule(self, rule_id: int) -> bool: + cur = self.conn.execute("DELETE FROM signal_rules WHERE id = ?", (rule_id,)) + self.conn.commit() + return cur.rowcount > 0 + + # ------------------------------------------------------------------ # + # Signals + # ------------------------------------------------------------------ # + + def list_signals( + self, + status: str | None = None, + platform: str | None = None, + sub: str | None = None, + limit: int = 100, + ) -> list[dict]: + clauses, params = [], [] + if status: + clauses.append("status = ?") + params.append(status) + if platform: + clauses.append("platform = ?") + params.append(platform) + if sub: + clauses.append("sub = ?") + params.append(sub) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(limit) + return self._fetchall( + f"SELECT * FROM signals {where} ORDER BY surfaced_at DESC LIMIT ?", tuple(params) + ) + + def get_signal(self, signal_id: int) -> dict | None: + return self._fetchone("SELECT * FROM signals WHERE id = ?", (signal_id,)) + + def upsert_signal( + self, + platform: str, + sub: str, + post_id: str, + title: str, + body_snippet: str | None = None, + score: int | None = None, + comment_count: int | None = None, + author: str | None = None, + url: str | None = None, + posted_at: str | None = None, + matched_keywords: list[str] | None = None, + ) -> dict: + """Insert or ignore (dedup on platform+post_id). Returns the row either way.""" + self.conn.execute( + "INSERT OR IGNORE INTO signals" + " (platform, sub, post_id, title, body_snippet, score, comment_count," + " author, url, posted_at, matched_keywords)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (platform, sub, post_id, title, body_snippet, score, comment_count, + author, url, posted_at, json.dumps(matched_keywords or [])), + ) + self.conn.commit() + return self._fetchone( + "SELECT * FROM signals WHERE platform = ? AND post_id = ?", (platform, post_id) + ) + + def record_signal_rule_match(self, signal_id: int, rule_id: int) -> None: + """Record that a rule matched a signal (many-to-many, ignore duplicates).""" + self.conn.execute( + "INSERT OR IGNORE INTO signal_rule_matches (signal_id, rule_id) VALUES (?,?)", + (signal_id, rule_id), + ) + self.conn.commit() + + def update_signal_status( + self, signal_id: int, status: str, notes: str | None = None + ) -> dict | None: + self.conn.execute( + "UPDATE signals SET status = ?, notes = COALESCE(?, notes) WHERE id = ?", + (status, notes, signal_id), + ) + self.conn.commit() + return self.get_signal(signal_id) + + def get_signal_rule_matches(self, signal_id: int) -> list[dict]: + """Return all rules that matched a given signal.""" + return self._fetchall( + "SELECT sr.* FROM signal_rules sr" + " JOIN signal_rule_matches srm ON sr.id = srm.rule_id" + " WHERE srm.signal_id = ?", + (signal_id,), + ) + + # ------------------------------------------------------------------ # + # Signal scrape state (per-sub cursor tracking) + # ------------------------------------------------------------------ # + + def get_scrape_state(self, sub: str, platform: str = "reddit") -> dict | None: + return self._fetchone( + "SELECT * FROM signal_scrape_state WHERE platform = ? AND sub = ?", + (platform, sub), + ) + + def update_scrape_state( + self, + sub: str, + platform: str = "reddit", + last_fullname: str | None = None, + posts_seen_delta: int = 0, + signals_found_delta: int = 0, + ) -> None: + self.conn.execute( + """ + INSERT INTO signal_scrape_state + (platform, sub, last_fullname, last_scraped_at, posts_seen, signals_found) + VALUES (?, ?, ?, datetime('now'), ?, ?) + ON CONFLICT(platform, sub) DO UPDATE SET + last_fullname = excluded.last_fullname, + last_scraped_at = excluded.last_scraped_at, + posts_seen = signal_scrape_state.posts_seen + excluded.posts_seen, + signals_found = signal_scrape_state.signals_found + excluded.signals_found + """, + (platform, sub, last_fullname, posts_seen_delta, signals_found_delta), + ) + self.conn.commit() + + def list_scrape_states(self, platform: str = "reddit") -> list[dict]: + return self._fetchall( + "SELECT * FROM signal_scrape_state WHERE platform = ? ORDER BY sub", + (platform,), + ) diff --git a/app/main.py b/app/main.py index 912ee30..4e3ac40 100644 --- a/app/main.py +++ b/app/main.py @@ -9,7 +9,10 @@ from fastapi.middleware.cors import CORSMiddleware from app.api.routes import register_routes from app.core.config import get_settings from app.db.store import Store -from app.services.scheduler import start_scheduler, stop_scheduler, sync_all_campaigns +from app.services.scheduler import ( + start_scheduler, stop_scheduler, sync_all_campaigns, + start_scraper_job, +) logger = logging.getLogger(__name__) @@ -33,6 +36,14 @@ async def lifespan(app: FastAPI): app.state.scheduler = None logger.info("Magpie started — scheduler disabled") + # Start signal scraper job + if settings.scraper_enabled: + if not settings.scheduler_enabled: + # Scraper needs the scheduler even if campaign scheduling is off + start_scheduler() + start_scraper_job(interval_mins=settings.scraper_interval_mins) + logger.info("Signal scraper scheduled every %d min", settings.scraper_interval_mins) + store.close() yield diff --git a/app/services/lemmy/__init__.py b/app/services/lemmy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/lemmy/client.py b/app/services/lemmy/client.py new file mode 100644 index 0000000..d27ff28 --- /dev/null +++ b/app/services/lemmy/client.py @@ -0,0 +1,122 @@ +""" +Lemmy API v3 client — read-only, no auth required for public communities. + +Community addressing convention: @ + e.g. "selfhosted@lemmy.world", "technology@reddthat.com" + +API reference: https://{instance}/api/v3/ +Key endpoints used: + GET /api/v3/post/list?community_name=&sort=New&limit=&page=

+ GET /api/v3/community?name= (validation / metadata) +""" +from __future__ import annotations + +import logging +from typing import Any + +import httpx + +logger = logging.getLogger(__name__) + +# Instances known to be reliably federated and publicly accessible +KNOWN_INSTANCES = frozenset([ + "lemmy.world", + "lemmy.ml", + "beehaw.org", + "reddthat.com", + "sh.itjust.works", + "feddit.de", + "aussie.zone", + "lemmy.ca", +]) + +_DEFAULT_TIMEOUT = 15.0 +_DEFAULT_USER_AGENT = "Magpie/0.1 signal-monitor (by CircuitForge)" + + +def parse_community_target(sub: str) -> tuple[str, str]: + """ + Parse a 'community@instance' string into (community_name, instance_host). + + Raises ValueError if the format is invalid. + """ + if "@" not in sub: + raise ValueError( + f"Lemmy community must be in 'community@instance' format, got: {sub!r}" + ) + community, instance = sub.rsplit("@", 1) + if not community or not instance: + raise ValueError(f"Invalid Lemmy community target: {sub!r}") + return community.strip(), instance.strip().lower() + + +async def fetch_new_posts( + community: str, + instance: str, + last_seen_id: int | None, + limit: int = 50, + user_agent: str = _DEFAULT_USER_AGENT, +) -> tuple[list[dict[str, Any]], int | None]: + """ + Fetch new posts from a Lemmy community. + + Fetches the first page sorted by New, then filters to posts with + id > last_seen_id (client-side cursor — Lemmy has no 'before' param). + + Returns: + posts — filtered list of post dicts, newest first + new_max_id — highest post ID seen this run (use as next cursor), or None + """ + url = f"https://{instance}/api/v3/post/list" + params = { + "community_name": community, + "sort": "New", + "limit": min(limit, 50), # Lemmy max is 50 per page + "type_": "Local", # stay within the instance's local community + } + + async with httpx.AsyncClient( + headers={"User-Agent": user_agent}, + follow_redirects=True, + timeout=_DEFAULT_TIMEOUT, + ) as client: + resp = await client.get(url, params=params) + resp.raise_for_status() + data = resp.json() + + post_views = data.get("posts", []) + + # Normalize to flat dicts and apply cursor filter + posts = [] + new_max_id: int | None = None + + for pv in post_views: + post = pv.get("post", {}) + counts = pv.get("counts", {}) + creator = pv.get("creator", {}) + + post_id: int = post.get("id", 0) + + # Track max ID regardless of cursor (always advance) + if new_max_id is None or post_id > new_max_id: + new_max_id = post_id + + # Apply cursor: skip posts we've already seen + if last_seen_id is not None and post_id <= last_seen_id: + continue + + posts.append({ + "id": post_id, + "ap_id": post.get("ap_id", f"https://{instance}/post/{post_id}"), + "title": post.get("name", ""), + "body": post.get("body", "") or "", + "url": post.get("ap_id", ""), # ap_id is the canonical URL + "author": creator.get("name", ""), + "score": counts.get("score", 0), + "comment_count": counts.get("comments", 0), + "published": post.get("published", ""), + "community": community, + "instance": instance, + }) + + return posts, new_max_id diff --git a/app/services/scheduler.py b/app/services/scheduler.py index 71795fe..9089ece 100644 --- a/app/services/scheduler.py +++ b/app/services/scheduler.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.cron import CronTrigger +from apscheduler.triggers.interval import IntervalTrigger from app.services.poster import run_campaign @@ -119,3 +120,54 @@ def sync_all_campaigns(campaigns: list[dict]) -> None: for campaign in campaigns: sync_campaign(campaign) logger.info("Synced %d campaign(s) to scheduler", len(campaigns)) + + +# ------------------------------------------------------------------ # +# Signal scraper job +# ------------------------------------------------------------------ # + +_SCRAPER_JOB_ID = "signal_scraper" + + +async def _run_scraper_job() -> None: + """APScheduler coroutine target: run one pass of signal scraping.""" + from app.services.scraper import scrape_signals # deferred to avoid circular imports + logger.info("Signal scraper job starting") + try: + summary = await scrape_signals() + logger.info( + "Signal scraper done: %d sub(s), %d post(s) seen, %d signal(s) found", + summary["subs_scraped"], summary["posts_seen"], summary["signals_found"], + ) + except Exception: + logger.exception("Unhandled error in signal scraper job") + + +def start_scraper_job(interval_mins: int = 30) -> None: + """ + Register (or replace) the signal scraper interval job. + + Call this at startup when scraper_enabled=True. + """ + sched = get_scheduler() + existing = sched.get_job(_SCRAPER_JOB_ID) + if existing: + existing.remove() + + sched.add_job( + _run_scraper_job, + trigger=IntervalTrigger(minutes=interval_mins, timezone="UTC"), + id=_SCRAPER_JOB_ID, + replace_existing=True, + misfire_grace_time=600, + ) + logger.info("Signal scraper scheduled every %d minute(s)", interval_mins) + + +def stop_scraper_job() -> None: + """Remove the signal scraper job (e.g. when disabling via settings).""" + sched = get_scheduler() + existing = sched.get_job(_SCRAPER_JOB_ID) + if existing: + existing.remove() + logger.info("Signal scraper job removed") diff --git a/app/services/scraper.py b/app/services/scraper.py new file mode 100644 index 0000000..b0dae33 --- /dev/null +++ b/app/services/scraper.py @@ -0,0 +1,364 @@ +""" +Signal scraper: polls Reddit and Lemmy for posts matching signal_rules. + +Architecture: + - One global APScheduler interval job dispatches to per-platform fetchers. + - Reddit: public JSON API, cursor via fullname (t3_xxxxx) + `before` param. + - Lemmy: public v3 API, cursor via max integer post ID (client-side filter). + - Correctness dedup lives in the DB (UNIQUE platform+post_id + INSERT OR IGNORE). + Cursors are a performance optimization, not a correctness mechanism. + - Keyword matching (_matches_rule) is platform-agnostic — all rules evaluated + in one pass per fetched post to avoid redundant work. + +Adding a new platform: + 1. Add a fetch function returning (posts: list[NormalizedPost], cursor: str | None). + 2. Add a branch in _process_platform_sub(). + 3. Add the platform string to signal_rules rows. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import re +from datetime import datetime, timezone +from typing import Any + +import httpx + +from app.core.config import get_settings +from app.db.store import Store + +logger = logging.getLogger(__name__) + +# ------------------------------------------------------------------ # +# Reddit fetch +# ------------------------------------------------------------------ # + +async def _fetch_reddit_posts( + sub: str, + before: str | None, + limit: int, + user_agent: str, +) -> tuple[list[dict[str, Any]], str | None]: + """ + Fetch new posts from r/{sub}. + + Returns: + posts — list of raw post data dicts (newest first) + new_cursor — fullname of the newest post fetched, or None if empty + """ + params: dict[str, Any] = {"limit": limit, "raw_json": 1} + if before: + # 'before' in Reddit listing API = posts ABOVE this fullname = newer posts + params["before"] = before + + url = f"https://www.reddit.com/r/{sub}/new.json" + + async with httpx.AsyncClient( + headers={"User-Agent": user_agent}, + follow_redirects=True, + timeout=15.0, + ) as client: + resp = await client.get(url, params=params) + resp.raise_for_status() + data = resp.json() + + children = data.get("data", {}).get("children", []) + posts = [c["data"] for c in children if c.get("kind") == "t3"] + + # Newest post is first in the list; its fullname becomes the next cursor + new_cursor = posts[0]["name"] if posts else None + + return posts, new_cursor + + +# ------------------------------------------------------------------ # +# Keyword matching +# ------------------------------------------------------------------ # + +def _matches_rule(post: dict[str, Any], rule: dict[str, Any]) -> list[str]: + """ + Check if a post matches a signal rule. + + Returns a list of matched keywords (non-empty = match), or [] (no match). + """ + haystack = f"{post.get('title', '')} {post.get('selftext', '')}".lower() + keywords: list[str] = json.loads(rule["keywords"]) if isinstance(rule["keywords"], str) else rule["keywords"] + match_mode: str = rule.get("match_mode", "any") + min_score: int = rule.get("min_score", 0) + + # Score gate: skip posts below the minimum + post_score = post.get("score", 0) or 0 + if post_score < min_score: + return [] + + # No keywords = match everything (useful for "watch this sub broadly") + if not keywords: + return ["*"] + + if match_mode == "regex": + matched = [] + for pattern in keywords: + try: + if re.search(pattern, haystack, re.IGNORECASE): + matched.append(pattern) + except re.error: + logger.warning("Invalid regex in rule %d: %r", rule["id"], pattern) + return matched if matched else [] + + if match_mode == "all": + matched = [kw for kw in keywords if kw.lower() in haystack] + return matched if len(matched) == len(keywords) else [] + + # Default: any + return [kw for kw in keywords if kw.lower() in haystack] + + +# ------------------------------------------------------------------ # +# Normalized post type +# ------------------------------------------------------------------ # + +# Both platform fetchers normalize their output to this shape before +# the common matching + upsert pipeline runs. +# +# Fields mirror Reddit's naming where possible so _matches_rule() which +# uses "title" and "selftext" works without platform-specific branches. +class NormalizedPost(dict): + """ + Keys guaranteed present: + post_id str — unique within platform (Reddit fullname, Lemmy ap_id URL) + title str + selftext str — body text (empty string if none) + score int + num_comments int + author str + url str + created_utc float | None + """ + + +def _normalize_reddit(post: dict[str, Any]) -> NormalizedPost: + return NormalizedPost({ + "post_id": post["name"], + "title": post.get("title", ""), + "selftext": post.get("selftext", "") or "", + "score": post.get("score", 0), + "num_comments": post.get("num_comments", 0), + "author": post.get("author", ""), + "url": post.get("url", ""), + "created_utc": post.get("created_utc"), + }) + + +def _normalize_lemmy(post: dict[str, Any]) -> NormalizedPost: + published = post.get("published", "") + try: + ts = datetime.fromisoformat(published.rstrip("Z")).replace(tzinfo=timezone.utc).timestamp() + except (ValueError, AttributeError): + ts = None + return NormalizedPost({ + "post_id": post["ap_id"], + "title": post.get("title", ""), + "selftext": post.get("body", "") or "", + "score": post.get("score", 0), + "num_comments": post.get("comment_count", 0), + "author": post.get("author", ""), + "url": post.get("url", "") or post.get("ap_id", ""), + "created_utc": ts, + }) + + +# ------------------------------------------------------------------ # +# Common match + upsert pipeline +# ------------------------------------------------------------------ # + +def _run_matching( + post: NormalizedPost, + rules: list[dict], +) -> tuple[list[int], list[str]]: + """Apply all rules to a post. Returns (matched_rule_ids, matched_keywords).""" + matched_kws: set[str] = set() + matched_rule_ids: list[int] = [] + for rule in rules: + kws = _matches_rule(post, rule) + if kws: + matched_kws.update(k for k in kws if k != "*") + matched_rule_ids.append(rule["id"]) + return matched_rule_ids, sorted(matched_kws) + + +def _upsert_post( + post: NormalizedPost, + platform: str, + sub: str, + matched_rule_ids: list[int], + matched_kws: list[str], + store: Store, +) -> None: + posted_at = ( + datetime.fromtimestamp(post["created_utc"], tz=timezone.utc).isoformat() + if post.get("created_utc") + else None + ) + signal = store.upsert_signal( + platform=platform, + sub=sub, + post_id=post["post_id"], + title=post["title"], + body_snippet=(post["selftext"][:500] or None), + score=post["score"], + comment_count=post["num_comments"], + author=post["author"], + url=post["url"], + posted_at=posted_at, + matched_keywords=matched_kws, + ) + for rule_id in matched_rule_ids: + store.record_signal_rule_match(signal["id"], rule_id) + + +# ------------------------------------------------------------------ # +# Core scraper +# ------------------------------------------------------------------ # + +def _build_target_map( + all_rules: list[dict], +) -> dict[tuple[str, str], list[dict]]: + """ + Build a (platform, sub) → [rules] map from active rules. + + Rules with sub=None are global and appended to every explicit target. + Returns an empty dict if no sub-specific targets exist. + """ + global_rules_by_platform: dict[str, list[dict]] = {} + target_map: dict[tuple[str, str], list[dict]] = {} + + for rule in all_rules: + platform = rule.get("platform", "reddit") + if not rule["sub"]: + global_rules_by_platform.setdefault(platform, []).append(rule) + else: + key = (platform, rule["sub"]) + target_map.setdefault(key, []).append(rule) + + # Attach global rules to each explicit target of the same platform + for (platform, _sub), rules in target_map.items(): + rules.extend(global_rules_by_platform.get(platform, [])) + + return target_map + + +async def scrape_signals() -> dict[str, int]: + """ + Run one full scrape pass across all platforms and monitored communities. + + Returns: {"subs_scraped": N, "posts_seen": N, "signals_found": N} + """ + from app.services.lemmy.client import fetch_new_posts as lemmy_fetch, parse_community_target + + settings = get_settings() + store = Store(settings.db_path) + + try: + all_rules = store.list_signal_rules(active_only=True) + if not all_rules: + logger.info("Signal scraper: no active rules, skipping") + return {"subs_scraped": 0, "posts_seen": 0, "signals_found": 0} + + target_map = _build_target_map(all_rules) + if not target_map: + logger.info("Signal scraper: global rules but no sub targets — add sub-specific rules") + return {"subs_scraped": 0, "posts_seen": 0, "signals_found": 0} + + total_posts = 0 + total_signals = 0 + + for (platform, sub), rules in target_map.items(): + state = store.get_scrape_state(sub, platform) + cursor = state["last_fullname"] if state else None + label = f"{platform}:{sub}" + + logger.info("Scraping %s (cursor=%s, rules=%d)", label, cursor, len(rules)) + + # ---- Fetch ----------------------------------------------- + raw_posts: list[dict[str, Any]] = [] + new_cursor: str | None = None + + try: + if platform == "reddit": + raw_posts, new_cursor = await _fetch_reddit_posts( + sub=sub, + before=cursor, + limit=settings.scraper_fetch_limit, + user_agent=settings.scraper_user_agent, + ) + normalize = _normalize_reddit + + elif platform == "lemmy": + community, instance = parse_community_target(sub) + last_id = int(cursor) if cursor and cursor.isdigit() else None + lemmy_posts, max_id = await lemmy_fetch( + community=community, + instance=instance, + last_seen_id=last_id, + limit=settings.scraper_fetch_limit, + user_agent=settings.scraper_user_agent, + ) + raw_posts = lemmy_posts + new_cursor = str(max_id) if max_id is not None else None + normalize = _normalize_lemmy + + else: + logger.warning("Unknown platform %r for sub %r — skipping", platform, sub) + continue + + except httpx.HTTPStatusError as exc: + if exc.response.status_code in (403, 404): + logger.warning("%s returned %d — community may be private", label, exc.response.status_code) + else: + logger.error("HTTP error scraping %s: %s", label, exc) + await asyncio.sleep(settings.scraper_request_delay_secs) + continue + except ValueError as exc: + logger.error("Config error for %s: %s", label, exc) + continue + except Exception: + logger.exception("Unexpected error scraping %s", label) + await asyncio.sleep(settings.scraper_request_delay_secs) + continue + + # ---- Match + upsert -------------------------------------- + sub_signals = 0 + for raw in raw_posts: + total_posts += 1 + post = normalize(raw) + matched_rule_ids, matched_kws = _run_matching(post, rules) + if not matched_rule_ids: + continue + _upsert_post(post, platform, sub, matched_rule_ids, matched_kws, store) + sub_signals += 1 + logger.debug("Signal: %s — %s", label, post["title"][:60]) + + total_signals += sub_signals + + if new_cursor: + store.update_scrape_state( + sub=sub, + platform=platform, + last_fullname=new_cursor, + posts_seen_delta=len(raw_posts), + signals_found_delta=sub_signals, + ) + + logger.info("%s: %d posts checked, %d signal(s) found", label, len(raw_posts), sub_signals) + await asyncio.sleep(settings.scraper_request_delay_secs) + + return { + "subs_scraped": len(target_map), + "posts_seen": total_posts, + "signals_found": total_signals, + } + + finally: + store.close() diff --git a/frontend/index.html b/frontend/index.html index dc47d0a..f1fac05 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -4,6 +4,11 @@ Magpie — Campaign Dashboard + + + + +

diff --git a/frontend/src/App.vue b/frontend/src/App.vue index a193309..5f9e66c 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -1,14 +1,52 @@ diff --git a/frontend/src/components/CampaignDetail.vue b/frontend/src/components/CampaignDetail.vue index cacc174..f6d4a88 100644 --- a/frontend/src/components/CampaignDetail.vue +++ b/frontend/src/components/CampaignDetail.vue @@ -12,7 +12,7 @@ -
+
@@ -52,7 +52,7 @@
Recent Posts
- +
@@ -62,13 +62,13 @@ - - + - + @@ -210,4 +210,12 @@ function formatDate(iso: string) { .modal { padding: var(--spacing-lg); } .variant-row { padding: var(--spacing-sm); border-bottom: 1px solid var(--color-border); } .variant-row:last-child { border-bottom: none; } +.detail-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: var(--spacing-lg); +} +@media (max-width: 768px) { + .detail-grid { grid-template-columns: 1fr; } +} diff --git a/frontend/src/components/CampaignList.vue b/frontend/src/components/CampaignList.vue index 87d0a39..62dad1b 100644 --- a/frontend/src/components/CampaignList.vue +++ b/frontend/src/components/CampaignList.vue @@ -12,7 +12,7 @@
-
Sub
r/{{ p.target }} + r/{{ p.target }} {{ p.status }} {{ p.status }} {{ formatDate(p.posted_at) }}{{ formatDate(p.posted_at) }}
No posts yet.
+
@@ -24,21 +24,21 @@ - - - + - -
Name
+ {{ c.name }} {{ c.product }} + {{ c.product }} {{ c.cron_schedule ?? '— manual' }} + {{ c.active ? 'active' : 'paused' }} +
Loading...
+
+ Could not load opportunities: {{ loadError }} +
No opportunities{{ filterStatus ? ` with status "${filterStatus}"` : '' }}.
@@ -199,6 +202,7 @@ const selected = ref(null) const showAddModal = ref(false) const copied = ref(false) const filterStatus = ref('') +const loadError = ref(null) const editBody = ref('') const editTitle = ref('') @@ -222,8 +226,11 @@ const filtered = computed(() => async function load() { loading.value = true + loadError.value = null try { opportunities.value = await api.opportunities.list() + } catch (e: unknown) { + loadError.value = e instanceof Error ? e.message : 'Failed to load opportunities' } finally { loading.value = false } diff --git a/frontend/src/components/PostsView.vue b/frontend/src/components/PostsView.vue index 20ab866..2b8a5ce 100644 --- a/frontend/src/components/PostsView.vue +++ b/frontend/src/components/PostsView.vue @@ -5,7 +5,7 @@
- +
@@ -18,15 +18,15 @@ - - - + + - - - + + diff --git a/frontend/src/components/SignalsView.vue b/frontend/src/components/SignalsView.vue new file mode 100644 index 0000000..3a78464 --- /dev/null +++ b/frontend/src/components/SignalsView.vue @@ -0,0 +1,351 @@ + + + + + diff --git a/frontend/src/components/SubRulesView.vue b/frontend/src/components/SubRulesView.vue index b3bc829..a13b33a 100644 --- a/frontend/src/components/SubRulesView.vue +++ b/frontend/src/components/SubRulesView.vue @@ -6,7 +6,7 @@
-
Campaign
{{ campaignName(p.campaign_id) }}{{ p.target }} + {{ campaignName(p.campaign_id) }}{{ p.target }} {{ p.status }} ⚠ {{ p.triggered_by }}{{ formatDate(p.posted_at) }} + {{ p.triggered_by }}{{ formatDate(p.posted_at) }} view → —
+
@@ -20,25 +20,25 @@ - - - + + - - - - diff --git a/frontend/src/main.ts b/frontend/src/main.ts index fdbf31f..747dd47 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -9,11 +9,13 @@ import CampaignDetail from './components/CampaignDetail.vue' import SubRulesView from './components/SubRulesView.vue' import PostsView from './components/PostsView.vue' import OpportunitiesView from './components/OpportunitiesView.vue' +import SignalsView from './components/SignalsView.vue' const router = createRouter({ history: createWebHistory(), routes: [ - { path: '/', redirect: '/opportunities' }, + { path: '/', redirect: '/signals' }, + { path: '/signals', component: SignalsView }, { path: '/opportunities', component: OpportunitiesView }, { path: '/campaigns', component: CampaignList }, { path: '/campaigns/:id', component: CampaignDetail }, diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 837b877..aa862b8 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -147,6 +147,54 @@ export interface ApproveResult { instructions: string } +export type SignalMatchMode = 'any' | 'all' | 'regex' +export type SignalLabel = 'pain-point' | 'feedback' | 'mention' | 'trust' +export type SignalStatus = 'new' | 'saved' | 'dismissed' + +export interface SignalRule { + id: number + name: string + platform: string + sub: string | null + keywords: string[] + match_mode: SignalMatchMode + min_score: number + label: SignalLabel | null + active: number + created_at: string + notes: string | null +} + +export interface SignalRuleCreate { + name: string + platform?: string + sub?: string | null + keywords?: string[] + match_mode?: SignalMatchMode + min_score?: number + label?: SignalLabel | null + notes?: string | null +} + +export interface Signal { + id: number + platform: string + sub: string + post_id: string + title: string + body_snippet: string | null + score: number | null + comment_count: number | null + author: string | null + url: string | null + posted_at: string | null + surfaced_at: string + matched_keywords: string[] + status: SignalStatus + notes: string | null + matched_rules?: SignalRule[] +} + // ------------------------------------------------------------------ // // Campaigns // ------------------------------------------------------------------ // @@ -233,4 +281,32 @@ export const api = { dismiss: (id: number, note?: string) => http.post(`/opportunities/${id}/dismiss`, { note: note ?? null }).then(r => r.data), }, + + signalRules: { + list: (activeOnly = false) => + http.get('/signal-rules', { params: { active_only: activeOnly } }).then(r => r.data), + + create: (data: SignalRuleCreate) => + http.post('/signal-rules', data).then(r => r.data), + + get: (id: number) => + http.get(`/signal-rules/${id}`).then(r => r.data), + + update: (id: number, data: Partial & { active?: boolean }) => + http.patch(`/signal-rules/${id}`, data).then(r => r.data), + + delete: (id: number) => + http.delete(`/signal-rules/${id}`), + }, + + signals: { + list: (params?: { status?: SignalStatus; platform?: string; sub?: string; limit?: number }) => + http.get('/signals', { params }).then(r => r.data), + + get: (id: number) => + http.get(`/signals/${id}`).then(r => r.data), + + updateStatus: (id: number, status: SignalStatus, notes?: string) => + http.patch(`/signals/${id}/status`, { status, notes: notes ?? null }).then(r => r.data), + }, } diff --git a/frontend/src/theme.css b/frontend/src/theme.css index 0093455..3d61e72 100644 --- a/frontend/src/theme.css +++ b/frontend/src/theme.css @@ -1,130 +1,537 @@ /** * Magpie — Central Theme - * Theme-aware, responsive CSS classes. All components use these. + * Mobile-first. Bottom nav on small screens, sidebar on ≥768px. */ -:root { - --color-bg: #0f1117; - --color-bg-secondary: #1a1d27; - --color-bg-card: #1e2130; - --color-border: #2d3148; - --color-text: #e2e8f0; - --color-text-muted: #8892a4; - --color-primary: #7c6af7; - --color-primary-dim: #3d3578; - --color-success: #34d399; - --color-warning: #fbbf24; - --color-danger: #f87171; - --color-info: #60a5fa; +/* ---- Fonts ---- */ +@import url('https://fonts.googleapis.com/css2?family=Space+Mono:wght@400;700&family=DM+Sans:ital,opsz,wght@0,9..40,300;0,9..40,400;0,9..40,500;0,9..40,600;1,9..40,300&display=swap'); +:root { + /* Colors */ + --color-bg: #0b0d14; + --color-bg-secondary: #13161f; + --color-bg-card: #181c28; + --color-bg-lift: #1e2235; + --color-border: #252a3d; + --color-border-soft: #1c2030; + --color-text: #dde4f0; + --color-text-muted: #6b7694; + --color-text-dim: #3d455e; + + /* Brand accent — amber-gold for the "field ops" feel */ + --color-primary: #e8a93a; + --color-primary-dim: rgba(232,169,58,0.12); + --color-primary-glow: rgba(232,169,58,0.25); + + /* Semantic */ + --color-success: #3ecf8e; + --color-warning: #f0b429; + --color-danger: #f26b6b; + --color-info: #5ba4f5; + + /* Spacing */ --spacing-xs: 4px; --spacing-sm: 8px; --spacing-md: 16px; --spacing-lg: 24px; --spacing-xl: 40px; + /* Radii */ --radius-sm: 4px; --radius-md: 8px; - --radius-lg: 12px; + --radius-lg: 14px; + --radius-pill: 999px; - --font-mono: 'JetBrains Mono', 'Fira Code', monospace; - --font-sans: system-ui, -apple-system, sans-serif; + /* Typography */ + --font-sans: 'DM Sans', system-ui, sans-serif; + --font-mono: 'Space Mono', 'JetBrains Mono', monospace; + + /* Layout */ + --sidebar-w: 220px; + --bottom-nav-h: 60px; + --touch-min: 44px; } @media (prefers-color-scheme: light) { :root { - --color-bg: #f8f9fc; + --color-bg: #f2f4f8; --color-bg-secondary: #ffffff; --color-bg-card: #ffffff; + --color-bg-lift: #f8f9fc; --color-border: #dde1ec; - --color-text: #1a1d27; + --color-border-soft: #eaecf4; + --color-text: #1a1d2e; --color-text-muted: #6b7280; - --color-primary: #5b4fcf; - --color-primary-dim: #ede9ff; + --color-text-dim: #c0c5d4; + --color-primary: #c47b0a; + --color-primary-dim: rgba(196,123,10,0.10); + --color-primary-glow: rgba(196,123,10,0.20); } } -* { box-sizing: border-box; margin: 0; padding: 0; } +/* ================================================================== */ +/* Reset & Base */ +/* ================================================================== */ + +*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + +html { -webkit-text-size-adjust: 100%; } body { font-family: var(--font-sans); background: var(--color-bg); color: var(--color-text); font-size: 14px; - line-height: 1.5; + line-height: 1.55; + /* Subtle noise texture */ + background-image: + url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='300' height='300'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.75' numOctaves='4' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='300' height='300' filter='url(%23n)' opacity='0.03'/%3E%3C/svg%3E"); } -/* ---- Layout ---- */ -.app-shell { display: flex; min-height: 100vh; } -.sidebar { width: 220px; background: var(--color-bg-secondary); border-right: 1px solid var(--color-border); padding: var(--spacing-lg) var(--spacing-md); flex-shrink: 0; } -.main-content { flex: 1; padding: var(--spacing-lg); overflow-y: auto; } +:focus-visible { + outline: 2px solid var(--color-primary); + outline-offset: 2px; + border-radius: var(--radius-sm); +} -/* ---- Nav ---- */ -.nav-brand { font-size: 18px; font-weight: 700; color: var(--color-primary); margin-bottom: var(--spacing-lg); display: flex; align-items: center; gap: var(--spacing-sm); } -.nav-link { display: block; padding: var(--spacing-sm) var(--spacing-md); border-radius: var(--radius-md); color: var(--color-text-muted); text-decoration: none; margin-bottom: 2px; transition: background 0.15s, color 0.15s; } -.nav-link:hover, .nav-link.active { background: var(--color-primary-dim); color: var(--color-primary); } +/* ================================================================== */ +/* App Shell — mobile-first */ +/* ================================================================== */ -/* ---- Cards ---- */ -.card { background: var(--color-bg-card); border: 1px solid var(--color-border); border-radius: var(--radius-lg); padding: var(--spacing-md); } -.card-title { font-size: 13px; font-weight: 600; color: var(--color-text-muted); text-transform: uppercase; letter-spacing: 0.06em; margin-bottom: var(--spacing-md); } +.app-shell { + display: flex; + flex-direction: column; + min-height: 100dvh; /* dvh accounts for mobile browser chrome */ + min-height: 100vh; /* fallback */ +} -/* ---- Buttons ---- */ -.btn { display: inline-flex; align-items: center; gap: var(--spacing-xs); padding: 6px var(--spacing-md); border-radius: var(--radius-md); border: none; cursor: pointer; font-size: 13px; font-weight: 500; transition: opacity 0.15s; } -.btn:hover { opacity: 0.85; } -.btn:disabled { opacity: 0.4; cursor: not-allowed; } -.btn-primary { background: var(--color-primary); color: #fff; } -.btn-ghost { background: transparent; color: var(--color-text-muted); border: 1px solid var(--color-border); } -.btn-danger { background: var(--color-danger); color: #fff; } +/* Main content area — leaves room for bottom nav on mobile */ +.main-content { + flex: 1; + overflow-y: auto; + padding: var(--spacing-md); + padding-bottom: calc(var(--bottom-nav-h) + var(--spacing-md)); +} + +/* ---- Bottom Nav (mobile) ---- */ +.bottom-nav { + position: fixed; + bottom: 0; left: 0; right: 0; + height: var(--bottom-nav-h); + background: var(--color-bg-secondary); + border-top: 1px solid var(--color-border); + display: flex; + align-items: stretch; + z-index: 50; + /* Safe area for notched phones */ + padding-bottom: env(safe-area-inset-bottom); +} + +.bottom-nav-item { + flex: 1; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 3px; + text-decoration: none; + color: var(--color-text-muted); + font-size: 10px; + font-family: var(--font-mono); + letter-spacing: 0.04em; + text-transform: uppercase; + transition: color 0.15s; + min-height: var(--touch-min); + -webkit-tap-highlight-color: transparent; +} + +.bottom-nav-item .nav-icon { + font-size: 18px; + line-height: 1; +} + +.bottom-nav-item.active, +.bottom-nav-item:active { + color: var(--color-primary); +} + +.bottom-nav-item.active .nav-icon { + filter: drop-shadow(0 0 6px var(--color-primary-glow)); +} + +/* ---- Sidebar (desktop ≥768px) ---- */ +.sidebar { + display: none; +} + +@media (min-width: 768px) { + .app-shell { + flex-direction: row; + } + + .main-content { + padding-bottom: var(--spacing-lg); + } + + .bottom-nav { + display: none; + } + + .sidebar { + display: flex; + flex-direction: column; + width: var(--sidebar-w); + background: var(--color-bg-secondary); + border-right: 1px solid var(--color-border); + padding: var(--spacing-lg) var(--spacing-md); + flex-shrink: 0; + position: sticky; + top: 0; + height: 100vh; + overflow-y: auto; + } + + .nav-brand { + font-family: var(--font-mono); + font-size: 15px; + font-weight: 700; + color: var(--color-primary); + margin-bottom: var(--spacing-lg); + display: flex; + align-items: center; + gap: var(--spacing-sm); + letter-spacing: -0.01em; + } + + .nav-link { + display: flex; + align-items: center; + gap: var(--spacing-sm); + padding: var(--spacing-sm) var(--spacing-md); + border-radius: var(--radius-md); + color: var(--color-text-muted); + text-decoration: none; + margin-bottom: 2px; + font-size: 13px; + transition: background 0.15s, color 0.15s; + } + + .nav-link:hover { background: var(--color-primary-dim); color: var(--color-text); } + .nav-link.active { background: var(--color-primary-dim); color: var(--color-primary); font-weight: 500; } +} + +/* ================================================================== */ +/* Cards */ +/* ================================================================== */ + +.card { + background: var(--color-bg-card); + border: 1px solid var(--color-border); + border-radius: var(--radius-lg); + padding: var(--spacing-md); +} + +.card-title { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 700; + color: var(--color-text-dim); + text-transform: uppercase; + letter-spacing: 0.1em; + margin-bottom: var(--spacing-md); +} + +/* ================================================================== */ +/* Buttons — touch-safe */ +/* ================================================================== */ + +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: var(--spacing-xs); + padding: 0 var(--spacing-md); + height: var(--touch-min); + border-radius: var(--radius-md); + border: none; + cursor: pointer; + font-family: var(--font-sans); + font-size: 13px; + font-weight: 500; + text-decoration: none; + transition: opacity 0.15s, transform 0.1s; + -webkit-tap-highlight-color: transparent; + white-space: nowrap; +} + +.btn:active { transform: scale(0.97); } +.btn:disabled { opacity: 0.38; cursor: not-allowed; } + +.btn-primary { background: var(--color-primary); color: #111; } +.btn-ghost { background: transparent; color: var(--color-text-muted); border: 1px solid var(--color-border); } +.btn-danger { background: var(--color-danger); color: #fff; } .btn-success { background: var(--color-success); color: #111; } -.btn-sm { padding: 4px var(--spacing-sm); font-size: 12px; } -/* ---- Badges ---- */ -.badge { display: inline-flex; align-items: center; padding: 2px 8px; border-radius: 999px; font-size: 11px; font-weight: 600; } -.badge-success { background: color-mix(in srgb, var(--color-success) 20%, transparent); color: var(--color-success); } -.badge-warning { background: color-mix(in srgb, var(--color-warning) 20%, transparent); color: var(--color-warning); } -.badge-danger { background: color-mix(in srgb, var(--color-danger) 20%, transparent); color: var(--color-danger); } -.badge-info { background: color-mix(in srgb, var(--color-info) 20%, transparent); color: var(--color-info); } -.badge-muted { background: color-mix(in srgb, var(--color-text-muted) 15%, transparent); color: var(--color-text-muted); } +.btn-sm { + height: 32px; + padding: 0 var(--spacing-sm); + font-size: 12px; + border-radius: var(--radius-sm); +} + +/* ================================================================== */ +/* Badges */ +/* ================================================================== */ + +.badge { + display: inline-flex; + align-items: center; + padding: 2px 8px; + border-radius: var(--radius-pill); + font-family: var(--font-mono); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.badge-success { background: color-mix(in srgb, var(--color-success) 18%, transparent); color: var(--color-success); } +.badge-warning { background: color-mix(in srgb, var(--color-warning) 18%, transparent); color: var(--color-warning); } +.badge-danger { background: color-mix(in srgb, var(--color-danger) 18%, transparent); color: var(--color-danger); } +.badge-info { background: color-mix(in srgb, var(--color-info) 18%, transparent); color: var(--color-info); } +.badge-muted { background: color-mix(in srgb, var(--color-text-muted) 12%, transparent); color: var(--color-text-muted); } + +/* ================================================================== */ +/* Forms */ +/* ================================================================== */ + +.form-label { + display: block; + font-size: 11px; + font-family: var(--font-mono); + font-weight: 700; + color: var(--color-text-dim); + text-transform: uppercase; + letter-spacing: 0.08em; + margin-bottom: 6px; +} -/* ---- Forms ---- */ -.form-label { display: block; font-size: 12px; font-weight: 500; color: var(--color-text-muted); margin-bottom: var(--spacing-xs); } .form-input, .form-select, .form-textarea { - width: 100%; padding: 8px var(--spacing-sm); background: var(--color-bg-secondary); - border: 1px solid var(--color-border); border-radius: var(--radius-md); - color: var(--color-text); font-size: 13px; font-family: inherit; + width: 100%; + padding: 10px var(--spacing-sm); + background: var(--color-bg-secondary); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + color: var(--color-text); + font-size: 14px; + font-family: var(--font-sans); + /* Touch-safe height */ + min-height: var(--touch-min); + -webkit-appearance: none; + appearance: none; } + .form-input:focus, .form-select:focus, .form-textarea:focus { - outline: none; border-color: var(--color-primary); - box-shadow: 0 0 0 2px color-mix(in srgb, var(--color-primary) 25%, transparent); + outline: none; + border-color: var(--color-primary); + box-shadow: 0 0 0 2px var(--color-primary-glow); } -.form-textarea { resize: vertical; min-height: 100px; font-family: var(--font-mono); font-size: 12px; } + +.form-textarea { + resize: vertical; + min-height: 100px; + font-family: var(--font-mono); + font-size: 12px; + line-height: 1.6; +} + .form-group { margin-bottom: var(--spacing-md); } -/* ---- Table ---- */ -.table { width: 100%; border-collapse: collapse; font-size: 13px; } -.table th { text-align: left; padding: var(--spacing-sm) var(--spacing-md); font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; color: var(--color-text-muted); border-bottom: 1px solid var(--color-border); } -.table td { padding: var(--spacing-sm) var(--spacing-md); border-bottom: 1px solid var(--color-border); } -.table tr:last-child td { border-bottom: none; } -.table tr:hover td { background: color-mix(in srgb, var(--color-primary) 5%, transparent); } +/* ================================================================== */ +/* Responsive Table → Card stack */ +/* ================================================================== */ -/* ---- Status dots ---- */ -.status-dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; } -.status-dot.success { background: var(--color-success); } +.table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +.table th { + text-align: left; + padding: var(--spacing-sm) var(--spacing-md); + font-family: var(--font-mono); + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--color-text-dim); + border-bottom: 1px solid var(--color-border); +} + +.table td { + padding: var(--spacing-sm) var(--spacing-md); + border-bottom: 1px solid var(--color-border-soft); + vertical-align: middle; +} + +.table tr:last-child td { border-bottom: none; } + +.table tr:hover td { + background: color-mix(in srgb, var(--color-primary) 4%, transparent); +} + +/* Mobile: collapse table rows into stacked cards */ +@media (max-width: 640px) { + .table-responsive thead { display: none; } + + .table-responsive tbody tr { + display: block; + background: var(--color-bg-card); + border: 1px solid var(--color-border); + border-radius: var(--radius-lg); + margin-bottom: var(--spacing-sm); + padding: var(--spacing-sm) 0; + } + + .table-responsive tbody tr:hover { background: var(--color-bg-lift); } + + .table-responsive td { + display: flex; + align-items: center; + justify-content: space-between; + padding: 6px var(--spacing-md); + border-bottom: none; + } + + .table-responsive td::before { + content: attr(data-label); + font-family: var(--font-mono); + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--color-text-dim); + flex-shrink: 0; + margin-right: var(--spacing-sm); + } + + /* Hide empty/action cells label */ + .table-responsive td[data-label=""] { justify-content: flex-end; } + .table-responsive td[data-label=""]::before { display: none; } +} + +/* ================================================================== */ +/* Status indicators */ +/* ================================================================== */ + +.status-dot { + display: inline-block; + width: 7px; height: 7px; + border-radius: 50%; + margin-right: 6px; + flex-shrink: 0; +} + +.status-dot.success { background: var(--color-success); box-shadow: 0 0 6px color-mix(in srgb, var(--color-success) 60%, transparent); } .status-dot.failed { background: var(--color-danger); } .status-dot.pending { background: var(--color-warning); } .status-dot.skipped { background: var(--color-text-muted); } -.status-dot.running { background: var(--color-info); animation: pulse 1s infinite; } +.status-dot.running { background: var(--color-info); animation: pulse 1s ease-in-out infinite; } -@keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } } +@keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.3; } } -/* ---- Page header ---- */ -.page-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: var(--spacing-lg); } -.page-title { font-size: 20px; font-weight: 700; } +/* ================================================================== */ +/* Page header */ +/* ================================================================== */ -/* ---- Empty state ---- */ -.empty-state { text-align: center; padding: var(--spacing-xl); color: var(--color-text-muted); } +.page-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: var(--spacing-lg); + gap: var(--spacing-sm); +} -/* ---- SR only ---- */ -.sr-only { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0,0,0,0); white-space: nowrap; } +.page-title { + font-family: var(--font-mono); + font-size: 16px; + font-weight: 700; + letter-spacing: -0.02em; +} -:focus-visible { outline: 2px solid var(--color-primary); outline-offset: 2px; border-radius: var(--radius-sm); } +/* ================================================================== */ +/* Empty state */ +/* ================================================================== */ + +.empty-state { + text-align: center; + padding: var(--spacing-xl); + color: var(--color-text-muted); + font-size: 13px; +} + +/* ================================================================== */ +/* Modal / bottom sheet */ +/* ================================================================== */ + +.modal-backdrop { + position: fixed; inset: 0; + background: rgba(0, 0, 0, 0.7); + display: flex; + align-items: flex-end; /* bottom sheet on mobile */ + justify-content: center; + z-index: 100; + padding: 0; +} + +.modal { + background: var(--color-bg-secondary); + border: 1px solid var(--color-border); + border-radius: var(--radius-lg) var(--radius-lg) 0 0; + padding: var(--spacing-lg); + width: 100%; + max-height: 92dvh; + max-height: 92vh; + overflow-y: auto; +} + +/* Desktop: centered dialog */ +@media (min-width: 640px) { + .modal-backdrop { align-items: center; padding: var(--spacing-md); } + .modal { border-radius: var(--radius-lg); max-width: 520px; width: 100%; } + .modal.modal-wide { max-width: 680px; } +} + +/* Drag handle visual cue on mobile */ +.modal::before { + content: ''; + display: block; + width: 36px; height: 4px; + background: var(--color-border); + border-radius: var(--radius-pill); + margin: 0 auto var(--spacing-md); +} + +@media (min-width: 640px) { + .modal::before { display: none; } +} + +/* ================================================================== */ +/* Utilities */ +/* ================================================================== */ + +.sr-only { + position: absolute; width: 1px; height: 1px; + overflow: hidden; clip: rect(0,0,0,0); white-space: nowrap; +} + +.text-mono { font-family: var(--font-mono); } +.text-muted { color: var(--color-text-muted); } +.text-sm { font-size: 12px; } + +.flex { display: flex; } +.flex-center { display: flex; align-items: center; } +.gap-sm { gap: var(--spacing-sm); } +.gap-md { gap: var(--spacing-md); } +.ml-auto { margin-left: auto; } diff --git a/manage.sh b/manage.sh index 1169a0d..d2f7830 100755 --- a/manage.sh +++ b/manage.sh @@ -1,74 +1,298 @@ #!/usr/bin/env bash -# Magpie management script +# Magpie management script — native dev process manager set -euo pipefail REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$REPO_DIR" +# ------------------------------------------------------------------ # +# Config +# ------------------------------------------------------------------ # + API_PORT=8532 WEB_PORT=8531 -COMPOSE="docker compose" +CONDA_ENV=cf +DATA_DIR="${HOME}/.local/share/magpie" +LOG_DIR="${DATA_DIR}/logs" +PID_API="${DATA_DIR}/api.pid" +PID_WEB="${DATA_DIR}/web.pid" +LOG_API="${LOG_DIR}/api.log" +LOG_WEB="${LOG_DIR}/web.log" + +mkdir -p "$DATA_DIR" "$LOG_DIR" + +# ------------------------------------------------------------------ # +# Colors +# ------------------------------------------------------------------ # + +GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; CYAN='\033[0;36m'; RESET='\033[0m' +ok() { echo -e "${GREEN}✓${RESET} $*"; } +warn() { echo -e "${YELLOW}⚠${RESET} $*"; } +err() { echo -e "${RED}✗${RESET} $*"; } +info() { echo -e "${CYAN}→${RESET} $*"; } + +# ------------------------------------------------------------------ # +# Process helpers +# ------------------------------------------------------------------ # + +_pid_alive() { + local pid_file="$1" + [[ -f "$pid_file" ]] && kill -0 "$(cat "$pid_file")" 2>/dev/null +} + +_port_open() { + ss -tlnp 2>/dev/null | grep -q ":$1 " || \ + lsof -ti:"$1" >/dev/null 2>&1 +} + +_start_api() { + if _pid_alive "$PID_API"; then + warn "API already running (PID $(cat "$PID_API"))" + return + fi + if _port_open "$API_PORT"; then + warn "Port :${API_PORT} already in use by another process — stop it first or change API_PORT" + return + fi + info "Starting API on :${API_PORT}..." + conda run --no-capture-output -n "$CONDA_ENV" \ + uvicorn app.main:app \ + --host 0.0.0.0 --port "$API_PORT" \ + --reload --reload-dir app \ + >> "$LOG_API" 2>&1 & + echo $! > "$PID_API" + # Wait for port to open (up to 10s) + local i=0 + while ! _port_open "$API_PORT" && (( i++ < 20 )); do sleep 0.5; done + if _port_open "$API_PORT"; then + ok "API up → http://localhost:${API_PORT}/docs" + else + warn "API started (PID $(cat "$PID_API")) but port not open yet — check: ./manage.sh logs api" + fi +} + +_start_web() { + if _pid_alive "$PID_WEB"; then + warn "Web already running (PID $(cat "$PID_WEB"))" + return + fi + if _port_open "$WEB_PORT"; then + warn "Port :${WEB_PORT} already in use by another process — stop it first or change WEB_PORT" + return + fi + info "Starting web on :${WEB_PORT}..." + cd frontend + npm run dev >> "$LOG_WEB" 2>&1 & + echo $! > "$PID_WEB" + cd "$REPO_DIR" + local i=0 + while ! _port_open "$WEB_PORT" && (( i++ < 20 )); do sleep 0.5; done + if _port_open "$WEB_PORT"; then + ok "Web up → http://localhost:${WEB_PORT}" + else + warn "Web started (PID $(cat "$PID_WEB")) but port not open yet — check: ./manage.sh logs web" + fi +} + +_stop_service() { + local name="$1" pid_file="$2" + if _pid_alive "$pid_file"; then + local pid + pid=$(cat "$pid_file") + info "Stopping ${name} (PID ${pid})..." + kill "$pid" 2>/dev/null || true + # Give it up to 5s to exit + local i=0 + while _pid_alive "$pid_file" && (( i++ < 10 )); do sleep 0.5; done + if _pid_alive "$pid_file"; then + warn "${name} didn't stop cleanly — sending SIGKILL" + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + ok "${name} stopped" + else + rm -f "$pid_file" + info "${name} was not running" + fi +} + +# ------------------------------------------------------------------ # +# Commands +# ------------------------------------------------------------------ # cmd="${1:-help}" +shift || true case "$cmd" in + start) - echo "Starting Magpie (API :$API_PORT, Web :$WEB_PORT)..." - $COMPOSE up -d + _start_api + _start_web ;; + stop) - $COMPOSE stop + _stop_service "API" "$PID_API" + _stop_service "Web" "$PID_WEB" ;; + restart) - $COMPOSE restart + _stop_service "API" "$PID_API" + _stop_service "Web" "$PID_WEB" + _start_api + _start_web ;; + + restart-api) + _stop_service "API" "$PID_API" + _start_api + ;; + + restart-web) + _stop_service "Web" "$PID_WEB" + _start_web + ;; + status) - $COMPOSE ps - ;; - logs) - service="${2:-}" - if [[ -n "$service" ]]; then - $COMPOSE logs -f "$service" - else - $COMPOSE logs -f - fi - ;; - build) - $COMPOSE build - ;; - open) - xdg-open "http://localhost:$WEB_PORT" 2>/dev/null || open "http://localhost:$WEB_PORT" - ;; - dev-api) - echo "Starting API in dev mode (host network, port $API_PORT)..." - conda run -n cf uvicorn app.main:app --host 0.0.0.0 --port $API_PORT --reload - ;; - dev-web) - echo "Starting frontend dev server (port $WEB_PORT)..." - cd frontend && npm run dev - ;; - login) - echo "Refreshing Reddit session..." - conda run -n cf xvfb-run --auto-servernum python -m app.services.reddit.post --login - ;; - migrate) - echo "Running DB migrations..." - conda run -n cf python -c "from app.db.store import Store; from app.core.config import get_settings; s=Store(get_settings().db_path); s.run_migrations(); s.close(); print('Done.')" - ;; - help|*) - echo "Usage: ./manage.sh " echo "" - echo "Commands:" - echo " start Start all services via Docker Compose" - echo " stop Stop all services" - echo " restart Restart all services" - echo " status Show service status" - echo " logs [svc] Tail logs (optionally for one service)" - echo " build Build Docker images" - echo " open Open browser to dashboard" - echo " dev-api Run API in dev mode (conda, hot-reload)" - echo " dev-web Run frontend dev server" - echo " login Refresh Reddit Playwright session" - echo " migrate Run DB migrations standalone" + echo -e " ${CYAN}Magpie — Service Status${RESET}" + echo " ─────────────────────────────────────" + + # API + if _pid_alive "$PID_API"; then + local_pid=$(cat "$PID_API") + if _port_open "$API_PORT"; then + ok " API :${API_PORT} (PID ${local_pid})" + else + warn " API PID ${local_pid} alive but port :${API_PORT} not open" + fi + else + err " API stopped" + fi + + # Web + if _pid_alive "$PID_WEB"; then + local_pid=$(cat "$PID_WEB") + if _port_open "$WEB_PORT"; then + ok " Web :${WEB_PORT} (PID ${local_pid})" + else + warn " Web PID ${local_pid} alive but port :${WEB_PORT} not open" + fi + else + err " Web stopped" + fi + + # Scheduler (query API) + if _port_open "$API_PORT"; then + sched=$(curl -sf "http://localhost:${API_PORT}/api/v1/scheduler/status" \ + | python3 -c "import json,sys; d=json.load(sys.stdin); print(f\"{d.get('job_count',0)} job(s)\")" 2>/dev/null || echo "?") + info " Scheduler ${sched} scheduled" + fi + + # Session age + SESSION_FILE="${DATA_DIR}/session.json" + if [[ -f "$SESSION_FILE" ]]; then + age_h=$(python3 -c "import time,os; print(f\"{(time.time()-os.path.getmtime('$SESSION_FILE'))/3600:.1f}h\")" 2>/dev/null || echo "?") + if python3 -c "import time,os; exit(0 if (time.time()-os.path.getmtime('$SESSION_FILE'))/3600 < 12 else 1)" 2>/dev/null; then + ok " Session ${age_h} old (valid)" + else + warn " Session ${age_h} old (stale — run: ./manage.sh login)" + fi + else + warn " Session no session.json — run: ./manage.sh login" + fi + + echo "" + ;; + + logs) + target="${1:-all}" + case "$target" in + api) tail -f "$LOG_API" ;; + web) tail -f "$LOG_WEB" ;; + all|*) + echo "=== API log ===" && tail -f "$LOG_API" & + echo "=== Web log ===" && tail -f "$LOG_WEB" & + wait + ;; + esac + ;; + + open) + xdg-open "http://localhost:${WEB_PORT}" 2>/dev/null \ + || open "http://localhost:${WEB_PORT}" 2>/dev/null \ + || info "Open http://localhost:${WEB_PORT} in your browser" + ;; + + update) + info "Pulling latest changes..." + git pull + + info "Installing Python deps..." + conda run -n "$CONDA_ENV" pip install -q -e . + + info "Installing frontend deps..." + cd frontend && npm install --silent && cd "$REPO_DIR" + + # Restart API to pick up code changes; web hot-reloads itself + if _pid_alive "$PID_API"; then + info "Restarting API to pick up changes..." + _stop_service "API" "$PID_API" + _start_api + else + info "API not running — start with: ./manage.sh start" + fi + ok "Update complete" + ;; + + login) + info "Refreshing Reddit session (opens browser via Xvfb)..." + REDDIT_SESSION_FILE="${DATA_DIR}/session.json" \ + conda run --no-capture-output -n "$CONDA_ENV" \ + xvfb-run --auto-servernum \ + python -m app.services.reddit.post --login + ;; + + migrate) + info "Running DB migrations..." + conda run -n "$CONDA_ENV" python - <<'EOF' +from app.db.store import Store +from app.core.config import get_settings +s = Store(get_settings().db_path) +s.run_migrations() +s.close() +print("Migrations applied.") +EOF + ;; + + seed) + info "Seeding campaigns from legacy reddit-poster scripts..." + conda run -n "$CONDA_ENV" python scripts/seed_campaigns.py + ;; + + help|*) + echo "" + echo -e " ${CYAN}Usage: ./manage.sh [args]${RESET}" + echo "" + echo " Process control:" + echo " start Start API and web dev server in background" + echo " stop Stop both" + echo " restart Full restart" + echo " restart-api Restart API only (e.g. after code changes)" + echo " restart-web Restart web dev server only" + echo " status Show process status, scheduler jobs, session age" + echo "" + echo " Logs:" + echo " logs Tail all logs" + echo " logs api Tail API log only" + echo " logs web Tail web log only" + echo " Logs at: ${LOG_DIR}/" + echo "" + echo " Maintenance:" + echo " update git pull + pip/npm install + API restart" + echo " migrate Run DB migrations standalone" + echo " seed Seed campaigns from legacy scripts" + echo " login Refresh Reddit Playwright session" + echo " open Open dashboard in browser" + echo "" ;; esac
Sub / Channel
{{ r.sub }}{{ r.platform }} + {{ r.sub }}{{ r.platform }} {{ r.flair_to_use ?? '(required, unknown)' }} — + unknown allowed banned + yes — + {{ r.notes ?? '—' }} +