feat(db): SnipeSharedStore — Postgres backend for sellers, market_comps, reported_sellers
Implements SharedTableProtocol against a ThreadedConnectionPool (psycopg2). SnipeSharedDB handles pool lifecycle and idempotent SQL migrations. save_sellers uses COALESCE to preserve existing account_age_days when the new record omits it. All 6 Postgres tests skip cleanly without SNIPE_SHARED_DB_URL.
This commit is contained in:
parent
78809c761e
commit
1d6556072f
3 changed files with 375 additions and 0 deletions
250
app/db/pg_shared.py
Normal file
250
app/db/pg_shared.py
Normal file
|
|
@ -0,0 +1,250 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import psycopg2
|
||||||
|
from psycopg2.pool import ThreadedConnectionPool
|
||||||
|
|
||||||
|
from app.db.models import MarketComp, Seller
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_MIN_CONN = 2
|
||||||
|
_MAX_CONN = 20
|
||||||
|
|
||||||
|
|
||||||
|
class SnipeSharedDB:
|
||||||
|
"""Thread-safe Postgres connection pool for Snipe shared tables."""
|
||||||
|
|
||||||
|
def __init__(self, dsn: str) -> None:
|
||||||
|
self._pool = ThreadedConnectionPool(_MIN_CONN, _MAX_CONN, dsn=dsn)
|
||||||
|
|
||||||
|
def getconn(self):
|
||||||
|
return self._pool.getconn()
|
||||||
|
|
||||||
|
def putconn(self, conn) -> None:
|
||||||
|
self._pool.putconn(conn)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self._pool.closeall()
|
||||||
|
|
||||||
|
def run_migrations(self) -> None:
|
||||||
|
"""Apply pg_migrations/*.sql in filename order. Idempotent."""
|
||||||
|
migrations_dir = Path(__file__).parent / "pg_migrations"
|
||||||
|
files = sorted(migrations_dir.glob("*.sql"), key=lambda p: p.name)
|
||||||
|
|
||||||
|
conn = self.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS _snipe_shared_migrations (
|
||||||
|
filename TEXT PRIMARY KEY,
|
||||||
|
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.commit()
|
||||||
|
for f in files:
|
||||||
|
cur.execute(
|
||||||
|
"SELECT 1 FROM _snipe_shared_migrations WHERE filename = %s",
|
||||||
|
(f.name,),
|
||||||
|
)
|
||||||
|
if cur.fetchone():
|
||||||
|
continue
|
||||||
|
log.info("Applying migration: %s", f.name)
|
||||||
|
cur.execute(f.read_text())
|
||||||
|
cur.execute(
|
||||||
|
"INSERT INTO _snipe_shared_migrations (filename) VALUES (%s)",
|
||||||
|
(f.name,),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self.putconn(conn)
|
||||||
|
|
||||||
|
|
||||||
|
class SnipeSharedStore:
|
||||||
|
"""Postgres-backed store for sellers, market_comps, and reported_sellers.
|
||||||
|
|
||||||
|
Satisfies SharedTableProtocol. clone() returns self — ThreadedConnectionPool
|
||||||
|
is already thread-safe, so no new instance is needed per thread.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, db: SnipeSharedDB) -> None:
|
||||||
|
self._db = db
|
||||||
|
|
||||||
|
def clone(self) -> "SnipeSharedStore":
|
||||||
|
return self
|
||||||
|
|
||||||
|
# Sellers
|
||||||
|
|
||||||
|
def save_seller(self, seller: Seller) -> None:
|
||||||
|
self.save_sellers([seller])
|
||||||
|
|
||||||
|
def save_sellers(self, sellers: list[Seller]) -> None:
|
||||||
|
if not sellers:
|
||||||
|
return
|
||||||
|
conn = self._db.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.executemany(
|
||||||
|
"""
|
||||||
|
INSERT INTO sellers
|
||||||
|
(platform, platform_seller_id, username, account_age_days,
|
||||||
|
feedback_count, feedback_ratio, category_history_json)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||||
|
ON CONFLICT (platform, platform_seller_id) DO UPDATE SET
|
||||||
|
username = EXCLUDED.username,
|
||||||
|
feedback_count = EXCLUDED.feedback_count,
|
||||||
|
feedback_ratio = EXCLUDED.feedback_ratio,
|
||||||
|
account_age_days = COALESCE(
|
||||||
|
EXCLUDED.account_age_days,
|
||||||
|
sellers.account_age_days
|
||||||
|
),
|
||||||
|
category_history_json = COALESCE(
|
||||||
|
NULLIF(NULLIF(EXCLUDED.category_history_json, '{}'), ''),
|
||||||
|
NULLIF(NULLIF(sellers.category_history_json, '{}'), ''),
|
||||||
|
'{}'
|
||||||
|
),
|
||||||
|
fetched_at = NOW()
|
||||||
|
""",
|
||||||
|
[
|
||||||
|
(s.platform, s.platform_seller_id, s.username, s.account_age_days,
|
||||||
|
s.feedback_count, s.feedback_ratio, s.category_history_json or "{}")
|
||||||
|
for s in sellers
|
||||||
|
],
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self._db.putconn(conn)
|
||||||
|
|
||||||
|
def get_seller(self, platform: str, platform_seller_id: str) -> Optional[Seller]:
|
||||||
|
conn = self._db.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
SELECT platform, platform_seller_id, username, account_age_days,
|
||||||
|
feedback_count, feedback_ratio, category_history_json,
|
||||||
|
id, fetched_at
|
||||||
|
FROM sellers
|
||||||
|
WHERE platform = %s AND platform_seller_id = %s
|
||||||
|
""",
|
||||||
|
(platform, platform_seller_id),
|
||||||
|
)
|
||||||
|
row = cur.fetchone()
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
return Seller(*row[:7], id=row[7], fetched_at=str(row[8]))
|
||||||
|
finally:
|
||||||
|
self._db.putconn(conn)
|
||||||
|
|
||||||
|
def delete_seller_data(self, platform: str, platform_seller_id: str) -> None:
|
||||||
|
conn = self._db.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"DELETE FROM sellers WHERE platform = %s AND platform_seller_id = %s",
|
||||||
|
(platform, platform_seller_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self._db.putconn(conn)
|
||||||
|
|
||||||
|
# MarketComps
|
||||||
|
|
||||||
|
def save_market_comp(self, comp: MarketComp) -> None:
|
||||||
|
conn = self._db.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO market_comps
|
||||||
|
(platform, query_hash, median_price, sample_count, expires_at)
|
||||||
|
VALUES (%s, %s, %s, %s, %s::TIMESTAMPTZ)
|
||||||
|
ON CONFLICT (platform, query_hash) DO UPDATE SET
|
||||||
|
median_price = EXCLUDED.median_price,
|
||||||
|
sample_count = EXCLUDED.sample_count,
|
||||||
|
expires_at = EXCLUDED.expires_at,
|
||||||
|
fetched_at = NOW()
|
||||||
|
""",
|
||||||
|
(comp.platform, comp.query_hash, comp.median_price,
|
||||||
|
comp.sample_count, comp.expires_at),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self._db.putconn(conn)
|
||||||
|
|
||||||
|
def get_market_comp(self, platform: str, query_hash: str) -> Optional[MarketComp]:
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
conn = self._db.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
SELECT platform, query_hash, median_price, sample_count,
|
||||||
|
expires_at, id, fetched_at
|
||||||
|
FROM market_comps
|
||||||
|
WHERE platform = %s AND query_hash = %s AND expires_at > %s::TIMESTAMPTZ
|
||||||
|
""",
|
||||||
|
(platform, query_hash, now),
|
||||||
|
)
|
||||||
|
row = cur.fetchone()
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
return MarketComp(*row[:5], id=row[5], fetched_at=str(row[6]))
|
||||||
|
finally:
|
||||||
|
self._db.putconn(conn)
|
||||||
|
|
||||||
|
# Reported Sellers
|
||||||
|
|
||||||
|
def mark_reported(
|
||||||
|
self,
|
||||||
|
platform: str,
|
||||||
|
platform_seller_id: str,
|
||||||
|
username: Optional[str] = None,
|
||||||
|
reported_by: str = "user",
|
||||||
|
) -> None:
|
||||||
|
conn = self._db.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO reported_sellers
|
||||||
|
(platform, platform_seller_id, username, reported_by)
|
||||||
|
VALUES (%s, %s, %s, %s)
|
||||||
|
ON CONFLICT (platform, platform_seller_id) DO NOTHING
|
||||||
|
""",
|
||||||
|
(platform, platform_seller_id, username, reported_by),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self._db.putconn(conn)
|
||||||
|
|
||||||
|
def list_reported(self, platform: str = "ebay") -> list[str]:
|
||||||
|
conn = self._db.getconn()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"SELECT platform_seller_id FROM reported_sellers WHERE platform = %s",
|
||||||
|
(platform,),
|
||||||
|
)
|
||||||
|
return [row[0] for row in cur.fetchall()]
|
||||||
|
finally:
|
||||||
|
self._db.putconn(conn)
|
||||||
17
tests/conftest.py
Normal file
17
tests/conftest.py
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
import os
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_configure(config):
|
||||||
|
config.addinivalue_line(
|
||||||
|
"markers",
|
||||||
|
"postgres: mark test as requiring a live Postgres instance (SNIPE_SHARED_DB_URL must be set)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def postgres_dsn():
|
||||||
|
dsn = os.environ.get("SNIPE_SHARED_DB_URL")
|
||||||
|
if not dsn:
|
||||||
|
pytest.skip("SNIPE_SHARED_DB_URL not set — skipping Postgres tests")
|
||||||
|
return dsn
|
||||||
108
tests/db/test_pg_shared.py
Normal file
108
tests/db/test_pg_shared.py
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
"""Tests for SnipeSharedStore — requires live Postgres via SNIPE_SHARED_DB_URL."""
|
||||||
|
import pytest
|
||||||
|
from app.db.models import MarketComp, Seller
|
||||||
|
from app.db.pg_shared import SnipeSharedDB, SnipeSharedStore
|
||||||
|
from app.db.protocol import SharedTableProtocol
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.postgres
|
||||||
|
def test_snipe_shared_store_satisfies_protocol(postgres_dsn):
|
||||||
|
assert issubclass(SnipeSharedStore, SharedTableProtocol)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.postgres
|
||||||
|
def test_save_and_get_seller(postgres_dsn):
|
||||||
|
db = SnipeSharedDB(postgres_dsn)
|
||||||
|
db.run_migrations()
|
||||||
|
store = SnipeSharedStore(db)
|
||||||
|
|
||||||
|
seller = Seller(
|
||||||
|
platform="ebay",
|
||||||
|
platform_seller_id="test-seller-001",
|
||||||
|
username="testseller",
|
||||||
|
account_age_days=365,
|
||||||
|
feedback_count=100,
|
||||||
|
feedback_ratio=0.99,
|
||||||
|
category_history_json='{"electronics": 5}',
|
||||||
|
)
|
||||||
|
store.save_seller(seller)
|
||||||
|
|
||||||
|
result = store.get_seller("ebay", "test-seller-001")
|
||||||
|
assert result is not None
|
||||||
|
assert result.username == "testseller"
|
||||||
|
assert result.feedback_count == 100
|
||||||
|
|
||||||
|
store.delete_seller_data("ebay", "test-seller-001")
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.postgres
|
||||||
|
def test_save_sellers_coalesce_preserves_age(postgres_dsn):
|
||||||
|
db = SnipeSharedDB(postgres_dsn)
|
||||||
|
db.run_migrations()
|
||||||
|
store = SnipeSharedStore(db)
|
||||||
|
|
||||||
|
seller_with_age = Seller(
|
||||||
|
platform="ebay", platform_seller_id="coalesce-test",
|
||||||
|
username="u", account_age_days=730,
|
||||||
|
feedback_count=50, feedback_ratio=0.95, category_history_json="{}",
|
||||||
|
)
|
||||||
|
store.save_seller(seller_with_age)
|
||||||
|
|
||||||
|
seller_without_age = Seller(
|
||||||
|
platform="ebay", platform_seller_id="coalesce-test",
|
||||||
|
username="u", account_age_days=None,
|
||||||
|
feedback_count=60, feedback_ratio=0.96, category_history_json="{}",
|
||||||
|
)
|
||||||
|
store.save_sellers([seller_without_age])
|
||||||
|
|
||||||
|
result = store.get_seller("ebay", "coalesce-test")
|
||||||
|
assert result.account_age_days == 730
|
||||||
|
assert result.feedback_count == 60
|
||||||
|
|
||||||
|
store.delete_seller_data("ebay", "coalesce-test")
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.postgres
|
||||||
|
def test_market_comp_cache(postgres_dsn):
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
db = SnipeSharedDB(postgres_dsn)
|
||||||
|
db.run_migrations()
|
||||||
|
store = SnipeSharedStore(db)
|
||||||
|
|
||||||
|
expires = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat()
|
||||||
|
comp = MarketComp(
|
||||||
|
platform="ebay", query_hash="abc123",
|
||||||
|
median_price=49.99, sample_count=10, expires_at=expires,
|
||||||
|
)
|
||||||
|
store.save_market_comp(comp)
|
||||||
|
|
||||||
|
result = store.get_market_comp("ebay", "abc123")
|
||||||
|
assert result is not None
|
||||||
|
assert result.median_price == 49.99
|
||||||
|
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.postgres
|
||||||
|
def test_reported_sellers(postgres_dsn):
|
||||||
|
db = SnipeSharedDB(postgres_dsn)
|
||||||
|
db.run_migrations()
|
||||||
|
store = SnipeSharedStore(db)
|
||||||
|
|
||||||
|
store.mark_reported("ebay", "bad-seller-99", username="badguy")
|
||||||
|
reported = store.list_reported("ebay")
|
||||||
|
assert "bad-seller-99" in reported
|
||||||
|
|
||||||
|
store.mark_reported("ebay", "bad-seller-99") # idempotent
|
||||||
|
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.postgres
|
||||||
|
def test_clone_returns_self(postgres_dsn):
|
||||||
|
db = SnipeSharedDB(postgres_dsn)
|
||||||
|
store = SnipeSharedStore(db)
|
||||||
|
assert store.clone() is store
|
||||||
|
db.close()
|
||||||
Loading…
Reference in a new issue