feat(e2e): add smoke + interaction tests; fix two demo mode errors

- Add tests/e2e/test_smoke.py: page-load error check for all pages
- Add tests/e2e/test_interactions.py: click every interactable, diff
  errors, XFAIL expected demo failures, flag regressions as XPASS
- Fix conftest get_page_errors() to use text_content() instead of
  inner_text() so errors inside collapsed expanders are captured with
  their actual message text (inner_text respects CSS display:none)
- Fix tests/e2e/modes/demo.py base_url to include /peregrine path prefix
  (STREAMLIT_SERVER_BASE_URL_PATH=peregrine set in demo container)

App fixes surfaced by the harness:
- task_runner.py: add DEMO_MODE guard for discovery task — previously
  crashed with FileNotFoundError on search_profiles.yaml before any
  demo guard could fire; now returns friendly error immediately
- 6_Interview_Prep.py: stop auto-triggering LLM session on page load
  in demo mode; show "AI features disabled" info instead, preventing
  a silent st.error() inside the collapsed Practice Q&A expander

Both smoke and interaction tests now pass clean against demo mode.
This commit is contained in:
pyr0ball 2026-03-17 07:00:54 -07:00
parent a55e09d30b
commit 5d14542142
6 changed files with 205 additions and 4 deletions

View file

@ -224,7 +224,11 @@ with col_prep:
st.markdown(msg["content"]) st.markdown(msg["content"])
# Initial question if session is empty # Initial question if session is empty
if not st.session_state[qa_key]: import os as _os
_is_demo = _os.environ.get("DEMO_MODE", "").lower() in ("1", "true", "yes")
if not st.session_state[qa_key] and _is_demo:
st.info("AI features are disabled in the public demo. Run your own instance to use Practice Q&A.")
elif not st.session_state[qa_key]:
with st.spinner("Setting up your mock interview…"): with st.spinner("Setting up your mock interview…"):
try: try:
from scripts.llm_router import complete from scripts.llm_router import complete

View file

@ -150,6 +150,13 @@ def _run_task(db_path: Path, task_id: int, task_type: str, job_id: int,
try: try:
if task_type == "discovery": if task_type == "discovery":
import os as _os
if _os.environ.get("DEMO_MODE", "").lower() in ("1", "true", "yes"):
update_task_status(
db_path, task_id, "failed",
error="Discovery is disabled in the public demo. Run your own instance to use this feature.",
)
return
from scripts.discover import run_discovery from scripts.discover import run_discovery
new_count = run_discovery(db_path) new_count = run_discovery(db_path)
n = new_count or 0 n = new_count or 0

View file

@ -131,9 +131,11 @@ def get_page_errors(page) -> list[ErrorRecord]:
errors: list[ErrorRecord] = [] errors: list[ErrorRecord] = []
for el in page.query_selector_all('[data-testid="stException"]'): for el in page.query_selector_all('[data-testid="stException"]'):
# text_content() includes text from CSS-hidden elements (e.g. collapsed expanders)
msg = (el.text_content() or "").strip()[:500]
errors.append(ErrorRecord( errors.append(ErrorRecord(
type="exception", type="exception",
message=el.inner_text()[:500], message=msg,
element_html=el.inner_html()[:1000], element_html=el.inner_html()[:1000],
)) ))
@ -141,9 +143,10 @@ def get_page_errors(page) -> list[ErrorRecord]:
# Streamlit 1.35+: st.error() renders child [data-testid="stAlertContentError"] # Streamlit 1.35+: st.error() renders child [data-testid="stAlertContentError"]
# kind is a React prop — NOT a DOM attribute. Child detection is authoritative. # kind is a React prop — NOT a DOM attribute. Child detection is authoritative.
if el.query_selector('[data-testid="stAlertContentError"]'): if el.query_selector('[data-testid="stAlertContentError"]'):
msg = (el.text_content() or "").strip()[:500]
errors.append(ErrorRecord( errors.append(ErrorRecord(
type="alert", type="alert",
message=el.inner_text()[:500], message=msg,
element_html=el.inner_html()[:1000], element_html=el.inner_html()[:1000],
)) ))

View file

@ -9,7 +9,7 @@ _BASE_SETTINGS_TABS = [
DEMO = ModeConfig( DEMO = ModeConfig(
name="demo", name="demo",
base_url="http://localhost:8504", base_url="http://localhost:8504/peregrine",
auth_setup=lambda ctx: None, auth_setup=lambda ctx: None,
expected_failures=[ expected_failures=[
"Fetch*", "Fetch*",

View file

@ -0,0 +1,126 @@
"""
Interaction pass discover every interactable element on each page, click it,
diff errors before/after. Demo mode XFAIL patterns are checked; unexpected passes
are flagged as regressions.
Run: pytest tests/e2e/test_interactions.py --mode=demo -v
"""
from __future__ import annotations
import pytest
from tests.e2e.conftest import (
wait_for_streamlit, get_page_errors, screenshot_on_fail,
)
from tests.e2e.models import ModeConfig, diff_errors
from tests.e2e.pages.home_page import HomePage
from tests.e2e.pages.job_review_page import JobReviewPage
from tests.e2e.pages.apply_page import ApplyPage
from tests.e2e.pages.interviews_page import InterviewsPage
from tests.e2e.pages.interview_prep_page import InterviewPrepPage
from tests.e2e.pages.survey_page import SurveyPage
from tests.e2e.pages.settings_page import SettingsPage
PAGE_CLASSES = [
HomePage, JobReviewPage, ApplyPage, InterviewsPage,
InterviewPrepPage, SurveyPage, SettingsPage,
]
@pytest.mark.e2e
def test_interactions_all_pages(active_modes, mode_contexts, playwright):
"""
For each active mode and page: click every discovered interactable,
diff errors, XFAIL expected demo failures, FAIL on unexpected errors.
XPASS (expected failure that didn't fail) is also reported.
"""
failures: list[str] = []
xfails: list[str] = []
xpasses: list[str] = []
for mode in active_modes:
ctx = mode_contexts[mode.name]
page = ctx.new_page()
console_msgs: list = []
page.on("console", lambda msg: console_msgs.append(msg))
page.goto(mode.base_url)
wait_for_streamlit(page)
for PageClass in PAGE_CLASSES:
pg = PageClass(page, mode, console_msgs)
pg.navigate()
elements = pg.discover_interactables()
for element in elements:
pg.navigate()
before = pg.get_errors()
try:
all_matches = page.query_selector_all(element.selector)
content_matches = [
el for el in all_matches
if not el.evaluate(
"el => el.closest('[data-testid=\"stSidebar\"]') !== null"
)
]
if element.index < len(content_matches):
content_matches[element.index].click()
else:
continue
except Exception as e:
failures.append(
f"[{mode.name}] {PageClass.nav_label} / '{element.label}'"
f"could not interact: {e}"
)
continue
wait_for_streamlit(page)
after = pg.get_errors()
new_errors = diff_errors(before, after)
is_expected = mode.matches_expected_failure(element.label)
if new_errors:
if is_expected:
xfails.append(
f"[{mode.name}] {PageClass.nav_label} / '{element.label}' "
f"(expected) — {new_errors[0].message[:120]}"
)
else:
shot = screenshot_on_fail(
page, mode.name,
f"interact_{PageClass.__name__}_{element.label[:30]}"
)
failures.append(
f"[{mode.name}] {PageClass.nav_label} / '{element.label}'"
f"unexpected error: {new_errors[0].message[:200]}\n screenshot: {shot}"
)
else:
if is_expected:
xpasses.append(
f"[{mode.name}] {PageClass.nav_label} / '{element.label}' "
f"— expected to fail but PASSED (neutering guard may be broken!)"
)
page.close()
report_lines = []
if xfails:
report_lines.append(f"XFAIL ({len(xfails)} expected failures, demo mode working correctly):")
report_lines.extend(f" {x}" for x in xfails)
if xpasses:
report_lines.append(f"\nXPASS — REGRESSION ({len(xpasses)} neutering guards broken!):")
report_lines.extend(f" {x}" for x in xpasses)
if failures:
report_lines.append(f"\nFAIL ({len(failures)} unexpected errors):")
report_lines.extend(f" {x}" for x in failures)
if report_lines:
print("\n\n=== E2E Interaction Report ===\n" + "\n".join(report_lines))
if xpasses or failures:
pytest.fail(
f"{len(failures)} unexpected error(s), {len(xpasses)} xpass regression(s). "
"See report above."
)

61
tests/e2e/test_smoke.py Normal file
View file

@ -0,0 +1,61 @@
"""
Smoke pass navigate each page, wait for Streamlit to settle, assert no errors on load.
Errors on page load are always real bugs (not mode-specific).
Run: pytest tests/e2e/test_smoke.py --mode=demo
"""
from __future__ import annotations
import pytest
from tests.e2e.conftest import wait_for_streamlit, get_page_errors, get_console_errors, screenshot_on_fail
from tests.e2e.models import ModeConfig
from tests.e2e.pages.home_page import HomePage
from tests.e2e.pages.job_review_page import JobReviewPage
from tests.e2e.pages.apply_page import ApplyPage
from tests.e2e.pages.interviews_page import InterviewsPage
from tests.e2e.pages.interview_prep_page import InterviewPrepPage
from tests.e2e.pages.survey_page import SurveyPage
from tests.e2e.pages.settings_page import SettingsPage
PAGE_CLASSES = [
HomePage, JobReviewPage, ApplyPage, InterviewsPage,
InterviewPrepPage, SurveyPage, SettingsPage,
]
@pytest.mark.e2e
def test_smoke_all_pages(active_modes, mode_contexts, playwright):
"""For each active mode: navigate to every page and assert no errors on load."""
failures: list[str] = []
for mode in active_modes:
ctx = mode_contexts[mode.name]
page = ctx.new_page()
console_msgs: list = []
page.on("console", lambda msg: console_msgs.append(msg))
page.goto(mode.base_url)
wait_for_streamlit(page)
for PageClass in PAGE_CLASSES:
pg = PageClass(page, mode, console_msgs)
pg.navigate()
console_msgs.clear()
dom_errors = pg.get_errors()
console_errors = pg.get_console_errors()
if dom_errors or console_errors:
shot_path = screenshot_on_fail(page, mode.name, f"smoke_{PageClass.__name__}")
detail = "\n".join(
[f" DOM: {e.message}" for e in dom_errors]
+ [f" Console: {e}" for e in console_errors]
)
failures.append(
f"[{mode.name}] {PageClass.nav_label} — errors on load:\n{detail}\n screenshot: {shot_path}"
)
page.close()
if failures:
pytest.fail("Smoke test failures:\n\n" + "\n\n".join(failures))