From 5d1454214259687799dbe4e2c0b2d36b90ee6b7e Mon Sep 17 00:00:00 2001 From: pyr0ball Date: Tue, 17 Mar 2026 07:00:54 -0700 Subject: [PATCH] feat(e2e): add smoke + interaction tests; fix two demo mode errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add tests/e2e/test_smoke.py: page-load error check for all pages - Add tests/e2e/test_interactions.py: click every interactable, diff errors, XFAIL expected demo failures, flag regressions as XPASS - Fix conftest get_page_errors() to use text_content() instead of inner_text() so errors inside collapsed expanders are captured with their actual message text (inner_text respects CSS display:none) - Fix tests/e2e/modes/demo.py base_url to include /peregrine path prefix (STREAMLIT_SERVER_BASE_URL_PATH=peregrine set in demo container) App fixes surfaced by the harness: - task_runner.py: add DEMO_MODE guard for discovery task — previously crashed with FileNotFoundError on search_profiles.yaml before any demo guard could fire; now returns friendly error immediately - 6_Interview_Prep.py: stop auto-triggering LLM session on page load in demo mode; show "AI features disabled" info instead, preventing a silent st.error() inside the collapsed Practice Q&A expander Both smoke and interaction tests now pass clean against demo mode. --- app/pages/6_Interview_Prep.py | 6 +- scripts/task_runner.py | 7 ++ tests/e2e/conftest.py | 7 +- tests/e2e/modes/demo.py | 2 +- tests/e2e/test_interactions.py | 126 +++++++++++++++++++++++++++++++++ tests/e2e/test_smoke.py | 61 ++++++++++++++++ 6 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/test_interactions.py create mode 100644 tests/e2e/test_smoke.py diff --git a/app/pages/6_Interview_Prep.py b/app/pages/6_Interview_Prep.py index 4f4e0e2..812bdd1 100644 --- a/app/pages/6_Interview_Prep.py +++ b/app/pages/6_Interview_Prep.py @@ -224,7 +224,11 @@ with col_prep: st.markdown(msg["content"]) # Initial question if session is empty - if not st.session_state[qa_key]: + import os as _os + _is_demo = _os.environ.get("DEMO_MODE", "").lower() in ("1", "true", "yes") + if not st.session_state[qa_key] and _is_demo: + st.info("AI features are disabled in the public demo. Run your own instance to use Practice Q&A.") + elif not st.session_state[qa_key]: with st.spinner("Setting up your mock interview…"): try: from scripts.llm_router import complete diff --git a/scripts/task_runner.py b/scripts/task_runner.py index f92b7b7..6bfdd4c 100644 --- a/scripts/task_runner.py +++ b/scripts/task_runner.py @@ -150,6 +150,13 @@ def _run_task(db_path: Path, task_id: int, task_type: str, job_id: int, try: if task_type == "discovery": + import os as _os + if _os.environ.get("DEMO_MODE", "").lower() in ("1", "true", "yes"): + update_task_status( + db_path, task_id, "failed", + error="Discovery is disabled in the public demo. Run your own instance to use this feature.", + ) + return from scripts.discover import run_discovery new_count = run_discovery(db_path) n = new_count or 0 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 9d2d91e..ff0f7a7 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -131,9 +131,11 @@ def get_page_errors(page) -> list[ErrorRecord]: errors: list[ErrorRecord] = [] for el in page.query_selector_all('[data-testid="stException"]'): + # text_content() includes text from CSS-hidden elements (e.g. collapsed expanders) + msg = (el.text_content() or "").strip()[:500] errors.append(ErrorRecord( type="exception", - message=el.inner_text()[:500], + message=msg, element_html=el.inner_html()[:1000], )) @@ -141,9 +143,10 @@ def get_page_errors(page) -> list[ErrorRecord]: # Streamlit 1.35+: st.error() renders child [data-testid="stAlertContentError"] # kind is a React prop — NOT a DOM attribute. Child detection is authoritative. if el.query_selector('[data-testid="stAlertContentError"]'): + msg = (el.text_content() or "").strip()[:500] errors.append(ErrorRecord( type="alert", - message=el.inner_text()[:500], + message=msg, element_html=el.inner_html()[:1000], )) diff --git a/tests/e2e/modes/demo.py b/tests/e2e/modes/demo.py index 4350820..17c7abf 100644 --- a/tests/e2e/modes/demo.py +++ b/tests/e2e/modes/demo.py @@ -9,7 +9,7 @@ _BASE_SETTINGS_TABS = [ DEMO = ModeConfig( name="demo", - base_url="http://localhost:8504", + base_url="http://localhost:8504/peregrine", auth_setup=lambda ctx: None, expected_failures=[ "Fetch*", diff --git a/tests/e2e/test_interactions.py b/tests/e2e/test_interactions.py new file mode 100644 index 0000000..7bc9adb --- /dev/null +++ b/tests/e2e/test_interactions.py @@ -0,0 +1,126 @@ +""" +Interaction pass — discover every interactable element on each page, click it, +diff errors before/after. Demo mode XFAIL patterns are checked; unexpected passes +are flagged as regressions. + +Run: pytest tests/e2e/test_interactions.py --mode=demo -v +""" +from __future__ import annotations +import pytest + +from tests.e2e.conftest import ( + wait_for_streamlit, get_page_errors, screenshot_on_fail, +) +from tests.e2e.models import ModeConfig, diff_errors +from tests.e2e.pages.home_page import HomePage +from tests.e2e.pages.job_review_page import JobReviewPage +from tests.e2e.pages.apply_page import ApplyPage +from tests.e2e.pages.interviews_page import InterviewsPage +from tests.e2e.pages.interview_prep_page import InterviewPrepPage +from tests.e2e.pages.survey_page import SurveyPage +from tests.e2e.pages.settings_page import SettingsPage + +PAGE_CLASSES = [ + HomePage, JobReviewPage, ApplyPage, InterviewsPage, + InterviewPrepPage, SurveyPage, SettingsPage, +] + + +@pytest.mark.e2e +def test_interactions_all_pages(active_modes, mode_contexts, playwright): + """ + For each active mode and page: click every discovered interactable, + diff errors, XFAIL expected demo failures, FAIL on unexpected errors. + XPASS (expected failure that didn't fail) is also reported. + """ + failures: list[str] = [] + xfails: list[str] = [] + xpasses: list[str] = [] + + for mode in active_modes: + ctx = mode_contexts[mode.name] + page = ctx.new_page() + console_msgs: list = [] + page.on("console", lambda msg: console_msgs.append(msg)) + + page.goto(mode.base_url) + wait_for_streamlit(page) + + for PageClass in PAGE_CLASSES: + pg = PageClass(page, mode, console_msgs) + pg.navigate() + + elements = pg.discover_interactables() + + for element in elements: + pg.navigate() + before = pg.get_errors() + + try: + all_matches = page.query_selector_all(element.selector) + content_matches = [ + el for el in all_matches + if not el.evaluate( + "el => el.closest('[data-testid=\"stSidebar\"]') !== null" + ) + ] + if element.index < len(content_matches): + content_matches[element.index].click() + else: + continue + except Exception as e: + failures.append( + f"[{mode.name}] {PageClass.nav_label} / '{element.label}' — " + f"could not interact: {e}" + ) + continue + + wait_for_streamlit(page) + after = pg.get_errors() + new_errors = diff_errors(before, after) + + is_expected = mode.matches_expected_failure(element.label) + + if new_errors: + if is_expected: + xfails.append( + f"[{mode.name}] {PageClass.nav_label} / '{element.label}' " + f"(expected) — {new_errors[0].message[:120]}" + ) + else: + shot = screenshot_on_fail( + page, mode.name, + f"interact_{PageClass.__name__}_{element.label[:30]}" + ) + failures.append( + f"[{mode.name}] {PageClass.nav_label} / '{element.label}' — " + f"unexpected error: {new_errors[0].message[:200]}\n screenshot: {shot}" + ) + else: + if is_expected: + xpasses.append( + f"[{mode.name}] {PageClass.nav_label} / '{element.label}' " + f"— expected to fail but PASSED (neutering guard may be broken!)" + ) + + page.close() + + report_lines = [] + if xfails: + report_lines.append(f"XFAIL ({len(xfails)} expected failures, demo mode working correctly):") + report_lines.extend(f" {x}" for x in xfails) + if xpasses: + report_lines.append(f"\nXPASS — REGRESSION ({len(xpasses)} neutering guards broken!):") + report_lines.extend(f" {x}" for x in xpasses) + if failures: + report_lines.append(f"\nFAIL ({len(failures)} unexpected errors):") + report_lines.extend(f" {x}" for x in failures) + + if report_lines: + print("\n\n=== E2E Interaction Report ===\n" + "\n".join(report_lines)) + + if xpasses or failures: + pytest.fail( + f"{len(failures)} unexpected error(s), {len(xpasses)} xpass regression(s). " + "See report above." + ) diff --git a/tests/e2e/test_smoke.py b/tests/e2e/test_smoke.py new file mode 100644 index 0000000..fc02b8a --- /dev/null +++ b/tests/e2e/test_smoke.py @@ -0,0 +1,61 @@ +""" +Smoke pass — navigate each page, wait for Streamlit to settle, assert no errors on load. +Errors on page load are always real bugs (not mode-specific). + +Run: pytest tests/e2e/test_smoke.py --mode=demo +""" +from __future__ import annotations +import pytest + +from tests.e2e.conftest import wait_for_streamlit, get_page_errors, get_console_errors, screenshot_on_fail +from tests.e2e.models import ModeConfig +from tests.e2e.pages.home_page import HomePage +from tests.e2e.pages.job_review_page import JobReviewPage +from tests.e2e.pages.apply_page import ApplyPage +from tests.e2e.pages.interviews_page import InterviewsPage +from tests.e2e.pages.interview_prep_page import InterviewPrepPage +from tests.e2e.pages.survey_page import SurveyPage +from tests.e2e.pages.settings_page import SettingsPage + +PAGE_CLASSES = [ + HomePage, JobReviewPage, ApplyPage, InterviewsPage, + InterviewPrepPage, SurveyPage, SettingsPage, +] + + +@pytest.mark.e2e +def test_smoke_all_pages(active_modes, mode_contexts, playwright): + """For each active mode: navigate to every page and assert no errors on load.""" + failures: list[str] = [] + + for mode in active_modes: + ctx = mode_contexts[mode.name] + page = ctx.new_page() + console_msgs: list = [] + page.on("console", lambda msg: console_msgs.append(msg)) + + page.goto(mode.base_url) + wait_for_streamlit(page) + + for PageClass in PAGE_CLASSES: + pg = PageClass(page, mode, console_msgs) + pg.navigate() + console_msgs.clear() + + dom_errors = pg.get_errors() + console_errors = pg.get_console_errors() + + if dom_errors or console_errors: + shot_path = screenshot_on_fail(page, mode.name, f"smoke_{PageClass.__name__}") + detail = "\n".join( + [f" DOM: {e.message}" for e in dom_errors] + + [f" Console: {e}" for e in console_errors] + ) + failures.append( + f"[{mode.name}] {PageClass.nav_label} — errors on load:\n{detail}\n screenshot: {shot_path}" + ) + + page.close() + + if failures: + pytest.fail("Smoke test failures:\n\n" + "\n\n".join(failures))