feat(e2e): add smoke + interaction tests; fix two demo mode errors
- Add tests/e2e/test_smoke.py: page-load error check for all pages - Add tests/e2e/test_interactions.py: click every interactable, diff errors, XFAIL expected demo failures, flag regressions as XPASS - Fix conftest get_page_errors() to use text_content() instead of inner_text() so errors inside collapsed expanders are captured with their actual message text (inner_text respects CSS display:none) - Fix tests/e2e/modes/demo.py base_url to include /peregrine path prefix (STREAMLIT_SERVER_BASE_URL_PATH=peregrine set in demo container) App fixes surfaced by the harness: - task_runner.py: add DEMO_MODE guard for discovery task — previously crashed with FileNotFoundError on search_profiles.yaml before any demo guard could fire; now returns friendly error immediately - 6_Interview_Prep.py: stop auto-triggering LLM session on page load in demo mode; show "AI features disabled" info instead, preventing a silent st.error() inside the collapsed Practice Q&A expander Both smoke and interaction tests now pass clean against demo mode.
This commit is contained in:
parent
a55e09d30b
commit
5d14542142
6 changed files with 205 additions and 4 deletions
|
|
@ -224,7 +224,11 @@ with col_prep:
|
|||
st.markdown(msg["content"])
|
||||
|
||||
# Initial question if session is empty
|
||||
if not st.session_state[qa_key]:
|
||||
import os as _os
|
||||
_is_demo = _os.environ.get("DEMO_MODE", "").lower() in ("1", "true", "yes")
|
||||
if not st.session_state[qa_key] and _is_demo:
|
||||
st.info("AI features are disabled in the public demo. Run your own instance to use Practice Q&A.")
|
||||
elif not st.session_state[qa_key]:
|
||||
with st.spinner("Setting up your mock interview…"):
|
||||
try:
|
||||
from scripts.llm_router import complete
|
||||
|
|
|
|||
|
|
@ -150,6 +150,13 @@ def _run_task(db_path: Path, task_id: int, task_type: str, job_id: int,
|
|||
|
||||
try:
|
||||
if task_type == "discovery":
|
||||
import os as _os
|
||||
if _os.environ.get("DEMO_MODE", "").lower() in ("1", "true", "yes"):
|
||||
update_task_status(
|
||||
db_path, task_id, "failed",
|
||||
error="Discovery is disabled in the public demo. Run your own instance to use this feature.",
|
||||
)
|
||||
return
|
||||
from scripts.discover import run_discovery
|
||||
new_count = run_discovery(db_path)
|
||||
n = new_count or 0
|
||||
|
|
|
|||
|
|
@ -131,9 +131,11 @@ def get_page_errors(page) -> list[ErrorRecord]:
|
|||
errors: list[ErrorRecord] = []
|
||||
|
||||
for el in page.query_selector_all('[data-testid="stException"]'):
|
||||
# text_content() includes text from CSS-hidden elements (e.g. collapsed expanders)
|
||||
msg = (el.text_content() or "").strip()[:500]
|
||||
errors.append(ErrorRecord(
|
||||
type="exception",
|
||||
message=el.inner_text()[:500],
|
||||
message=msg,
|
||||
element_html=el.inner_html()[:1000],
|
||||
))
|
||||
|
||||
|
|
@ -141,9 +143,10 @@ def get_page_errors(page) -> list[ErrorRecord]:
|
|||
# Streamlit 1.35+: st.error() renders child [data-testid="stAlertContentError"]
|
||||
# kind is a React prop — NOT a DOM attribute. Child detection is authoritative.
|
||||
if el.query_selector('[data-testid="stAlertContentError"]'):
|
||||
msg = (el.text_content() or "").strip()[:500]
|
||||
errors.append(ErrorRecord(
|
||||
type="alert",
|
||||
message=el.inner_text()[:500],
|
||||
message=msg,
|
||||
element_html=el.inner_html()[:1000],
|
||||
))
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ _BASE_SETTINGS_TABS = [
|
|||
|
||||
DEMO = ModeConfig(
|
||||
name="demo",
|
||||
base_url="http://localhost:8504",
|
||||
base_url="http://localhost:8504/peregrine",
|
||||
auth_setup=lambda ctx: None,
|
||||
expected_failures=[
|
||||
"Fetch*",
|
||||
|
|
|
|||
126
tests/e2e/test_interactions.py
Normal file
126
tests/e2e/test_interactions.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""
|
||||
Interaction pass — discover every interactable element on each page, click it,
|
||||
diff errors before/after. Demo mode XFAIL patterns are checked; unexpected passes
|
||||
are flagged as regressions.
|
||||
|
||||
Run: pytest tests/e2e/test_interactions.py --mode=demo -v
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import pytest
|
||||
|
||||
from tests.e2e.conftest import (
|
||||
wait_for_streamlit, get_page_errors, screenshot_on_fail,
|
||||
)
|
||||
from tests.e2e.models import ModeConfig, diff_errors
|
||||
from tests.e2e.pages.home_page import HomePage
|
||||
from tests.e2e.pages.job_review_page import JobReviewPage
|
||||
from tests.e2e.pages.apply_page import ApplyPage
|
||||
from tests.e2e.pages.interviews_page import InterviewsPage
|
||||
from tests.e2e.pages.interview_prep_page import InterviewPrepPage
|
||||
from tests.e2e.pages.survey_page import SurveyPage
|
||||
from tests.e2e.pages.settings_page import SettingsPage
|
||||
|
||||
PAGE_CLASSES = [
|
||||
HomePage, JobReviewPage, ApplyPage, InterviewsPage,
|
||||
InterviewPrepPage, SurveyPage, SettingsPage,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
def test_interactions_all_pages(active_modes, mode_contexts, playwright):
|
||||
"""
|
||||
For each active mode and page: click every discovered interactable,
|
||||
diff errors, XFAIL expected demo failures, FAIL on unexpected errors.
|
||||
XPASS (expected failure that didn't fail) is also reported.
|
||||
"""
|
||||
failures: list[str] = []
|
||||
xfails: list[str] = []
|
||||
xpasses: list[str] = []
|
||||
|
||||
for mode in active_modes:
|
||||
ctx = mode_contexts[mode.name]
|
||||
page = ctx.new_page()
|
||||
console_msgs: list = []
|
||||
page.on("console", lambda msg: console_msgs.append(msg))
|
||||
|
||||
page.goto(mode.base_url)
|
||||
wait_for_streamlit(page)
|
||||
|
||||
for PageClass in PAGE_CLASSES:
|
||||
pg = PageClass(page, mode, console_msgs)
|
||||
pg.navigate()
|
||||
|
||||
elements = pg.discover_interactables()
|
||||
|
||||
for element in elements:
|
||||
pg.navigate()
|
||||
before = pg.get_errors()
|
||||
|
||||
try:
|
||||
all_matches = page.query_selector_all(element.selector)
|
||||
content_matches = [
|
||||
el for el in all_matches
|
||||
if not el.evaluate(
|
||||
"el => el.closest('[data-testid=\"stSidebar\"]') !== null"
|
||||
)
|
||||
]
|
||||
if element.index < len(content_matches):
|
||||
content_matches[element.index].click()
|
||||
else:
|
||||
continue
|
||||
except Exception as e:
|
||||
failures.append(
|
||||
f"[{mode.name}] {PageClass.nav_label} / '{element.label}' — "
|
||||
f"could not interact: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
wait_for_streamlit(page)
|
||||
after = pg.get_errors()
|
||||
new_errors = diff_errors(before, after)
|
||||
|
||||
is_expected = mode.matches_expected_failure(element.label)
|
||||
|
||||
if new_errors:
|
||||
if is_expected:
|
||||
xfails.append(
|
||||
f"[{mode.name}] {PageClass.nav_label} / '{element.label}' "
|
||||
f"(expected) — {new_errors[0].message[:120]}"
|
||||
)
|
||||
else:
|
||||
shot = screenshot_on_fail(
|
||||
page, mode.name,
|
||||
f"interact_{PageClass.__name__}_{element.label[:30]}"
|
||||
)
|
||||
failures.append(
|
||||
f"[{mode.name}] {PageClass.nav_label} / '{element.label}' — "
|
||||
f"unexpected error: {new_errors[0].message[:200]}\n screenshot: {shot}"
|
||||
)
|
||||
else:
|
||||
if is_expected:
|
||||
xpasses.append(
|
||||
f"[{mode.name}] {PageClass.nav_label} / '{element.label}' "
|
||||
f"— expected to fail but PASSED (neutering guard may be broken!)"
|
||||
)
|
||||
|
||||
page.close()
|
||||
|
||||
report_lines = []
|
||||
if xfails:
|
||||
report_lines.append(f"XFAIL ({len(xfails)} expected failures, demo mode working correctly):")
|
||||
report_lines.extend(f" {x}" for x in xfails)
|
||||
if xpasses:
|
||||
report_lines.append(f"\nXPASS — REGRESSION ({len(xpasses)} neutering guards broken!):")
|
||||
report_lines.extend(f" {x}" for x in xpasses)
|
||||
if failures:
|
||||
report_lines.append(f"\nFAIL ({len(failures)} unexpected errors):")
|
||||
report_lines.extend(f" {x}" for x in failures)
|
||||
|
||||
if report_lines:
|
||||
print("\n\n=== E2E Interaction Report ===\n" + "\n".join(report_lines))
|
||||
|
||||
if xpasses or failures:
|
||||
pytest.fail(
|
||||
f"{len(failures)} unexpected error(s), {len(xpasses)} xpass regression(s). "
|
||||
"See report above."
|
||||
)
|
||||
61
tests/e2e/test_smoke.py
Normal file
61
tests/e2e/test_smoke.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
"""
|
||||
Smoke pass — navigate each page, wait for Streamlit to settle, assert no errors on load.
|
||||
Errors on page load are always real bugs (not mode-specific).
|
||||
|
||||
Run: pytest tests/e2e/test_smoke.py --mode=demo
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import pytest
|
||||
|
||||
from tests.e2e.conftest import wait_for_streamlit, get_page_errors, get_console_errors, screenshot_on_fail
|
||||
from tests.e2e.models import ModeConfig
|
||||
from tests.e2e.pages.home_page import HomePage
|
||||
from tests.e2e.pages.job_review_page import JobReviewPage
|
||||
from tests.e2e.pages.apply_page import ApplyPage
|
||||
from tests.e2e.pages.interviews_page import InterviewsPage
|
||||
from tests.e2e.pages.interview_prep_page import InterviewPrepPage
|
||||
from tests.e2e.pages.survey_page import SurveyPage
|
||||
from tests.e2e.pages.settings_page import SettingsPage
|
||||
|
||||
PAGE_CLASSES = [
|
||||
HomePage, JobReviewPage, ApplyPage, InterviewsPage,
|
||||
InterviewPrepPage, SurveyPage, SettingsPage,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
def test_smoke_all_pages(active_modes, mode_contexts, playwright):
|
||||
"""For each active mode: navigate to every page and assert no errors on load."""
|
||||
failures: list[str] = []
|
||||
|
||||
for mode in active_modes:
|
||||
ctx = mode_contexts[mode.name]
|
||||
page = ctx.new_page()
|
||||
console_msgs: list = []
|
||||
page.on("console", lambda msg: console_msgs.append(msg))
|
||||
|
||||
page.goto(mode.base_url)
|
||||
wait_for_streamlit(page)
|
||||
|
||||
for PageClass in PAGE_CLASSES:
|
||||
pg = PageClass(page, mode, console_msgs)
|
||||
pg.navigate()
|
||||
console_msgs.clear()
|
||||
|
||||
dom_errors = pg.get_errors()
|
||||
console_errors = pg.get_console_errors()
|
||||
|
||||
if dom_errors or console_errors:
|
||||
shot_path = screenshot_on_fail(page, mode.name, f"smoke_{PageClass.__name__}")
|
||||
detail = "\n".join(
|
||||
[f" DOM: {e.message}" for e in dom_errors]
|
||||
+ [f" Console: {e}" for e in console_errors]
|
||||
)
|
||||
failures.append(
|
||||
f"[{mode.name}] {PageClass.nav_label} — errors on load:\n{detail}\n screenshot: {shot_path}"
|
||||
)
|
||||
|
||||
page.close()
|
||||
|
||||
if failures:
|
||||
pytest.fail("Smoke test failures:\n\n" + "\n\n".join(failures))
|
||||
Loading…
Reference in a new issue