fix(linkedin): improve scraper error handling, current-job date range, add missing tests
This commit is contained in:
parent
32ed451933
commit
04d0a66f21
2 changed files with 110 additions and 60 deletions
|
|
@ -65,6 +65,8 @@ def scrape_profile(url: str, stage_path: Path) -> dict:
|
|||
"LinkedIn did not load in time — the request may have been blocked. "
|
||||
"Try the data export option instead."
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"LinkedIn scrape failed: {e}") from e
|
||||
|
||||
extracted = parse_html(raw_html)
|
||||
extracted["linkedin"] = url
|
||||
|
|
@ -94,11 +96,7 @@ def parse_export_zip(zip_bytes: bytes, stage_path: Path) -> dict:
|
|||
}
|
||||
|
||||
try:
|
||||
zf_handle = zipfile.ZipFile(io.BytesIO(zip_bytes))
|
||||
except zipfile.BadZipFile as e:
|
||||
raise ValueError(f"Not a valid zip file: {e}")
|
||||
|
||||
with zf_handle as zf:
|
||||
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
|
||||
names_in_zip = {n.lower(): n for n in zf.namelist()}
|
||||
|
||||
def _read_csv(filename: str) -> list[dict]:
|
||||
|
|
@ -122,7 +120,8 @@ def parse_export_zip(zip_bytes: bytes, stage_path: Path) -> dict:
|
|||
desc = row.get("Description", "").strip()
|
||||
start = row.get("Started On", "").strip()
|
||||
end = row.get("Finished On", "").strip()
|
||||
date_range = f"{start} – {end}".strip(" –") if start or end else ""
|
||||
end_label = end if end else ("Present" if start else "")
|
||||
date_range = f"{start} – {end_label}".strip(" –") if (start or end) else ""
|
||||
bullets = [d.strip() for d in re.split(r"[.•\n]+", desc) if d.strip() and len(d.strip()) > 3]
|
||||
if company or title:
|
||||
extracted["experience"].append({
|
||||
|
|
@ -157,6 +156,9 @@ def parse_export_zip(zip_bytes: bytes, stage_path: Path) -> dict:
|
|||
if name:
|
||||
extracted["achievements"].append(name)
|
||||
|
||||
except zipfile.BadZipFile as e:
|
||||
raise ValueError(f"Not a valid zip file: {e}")
|
||||
|
||||
_write_stage(stage_path, {
|
||||
"url": None,
|
||||
"scraped_at": datetime.now(timezone.utc).isoformat(),
|
||||
|
|
|
|||
|
|
@ -163,3 +163,51 @@ def test_parse_export_zip_writes_staging_file():
|
|||
data = json.loads(stage.read_text())
|
||||
assert data["source"] == "export_zip"
|
||||
assert data["raw_html"] is None
|
||||
|
||||
|
||||
def test_scrape_profile_sets_linkedin_url():
|
||||
from scripts.linkedin_scraper import scrape_profile
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
stage = Path(tmp) / "stage.json"
|
||||
fixture_html = (Path(__file__).parent / "fixtures" / "linkedin_profile.html").read_text()
|
||||
mock_page = MagicMock()
|
||||
mock_page.content.return_value = fixture_html
|
||||
mock_browser = MagicMock()
|
||||
mock_browser.new_page.return_value = mock_page
|
||||
mock_playwright = MagicMock()
|
||||
mock_playwright.chromium.launch.return_value = mock_browser
|
||||
with patch("scripts.linkedin_scraper.sync_playwright") as mock_sync_pw:
|
||||
mock_sync_pw.return_value.__enter__ = MagicMock(return_value=mock_playwright)
|
||||
mock_sync_pw.return_value.__exit__ = MagicMock(return_value=False)
|
||||
result = scrape_profile("https://linkedin.com/in/alanw", stage)
|
||||
assert result["linkedin"] == "https://linkedin.com/in/alanw"
|
||||
|
||||
|
||||
def test_parse_export_zip_bad_zip_raises():
|
||||
from scripts.linkedin_scraper import parse_export_zip
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
stage = Path(tmp) / "stage.json"
|
||||
try:
|
||||
parse_export_zip(b"not a zip file at all", stage)
|
||||
assert False, "should have raised"
|
||||
except ValueError as e:
|
||||
assert "zip" in str(e).lower()
|
||||
|
||||
|
||||
def test_parse_export_zip_current_job_shows_present():
|
||||
"""Empty Finished On renders as '– Present', not truncated."""
|
||||
from scripts.linkedin_scraper import parse_export_zip
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
zf.writestr("Position.csv",
|
||||
"Company Name,Title,Description,Started On,Finished On\n"
|
||||
"Acme Corp,Staff Engineer,,Jan 2022,\n"
|
||||
)
|
||||
zf.writestr("Profile.csv",
|
||||
"First Name,Last Name,Headline,Summary,Email Address\n"
|
||||
"Alan,Weinstock,Engineer,,\n"
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
stage = Path(tmp) / "stage.json"
|
||||
result = parse_export_zip(buf.getvalue(), stage)
|
||||
assert result["experience"][0]["date_range"] == "Jan 2022 – Present"
|
||||
|
|
|
|||
Loading…
Reference in a new issue