discovery: partial onboarding when the homepage is bot-blocked (robots/sitemaps/probes/subdomains still create sensors)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +31 −2
modified
src/companyatlas/services/discovery.py
+31 −2
@@ -544,13 +544,42 @@ async def discover_company(company: dict[str, Any], *, fetcher: Fetcher, dry_run | |||||
| 544 | return res | 544 | return res | ||
| 545 | 545 | ||||
| 546 | 546 | ||||
| 547 | + | ||||
| 548 | +async def _discover_without_homepage(company: dict[str, Any], fetcher: Fetcher, res: DiscoveryResult, budget: _Budget, canonical: str, *, fetch_now: bool) -> None: | ||||
| 549 | + base = f"https://www.{canonical}" if not canonical.startswith("www.") else f"https://{canonical}" | ||||
| 550 | + res.canonical_domain = canonical | ||||
| 551 | + res.notes.append("homepage blocked — partial onboarding from robots/sitemaps/probes/subdomains") | ||||
| 552 | + cands: list[Candidate] = [] | ||||
| 553 | + if budget.ok(10): | ||||
| 554 | + cands.extend(await _phase_robots_sitemaps(fetcher, base, res, budget, canonical)) | ||||
| 555 | + have = {c.surface for c in cands if c.confidence >= 0.7} | ||||
| 556 | + if budget.ok(10): | ||||
| 557 | + cands.extend(await _phase_probes(fetcher, base, have, res, budget, canonical)) | ||||
| 558 | + have = {c.surface for c in cands if c.confidence >= 0.7} | ||||
| 559 | + if budget.ok(10): | ||||
| 560 | + cands.extend(await _phase_subdomains(fetcher, canonical, have, res, budget)) | ||||
| 561 | + res.candidates = cands | ||||
| 562 | + res.sensors = select_sensors(cands, company={**company, "canonical_domain": canonical}, canonical_domain=canonical, now=datetime.now(UTC), | ||||
| 563 | + fetch_now=fetch_now, home_url=base) | ||||
| 564 | + if res.sensors: | ||||
| 565 | + res.status = OnboardingStatus.ACTIVE | ||||
| 566 | + res.notes.append(f"partial: {len(res.sensors)} sensors without homepage") | ||||
| 567 | + else: | ||||
| 568 | + res.status = OnboardingStatus.FAILED | ||||
| 569 | + res.error = res.error or "homepage blocked and no reachable surface" | ||||
| 570 | + | ||||
| 571 | + | ||||
| 547 | async def _discover(company: dict[str, Any], fetcher: Fetcher, res: DiscoveryResult, budget: _Budget, *, fetch_now: bool) -> None: | 572 | async def _discover(company: dict[str, Any], fetcher: Fetcher, res: DiscoveryResult, budget: _Budget, *, fetch_now: bool) -> None: | ||
| 548 | canonical = str(company.get("canonical_domain") or "").lower() | 573 | canonical = str(company.get("canonical_domain") or "").lower() | ||
| 549 | home, extraction = await _phase_homepage(fetcher, company, res, budget) | 574 | home, extraction = await _phase_homepage(fetcher, company, res, budget) | ||
| 550 | if home is None: | 575 | if home is None: | ||
| 551 | err = (res.error or "").upper() | 576 | err = (res.error or "").upper() | ||
| 552 | − res.status = OnboardingStatus.NO_WEBSITE if ("DNS" in err or "BLOCKED_DESTINATION" in err or not canonical) else OnboardingStatus.FAILED | 577 | + if "DNS" in err or "BLOCKED_DESTINATION" in err or not canonical: | ||
| 553 | − res.error = res.error or "homepage unreachable" | 578 | + res.status, res.error = OnboardingStatus.NO_WEBSITE, res.error or "homepage unreachable" | ||
| 579 | + return | ||||
| 580 | + # Homepage blocked (anti-bot 403, timeout, 5xx…): many such sites still expose robots/sitemaps, careers., investors., news. | ||||
| 581 | + # subdomains or plain paths. Partial onboarding keeps the company observable instead of failing it outright (spec §60, §190). | ||||
| 582 | + await _discover_without_homepage(company, fetcher, res, budget, canonical, fetch_now=fetch_now) | ||||
| 554 | return | 583 | return | ||
| 555 | res.final_url = home.final_url | 584 | res.final_url = home.final_url | ||
| 556 | final_dom = registrable_domain(home.final_url) | 585 | final_dom = registrable_domain(home.final_url) | ||
| 557 | 586 | ||||