discovery: partial onboarding when the homepage is bot-blocked (robots/sitemaps/probes/subdomains still create sensors)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 11 days ago (Sep 13, 2026) parent 29c7c57

1 changed file +31 −2

modified src/companyatlas/services/discovery.py +31 −2

		`@@ -544,13 +544,42 @@ async def discover_company(company: dict[str, Any], *, fetcher: Fetcher, dry_run`
544	544	return res
545	545
546	546
	547	+
	548	+async def _discover_without_homepage(company: dict[str, Any], fetcher: Fetcher, res: DiscoveryResult, budget: _Budget, canonical: str, *, fetch_now: bool) -> None:
	549	+ base = f"https://www.{canonical}" if not canonical.startswith("www.") else f"https://{canonical}"
	550	+ res.canonical_domain = canonical
	551	+ res.notes.append("homepage blocked — partial onboarding from robots/sitemaps/probes/subdomains")
	552	+ cands: list[Candidate] = []
	553	+ if budget.ok(10):
	554	+ cands.extend(await _phase_robots_sitemaps(fetcher, base, res, budget, canonical))
	555	+ have = {c.surface for c in cands if c.confidence >= 0.7}
	556	+ if budget.ok(10):
	557	+ cands.extend(await _phase_probes(fetcher, base, have, res, budget, canonical))
	558	+ have = {c.surface for c in cands if c.confidence >= 0.7}
	559	+ if budget.ok(10):
	560	+ cands.extend(await _phase_subdomains(fetcher, canonical, have, res, budget))
	561	+ res.candidates = cands
	562	+ res.sensors = select_sensors(cands, company={**company, "canonical_domain": canonical}, canonical_domain=canonical, now=datetime.now(UTC),
	563	+ fetch_now=fetch_now, home_url=base)
	564	+ if res.sensors:
	565	+ res.status = OnboardingStatus.ACTIVE
	566	+ res.notes.append(f"partial: {len(res.sensors)} sensors without homepage")
	567	+ else:
	568	+ res.status = OnboardingStatus.FAILED
	569	+ res.error = res.error or "homepage blocked and no reachable surface"
	570	+
	571	+
547	572	async def _discover(company: dict[str, Any], fetcher: Fetcher, res: DiscoveryResult, budget: _Budget, *, fetch_now: bool) -> None:
548	573	canonical = str(company.get("canonical_domain") or "").lower()
549	574	home, extraction = await _phase_homepage(fetcher, company, res, budget)
550	575	if home is None:
551	576	err = (res.error or "").upper()
552		− res.status = OnboardingStatus.NO_WEBSITE if ("DNS" in err or "BLOCKED_DESTINATION" in err or not canonical) else OnboardingStatus.FAILED
553		− res.error = res.error or "homepage unreachable"
	577	+ if "DNS" in err or "BLOCKED_DESTINATION" in err or not canonical:
	578	+ res.status, res.error = OnboardingStatus.NO_WEBSITE, res.error or "homepage unreachable"
	579	+ return
	580	+ # Homepage blocked (anti-bot 403, timeout, 5xx…): many such sites still expose robots/sitemaps, careers., investors., news.
	581	+ # subdomains or plain paths. Partial onboarding keeps the company observable instead of failing it outright (spec §60, §190).
	582	+ await _discover_without_homepage(company, fetcher, res, budget, canonical, fetch_now=fetch_now)
554	583	return
555	584	res.final_url = home.final_url
556	585	final_dom = registrable_domain(home.final_url)
557	586