spb/company-atlas
Public
Python 66.3%
TypeScript 22.7%
JavaScript 8.6%
HTML 1.4%
CSS 0.7%
1"""URL canonicalisation, domain identity and surface classification heuristics (spec §11, §117, §118).23`canonicalize_url` produces the stable key stored in `sensors.canonical_url`; the original URL is always kept separately.4`classify_url` maps a URL (+ optional anchor text / title) to a Surface with a confidence in 0–1 — deterministic, no LLM.5"""6from __future__ import annotations78import re9from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse1011import tldextract1213from companyatlas.taxonomy import Surface1415_extract = tldextract.TLDExtract(suffix_list_urls=(), fallback_to_snapshot=True) # offline PSL snapshot: no network at import1617TRACKING_PREFIXES = ("utm_", "ref", "fbclid", "gclid", "dclid", "msclkid", "mc_cid", "mc_eid", "_hs", "hsa_", "igshid", "yclid", "_ga",18 "_gl", "source", "campaign", "mkt_tok", "trk", "cmpid", "s_kwcid", "ef_id", "sessionid", "session_id", "phpsessid",19 "jsessionid", "sid", "cid", "icid", "ncid", "spm", "srsltid")20SESSION_PATH_RE = re.compile(r";jsessionid=[^/?#]+", re.IGNORECASE)21MULTI_SLASH_RE = re.compile(r"/{2,}")22STATIC_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", ".css", ".js", ".mjs", ".woff", ".woff2", ".ttf", ".eot", ".mp4",23 ".mp3", ".webm", ".mov", ".zip", ".gz", ".tar", ".dmg", ".exe", ".pkg", ".apk", ".ics", ".xlsx", ".pptx", ".docx")24DOCUMENT_EXT = (".pdf",)252627def canonicalize_url(url: str) -> str:28 """Stable key: lowercase scheme/host, default ports dropped, tracking/session params removed, params sorted, fragment dropped,29 duplicate slashes collapsed, trailing slash removed (except root). Preserves path case (many servers are case-sensitive)."""30 p = urlparse(url.strip())31 scheme = (p.scheme or "https").lower()32 host = (p.hostname or "").lower().rstrip(".")33 port = p.port34 if port and not ((scheme == "https" and port == 443) or (scheme == "http" and port == 80)):35 host = f"{host}:{port}"36 path = SESSION_PATH_RE.sub("", p.path or "/")37 path = MULTI_SLASH_RE.sub("/", path)38 if len(path) > 1:39 path = path.rstrip("/") or "/"40 kept = [(k, v) for k, v in parse_qsl(p.query, keep_blank_values=False) if not k.lower().startswith(TRACKING_PREFIXES)]41 query = urlencode(sorted(kept), doseq=True)42 return urlunparse((scheme, host, path, "", query, ""))434445def registrable_domain(url_or_host: str) -> str:46 """`https://jobs.eu.stripe.com/x` → `stripe.com` (public-suffix aware, offline)."""47 host = url_or_host if "://" not in url_or_host else (urlparse(url_or_host).hostname or "")48 host = host.lower().strip().rstrip(".")49 ext = _extract(host)50 if ext.domain and ext.suffix:51 return f"{ext.domain}.{ext.suffix}"52 return host.removeprefix("www.")535455def host_of(url: str) -> str:56 return (urlparse(url).hostname or "").lower()575859def same_company_host(url: str, canonical_domain: str) -> bool:60 """Is this URL on the company's registrable domain (any subdomain)?"""61 return registrable_domain(url) == registrable_domain(canonical_domain)626364def is_static_asset(url: str) -> bool:65 path = urlparse(url).path.lower()66 return path.endswith(STATIC_EXT)676869def is_document(url: str) -> bool:70 return urlparse(url).path.lower().endswith(DOCUMENT_EXT)717273def absolutize(base: str, href: str) -> str | None:74 href = (href or "").strip()75 if not href or href.startswith(("#", "mailto:", "tel:", "javascript:", "data:", "sms:", "whatsapp:")):76 return None77 try:78 out = urljoin(base, href)79 except ValueError:80 return None81 if not out.startswith(("http://", "https://")):82 return None83 return out848586# ------------------------------------------------------------------------------------------------------ crawl-trap heuristics8788TRAP_PARAM_RE = re.compile(r"(^|[?&])(page|p|offset|start|sort|order|filter|facet|color|size|price|min|max|year|month|day|date|q|s|search)=", re.IGNORECASE)89CALENDAR_RE = re.compile(r"/(19|20)\d{2}/(0?[1-9]|1[0-2])(/|$)")909192def looks_like_trap(url: str) -> bool:93 p = urlparse(url)94 if p.query.count("&") >= 4:95 return True96 if TRAP_PARAM_RE.search("?" + p.query) and p.query.count("=") >= 2:97 return True98 if CALENDAR_RE.search(p.path) and p.path.count("/") > 4:99 return True100 return len(url) > 400101102103# ------------------------------------------------------------------------------------------------------ surface classification104# Each rule: (surface, path regex, anchor/title regex, base confidence). Path evidence weighs more than anchor evidence; when both105# match, confidence is boosted. Order matters only for tie-breaks (first rule wins on equal confidence).106107_R = re.compile108RULES: list[tuple[Surface, re.Pattern[str] | None, re.Pattern[str] | None, float]] = [109 (Surface.PRICING, _R(r"/(pricing|plans|plans-and-pricing|pricing-plans|tarifs|preise|precios|prezzi|价格)(/|$)", re.IGNORECASE), _R(r"^(pricing|plans( & pricing| and pricing)?|see pricing|view pricing|tarifs|preise|precios)$", re.IGNORECASE), 0.95),110 (Surface.JOBS_BOARD, _R(r"(boards\.greenhouse\.io|job-boards\.greenhouse\.io|jobs\.lever\.co|jobs\.ashbyhq\.com|jobs\.smartrecruiters\.com|careers\.smartrecruiters\.com|myworkdayjobs\.com|apply\.workable\.com|jobs\.jobvite\.com|recruiting\.paylocity\.com|bamboohr\.com/careers|breezy\.hr|recruitee\.com|personio\.(de|com)|teamtailor\.com|icims\.com|taleo\.net|successfactors\.com|eightfold\.ai|phenom\.com|wd\d\.myworkdaysite\.com|careers-page\.com|homerun\.co|pinpointhq\.com|rippling-ats\.com|jobs\.gem\.com)", re.IGNORECASE), None, 0.97),111 (Surface.CAREERS, _R(r"/(careers?|jobs?|join(-us|us)?|work-with-us|work-for-us|working-at|open-positions|opportunities|vacancies|recruit(ing|ment)?|employment|karriere|stellen|emplois?|carri[eè]res?|empleo|trabaja-con-nosotros|lavora-con-noi|saiyou|採用)(/|$)", re.IGNORECASE), _R(r"^(careers?|jobs?|join (us|the team|our team)|work (with|for|at) us|open (roles|positions)|we'?re hiring|hiring|opportunities|vacancies|karriere|emplois?|carri[eè]res?|empleo)$", re.IGNORECASE), 0.93),112 (Surface.NEWSROOM, _R(r"/(news(room)?|press(-releases?|room|-center|-centre)?|media(-center|-centre|-room)?|announcements|releases|actualit[eé]s|presse|noticias|prensa|ニュース)(/|$)", re.IGNORECASE), _R(r"^(news(room)?|press( releases?| room| center)?|media( center| room)?|announcements|in the news|actualit[eé]s|presse|noticias)$", re.IGNORECASE), 0.9),113 (Surface.INVESTOR_RELATIONS, _R(r"/(investors?|investor-relations|ir|shareholders?|financials?|earnings|sec-filings|annual-reports?|investisseurs|investoren|inversores)(/|$)|^https?://(ir|investors?|investor)\.", re.IGNORECASE), _R(r"^(investors?|investor relations|shareholders?|financials?|ir|investisseurs|investoren)$", re.IGNORECASE), 0.92),114 (Surface.LEADERSHIP, _R(r"/(leadership|management(-team)?|executive(s|-team|-leadership)?|our-team|the-team|team|board(-of-directors)?|directors|founders|people|who-we-are|direction|equipe|équipe|equipo|vorstand|management-board|governance/(board|leadership))(/|$)", re.IGNORECASE), _R(r"^(leadership( team)?|management( team)?|executive (team|leadership)|our (team|leadership|people)|meet the team|board of directors|founders|team|direction|équipe)$", re.IGNORECASE), 0.85),115 (Surface.LOCATIONS, _R(r"/(locations?|offices?|our-offices|where-we-are|global-presence|worldwide|stores?|store-locator|find-a-store|branches|dealers?|showrooms?|sites|standorte|bureaux|ubicaciones|拠点)(/|$)", re.IGNORECASE), _R(r"^(locations?|our (locations|offices)|offices?|where we are|global presence|find a store|store locator|branches|standorte|bureaux)$", re.IGNORECASE), 0.88),116 (Surface.CHANGELOG, _R(r"/(changelog|change-log|changes|release-notes|releases|whats-new|what's-new|updates|product-updates)(/|$)|^https?://(changelog|releases|updates)\.", re.IGNORECASE), _R(r"^(changelog|release notes|what'?s new|product updates|updates|releases)$", re.IGNORECASE), 0.9),117 (Surface.API, _R(r"/(api|apis|api-reference|api-docs|reference)(/|$)|^https?://api-?docs?\.", re.IGNORECASE), _R(r"^(api( reference| docs| documentation)?|apis|rest api|graphql)$", re.IGNORECASE), 0.82),118 (Surface.DEVELOPER, _R(r"/(developers?|dev|devs|developer-portal|platform|sdks?|integrations?|build)(/|$)|^https?://(developers?|dev|build)\.", re.IGNORECASE), _R(r"^(developers?|developer (portal|center|hub)|for developers|sdks?|integrations?|build)$", re.IGNORECASE), 0.8),119 (Surface.DOCS, _R(r"/(docs|documentation|help-center|help|guides?|manuals?|knowledge-base|kb|learn|tutorials?)(/|$)|^https?://(docs|documentation|help|support|kb|learn|guides?)\.", re.IGNORECASE), _R(r"^(docs|documentation|guides?|help center|knowledge base|manuals?|tutorials?|learn)$", re.IGNORECASE), 0.8),120 (Surface.STATUS, _R(r"^https?://(status|health|uptime|trust)\.|/(status|system-status|service-status)(/|$)", re.IGNORECASE), _R(r"^(status|system status|service status|status page)$", re.IGNORECASE), 0.85),121 (Surface.BLOG, _R(r"/(blog|blogs|insights|stories|articles|journal|magazine|perspectives|thinking|ideas|posts|editorial|le-blog)(/|$)|^https?://(blog|insights|stories|medium)\.", re.IGNORECASE), _R(r"^(blog|insights|stories|articles|journal|perspectives|ideas|our blog)$", re.IGNORECASE), 0.82),122 (Surface.RESEARCH, _R(r"/(research|labs?|science|publications|papers|whitepapers?|reports|studies)(/|$)|^https?://(research|labs?|science)\.", re.IGNORECASE), _R(r"^(research|labs?|publications|whitepapers?|reports|science)$", re.IGNORECASE), 0.78),123 (Surface.PRODUCTS, _R(r"/(products?|product-catalog|catalog(ue)?|shop|store|collections|portfolio|offerings|our-products|produits|produkte|productos|prodotti|製品)(/|$)|^https?://(shop|store|products?)\.", re.IGNORECASE), _R(r"^(products?|our products|product catalog|catalog(ue)?|shop|store|portfolio|offerings|produits|produkte)$", re.IGNORECASE), 0.82),124 (Surface.SERVICES, _R(r"/(services?|what-we-do|capabilities|expertise|offerings|our-services|prestations|leistungen|servicios|servizi)(/|$)", re.IGNORECASE), _R(r"^(services?|our services|what we do|capabilities|expertise|leistungen|prestations)$", re.IGNORECASE), 0.78),125 (Surface.SOLUTIONS, _R(r"/(solutions?|use-cases|platform|technology|technologies|features)(/|$)", re.IGNORECASE), _R(r"^(solutions?|use cases|platform|features)$", re.IGNORECASE), 0.7),126 (Surface.INDUSTRIES, _R(r"/(industries|industry|sectors?|markets?|verticals?|who-we-serve)(/|$)", re.IGNORECASE), _R(r"^(industries|sectors?|markets?|who we serve|verticals?)$", re.IGNORECASE), 0.7),127 (Surface.CUSTOMERS, _R(r"/(customers?|customer-stories|case-studies|success-stories|clients?|references|testimonials|showcase|wall-of-love)(/|$)", re.IGNORECASE), _R(r"^(customers?|customer stories|case studies|success stories|clients?|our customers|testimonials)$", re.IGNORECASE), 0.78),128 (Surface.PARTNERS, _R(r"/(partners?|partnerships?|partner-program|alliances|ecosystem|marketplace|resellers?|channel)(/|$)|^https?://(partners?|marketplace)\.", re.IGNORECASE), _R(r"^(partners?|partnerships?|partner program|alliances|ecosystem|become a partner|marketplace)$", re.IGNORECASE), 0.78),129 (Surface.SECURITY, _R(r"/(security|trust(-center|-portal)?|compliance|privacy-and-security)(/|$)|^https?://(security|trust)\.", re.IGNORECASE), _R(r"^(security|trust( center)?|compliance|trust & safety)$", re.IGNORECASE), 0.8),130 (Surface.SUSTAINABILITY, _R(r"/(sustainability|esg|responsibility|corporate-responsibility|csr|impact|environment|climate|social-impact|citizenship|purpose|d[eé]veloppement-durable|nachhaltigkeit|sostenibilidad)(/|$)|^https?://(sustainability|esg|impact)\.", re.IGNORECASE), _R(r"^(sustainability|esg|(corporate |social )?responsibility|csr|impact|environment|climate|our impact|purpose)$", re.IGNORECASE), 0.82),131 (Surface.LEGAL_PRIVACY, _R(r"/(privacy(-policy|-notice|-statement)?|privacypolicy|datenschutz|confidentialit[eé]|privacidad|cookie-policy|cookies)(/|$)", re.IGNORECASE), _R(r"^(privacy( policy| notice| statement)?|datenschutz(erkl[aä]rung)?|politique de confidentialit[eé]|cookie policy)$", re.IGNORECASE), 0.9),132 (Surface.LEGAL_TERMS, _R(r"/(terms(-of-(service|use|sale|business))?|tos|legal|legal-notice|terms-and-conditions|conditions|eula|agb|mentions-l[eé]gales|cgu|cgv|aviso-legal|impressum|acceptable-use(-policy)?|aup)(/|$)", re.IGNORECASE), _R(r"^(terms( of (service|use|sale))?|terms (and|&) conditions|legal( notice)?|agb|mentions l[eé]gales|impressum|eula|acceptable use policy)$", re.IGNORECASE), 0.88),133 (Surface.SUPPORT, _R(r"/(support|customer-support|customer-service|contact-support|faq|faqs|community|forum)(/|$)|^https?://(community|forum|faq)\.", re.IGNORECASE), _R(r"^(support|customer (support|service|care)|faqs?|community|forum|help & support)$", re.IGNORECASE), 0.72),134 (Surface.CONTACT, _R(r"/(contact(-us|us|-sales)?|get-in-touch|reach-us|kontakt|contactez-nous|contacto|お問い合わせ)(/|$)", re.IGNORECASE), _R(r"^(contact( us| sales)?|get in touch|talk to (us|sales)|kontakt|contactez-nous|contacto)$", re.IGNORECASE), 0.85),135 (Surface.ABOUT, _R(r"/(about(-us|us|-company|-[a-z0-9-]+)?|company|our-company|our-story|who-we-are|mission|history|overview|corporate|a-propos|qui-sommes-nous|[uü]ber-uns|unternehmen|sobre-nosotros|chi-siamo|会社概要|企業情報)(/|$)", re.IGNORECASE), _R(r"^(about( us| the company)?|company|our (company|story|mission|history)|who we are|mission|history|overview|a propos|qui sommes-nous|[uü]ber uns|unternehmen)$", re.IGNORECASE), 0.85),136 (Surface.FEED, _R(r"/(feed|rss|atom|feeds)(\.xml|/|$)|\.(rss|atom)$|/rss\.xml|/feed\.xml|/atom\.xml", re.IGNORECASE), _R(r"^(rss|atom|feed|subscribe via rss)$", re.IGNORECASE), 0.9),137 (Surface.SITEMAP, _R(r"/sitemap[^/]*\.xml(\.gz)?$|/sitemap_index\.xml$|/sitemaps?/", re.IGNORECASE), None, 0.95),138]139140# Anchors that are almost always navigation noise, never surfaces.141_NOISE_ANCHOR = re.compile(r"^(home|back|next|previous|prev|more|read more|learn more|skip to content|menu|close|login|log in|sign in|sign up|"142 r"register|search|cart|account|download|share|print|top|en|fr|de|es|it|ja|zh|português|english|français|deutsch)$", re.IGNORECASE)143144145def classify_url(url: str, *, anchor: str | None = None, title: str | None = None, canonical_domain: str | None = None) -> tuple[Surface, float]:146 """Return (surface, confidence). Homepage detection needs `canonical_domain`. Unknown → (OTHER, 0.1)."""147 p = urlparse(url)148 path = p.path or "/"149 text = " ".join(x.strip() for x in (anchor, title) if x).strip()150 text_norm = re.sub(r"\s+", " ", text).strip(" -–|·»›>").lower()151 if canonical_domain and path in ("/", "") and not p.query and registrable_domain(url) == registrable_domain(canonical_domain) \152 and host_of(url) in (canonical_domain.lower(), "www." + canonical_domain.lower(), canonical_domain.lower().removeprefix("www.")):153 return Surface.HOMEPAGE, 0.99154 if is_static_asset(url):155 return Surface.OTHER, 0.0156 best: tuple[Surface, float] = (Surface.OTHER, 0.1)157 full = f"{p.scheme}://{p.netloc}{path}"158 depth = max(0, path.strip("/").count("/"))159 for surface, path_re, anchor_re, base in RULES:160 score = 0.0161 path_hit = bool(path_re and path_re.search(full))162 anchor_hit = bool(anchor_re and text_norm and anchor_re.search(text_norm))163 if path_hit:164 score = base165 if depth >= 2:166 score -= 0.12 * (depth - 1) # /news/2024/foo is an article, not the newsroom167 if anchor_hit:168 score = max(score, base - 0.2) + (0.05 if path_hit else 0.0)169 if is_document(url):170 score -= 0.3171 if p.query and surface not in (Surface.JOBS_BOARD, Surface.SITEMAP):172 score -= 0.1173 if score > best[1]:174 best = (surface, round(min(0.99, max(0.0, score)), 3))175 if best[0] is Surface.OTHER and text_norm and _NOISE_ANCHOR.match(text_norm):176 return Surface.OTHER, 0.0177 return best178179180ATS_PATTERNS: list[tuple[str, re.Pattern[str]]] = [181 ("greenhouse", re.compile(r"https?://(?:boards|job-boards)\.greenhouse\.io/([a-z0-9_-]+)", re.IGNORECASE)),182 ("greenhouse", re.compile(r"https?://boards-api\.greenhouse\.io/v1/boards/([a-z0-9_-]+)", re.IGNORECASE)),183 ("lever", re.compile(r"https?://jobs\.(?:eu\.)?lever\.co/([a-z0-9_-]+)", re.IGNORECASE)),184 ("ashby", re.compile(r"https?://jobs\.ashbyhq\.com/([a-z0-9_.-]+)", re.IGNORECASE)),185 ("smartrecruiters", re.compile(r"https?://(?:careers|jobs)\.smartrecruiters\.com/([a-z0-9_-]+)", re.IGNORECASE)),186 ("workable", re.compile(r"https?://apply\.workable\.com/([a-z0-9_-]+)", re.IGNORECASE)),187 ("workday", re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.IGNORECASE)),188 ("recruitee", re.compile(r"https?://([a-z0-9-]+)\.recruitee\.com", re.IGNORECASE)),189 ("personio", re.compile(r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)", re.IGNORECASE)),190 ("teamtailor", re.compile(r"https?://([a-z0-9-]+)\.teamtailor\.com", re.IGNORECASE)),191 ("bamboohr", re.compile(r"https?://([a-z0-9-]+)\.bamboohr\.com/careers", re.IGNORECASE)),192 ("breezy", re.compile(r"https?://([a-z0-9-]+)\.breezy\.hr", re.IGNORECASE)),193 ("jobvite", re.compile(r"https?://jobs\.jobvite\.com/([a-z0-9_-]+)", re.IGNORECASE)),194 ("pinpoint", re.compile(r"https?://([a-z0-9-]+)\.pinpointhq\.com", re.IGNORECASE)),195 ("rippling", re.compile(r"https?://ats\.rippling\.com/([a-z0-9_-]+)", re.IGNORECASE)),196]197198199def detect_ats(url: str) -> tuple[str, str] | None:200 """(vendor, board token) when the URL is a public ATS board we have a structured connector for."""201 for vendor, pat in ATS_PATTERNS:202 m = pat.search(url)203 if m:204 return vendor, m.group(1)205 return None206207208__all__ = ["ATS_PATTERNS", "RULES", "absolutize", "canonicalize_url", "classify_url", "detect_ats", "host_of", "is_document",209 "is_static_asset", "looks_like_trap", "registrable_domain", "same_company_host"]210