spb/company-atlas
Public
Python 66.3%
TypeScript 22.7%
JavaScript 8.6%
HTML 1.4%
CSS 0.7%
1"""Shared helpers for connector families: location / country parsing (never invent), date parsing, job normalisation,2fingerprints, JSON access and the ATS vendor → API mapping used by discovery. Deterministic, no network."""3from __future__ import annotations45import hashlib6import json7import re8from datetime import UTC, datetime9from typing import Any10from urllib.parse import urlparse1112from dateutil import parser as dtparser1314from companyatlas.fetch import FetchResult15from companyatlas.ids import stable_hash16from companyatlas.sdk.models import Block, ExtractedJob17from companyatlas.sdk.normalize import BLOCK_WEIGHTS, normalize_whitespace, normalized_text, simhash1819# ------------------------------------------------------------------------------------------------------------ countries2021_COUNTRIES: dict[str, str] = {22 "afghanistan": "AF", "albania": "AL", "algeria": "DZ", "andorra": "AD", "angola": "AO", "argentina": "AR", "armenia": "AM", "australia": "AU",23 "austria": "AT", "österreich": "AT", "azerbaijan": "AZ", "bahamas": "BS", "bahrain": "BH", "bangladesh": "BD", "belarus": "BY", "belgium": "BE",24 "belgique": "BE", "belgië": "BE", "bolivia": "BO", "bosnia and herzegovina": "BA", "bosnia": "BA", "botswana": "BW", "brazil": "BR", "brasil": "BR",25 "brunei": "BN", "bulgaria": "BG", "cambodia": "KH", "cameroon": "CM", "canada": "CA", "chile": "CL", "china": "CN", "中国": "CN",26 "colombia": "CO", "costa rica": "CR", "croatia": "HR", "cuba": "CU", "cyprus": "CY", "czech republic": "CZ", "czechia": "CZ", "denmark": "DK",27 "danmark": "DK", "dominican republic": "DO", "ecuador": "EC", "egypt": "EG", "el salvador": "SV", "estonia": "EE", "ethiopia": "ET",28 "finland": "FI", "suomi": "FI", "france": "FR", "georgia": "GE", "germany": "DE", "deutschland": "DE", "ghana": "GH", "greece": "GR",29 "guatemala": "GT", "honduras": "HN", "hong kong": "HK", "hungary": "HU", "iceland": "IS", "india": "IN", "indonesia": "ID", "iran": "IR",30 "iraq": "IQ", "ireland": "IE", "israel": "IL", "italy": "IT", "italia": "IT", "jamaica": "JM", "japan": "JP", "日本": "JP", "jordan": "JO",31 "kazakhstan": "KZ", "kenya": "KE", "kuwait": "KW", "latvia": "LV", "lebanon": "LB", "lithuania": "LT", "luxembourg": "LU", "macau": "MO",32 "malaysia": "MY", "malta": "MT", "mexico": "MX", "méxico": "MX", "moldova": "MD", "monaco": "MC", "mongolia": "MN", "montenegro": "ME",33 "morocco": "MA", "myanmar": "MM", "nepal": "NP", "netherlands": "NL", "the netherlands": "NL", "nederland": "NL", "holland": "NL",34 "new zealand": "NZ", "nicaragua": "NI", "nigeria": "NG", "north macedonia": "MK", "norway": "NO", "norge": "NO", "oman": "OM", "pakistan": "PK",35 "panama": "PA", "paraguay": "PY", "peru": "PE", "perú": "PE", "philippines": "PH", "poland": "PL", "polska": "PL", "portugal": "PT", "qatar": "QA",36 "romania": "RO", "românia": "RO", "russia": "RU", "russian federation": "RU", "rwanda": "RW", "saudi arabia": "SA", "senegal": "SN", "serbia": "RS",37 "singapore": "SG", "slovakia": "SK", "slovenia": "SI", "south africa": "ZA", "south korea": "KR", "korea": "KR", "republic of korea": "KR",38 "korea, republic of": "KR", "spain": "ES", "españa": "ES", "sri lanka": "LK", "sweden": "SE", "sverige": "SE", "switzerland": "CH", "schweiz": "CH",39 "suisse": "CH", "taiwan": "TW", "tanzania": "TZ", "thailand": "TH", "tunisia": "TN", "turkey": "TR", "türkiye": "TR", "turkiye": "TR",40 "uganda": "UG", "ukraine": "UA", "united arab emirates": "AE", "uae": "AE", "united kingdom": "GB", "uk": "GB", "u.k.": "GB", "great britain": "GB",41 "britain": "GB", "england": "GB", "scotland": "GB", "wales": "GB", "northern ireland": "GB", "united states": "US", "united states of america": "US",42 "usa": "US", "u.s.": "US", "u.s.a.": "US", "us": "US", "america": "US", "uruguay": "UY", "uzbekistan": "UZ", "venezuela": "VE", "vietnam": "VN",43 "viet nam": "VN", "zambia": "ZM", "zimbabwe": "ZW", "puerto rico": "PR", "european union": None, # type: ignore[dict-item]44}45ISO2 = {v for v in _COUNTRIES.values() if v}46ISO3 = {"USA": "US", "GBR": "GB", "DEU": "DE", "FRA": "FR", "CAN": "CA", "AUS": "AU", "JPN": "JP", "CHN": "CN", "IND": "IN", "BRA": "BR", "ESP": "ES",47 "ITA": "IT", "NLD": "NL", "SWE": "SE", "CHE": "CH", "SGP": "SG", "IRL": "IE", "MEX": "MX", "KOR": "KR", "POL": "PL", "BEL": "BE", "AUT": "AT",48 "DNK": "DK", "NOR": "NO", "FIN": "FI", "PRT": "PT", "ISR": "IL", "ARE": "AE", "NZL": "NZ", "ZAF": "ZA", "ARG": "AR", "CZE": "CZ", "HUN": "HU"}49US_STATES: dict[str, str] = {50 "alabama": "AL", "alaska": "AK", "arizona": "AZ", "arkansas": "AR", "california": "CA", "colorado": "CO", "connecticut": "CT", "delaware": "DE",51 "florida": "FL", "georgia": "GA", "hawaii": "HI", "idaho": "ID", "illinois": "IL", "indiana": "IN", "iowa": "IA", "kansas": "KS", "kentucky": "KY",52 "louisiana": "LA", "maine": "ME", "maryland": "MD", "massachusetts": "MA", "michigan": "MI", "minnesota": "MN", "mississippi": "MS",53 "missouri": "MO", "montana": "MT", "nebraska": "NE", "nevada": "NV", "new hampshire": "NH", "new jersey": "NJ", "new mexico": "NM",54 "new york": "NY", "north carolina": "NC", "north dakota": "ND", "ohio": "OH", "oklahoma": "OK", "oregon": "OR", "pennsylvania": "PA",55 "rhode island": "RI", "south carolina": "SC", "south dakota": "SD", "tennessee": "TN", "texas": "TX", "utah": "UT", "vermont": "VT",56 "virginia": "VA", "washington": "WA", "west virginia": "WV", "wisconsin": "WI", "wyoming": "WY", "district of columbia": "DC",57}58# two-letter US state codes that are NOT also ISO country codes (safe to infer US from "City, ST")59US_STATE_CODES_SAFE = {c for c in US_STATES.values()} - {"CA", "CO", "DE", "IN", "ME", "MD", "MT", "NE", "MA", "MO", "MN", "NC", "SC", "LA", "ID",60 "IL", "GA", "AL", "AR", "AZ", "KY", "PA", "TN", "VA", "MS"}61CA_PROVINCES: dict[str, str] = {"ontario": "ON", "quebec": "QC", "québec": "QC", "british columbia": "BC", "alberta": "AB", "manitoba": "MB",62 "saskatchewan": "SK", "nova scotia": "NS", "new brunswick": "NB", "newfoundland and labrador": "NL",63 "prince edward island": "PE"}64CA_PROVINCE_CODES_SAFE = {"ON", "QC", "BC", "AB", "MB", "SK", "NS", "NB"}65# two-letter tokens that are both a US state/CA province code and an ISO country code: never resolved to a country on their own66AMBIGUOUS_CODES = (set(US_STATES.values()) | set(CA_PROVINCES.values())) & ISO267REMOTE_RE = re.compile(r"\b(remote|work from home|wfh|anywhere|distributed|télétravail|homeoffice|home office)\b", re.IGNORECASE)68HYBRID_RE = re.compile(r"\bhybrid\b", re.IGNORECASE)69POSTAL_RE = re.compile(r"\b[A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2}\b|\b\d{4}\s?[A-Z]{2}\b|\b\d{4,6}(?:-\d{4})?\b|\b[A-Z]\d[A-Z]\s*\d[A-Z]\d\b") # UK · NL · numeric · CA707172def country_code(text: str | None) -> str | None:73 """ISO-2 for a country name / ISO-2 / ISO-3 token — None unless confident."""74 if not text:75 return None76 t = normalize_whitespace(text).strip(" .,;()").lower()77 if not t:78 return None79 if t in _COUNTRIES:80 return _COUNTRIES[t]81 up = t.upper()82 if len(up) == 2 and up in ISO2:83 return up84 if len(up) == 3 and up in ISO3:85 return ISO3[up]86 return None878889def _unambiguous_country(token: str) -> str | None:90 t = token.strip()91 if len(t) == 2 and t.upper() in AMBIGUOUS_CODES:92 return None93 return country_code(t)949596def norm_name(text: str) -> str:97 """Entity key for people/products/locations (`name_norm`): ASCII, lowercase, single spaces, punctuation dropped."""98 from slugify import slugify as _slug99100 return _slug(text or "", lowercase=True, regex_pattern=r"[^a-z0-9]+").replace("-", " ").strip()[:120]101102103def parse_location(text: str | None) -> dict[str, Any]:104 """'San Francisco, CA, USA' → {city, region, country, remote}. Only sets what the text states; never guesses a country."""105 out: dict[str, Any] = {"city": None, "region": None, "country": None, "remote": None}106 if not text:107 return out108 raw = normalize_whitespace(text)109 if REMOTE_RE.search(raw):110 out["remote"] = True111 elif HYBRID_RE.search(raw):112 out["remote"] = False113 cleaned = re.sub(r"\((?:remote|hybrid|on-?site)[^)]*\)", "", raw, flags=re.IGNORECASE)114 cleaned = re.sub(r"\b(remote|hybrid|on-?site|flexible|multiple locations|or)\b\s*[-–—,/]?\s*", "", cleaned, flags=re.IGNORECASE)115 cleaned = POSTAL_RE.sub(" ", cleaned).strip(" ,-–/")116 parts = [p.strip() for p in re.split(r"\s*[,|•·/;]\s*|\s+[-–—]\s+", cleaned) if p.strip()]117 if not parts:118 return out119 if len(parts) >= 2 and _unambiguous_country(parts[0]) and not _unambiguous_country(parts[-1]):120 parts.reverse() # Workday style "US, MA, Westford" → city last → first121 country = _unambiguous_country(parts[-1])122 if country:123 parts = parts[:-1]124 elif len(parts) == 1:125 # a bare token: a country name ("Germany") is a country; anything else is left alone (a city without a country is not guessed)126 return out127 out["country"] = country128 if parts:129 last = parts[-1]130 low = last.lower()131 two = last.upper() if len(last) == 2 and last.isalpha() else None132 state = US_STATES.get(low) or (two if two in US_STATE_CODES_SAFE else None)133 prov = CA_PROVINCES.get(low) or (two if two in CA_PROVINCE_CODES_SAFE else None)134 if state and (country in (None, "US")):135 out["region"], out["country"] = state, "US"136 parts = parts[:-1]137 elif prov and (country in (None, "CA")):138 out["region"], out["country"] = prov, "CA"139 parts = parts[:-1]140 elif two and two in AMBIGUOUS_CODES and len(parts) >= 2:141 out["region"] = two # "San Francisco, CA": region kept, country deliberately unknown142 parts = parts[:-1]143 elif len(parts) >= 2:144 out["region"] = last[:80]145 parts = parts[:-1]146 if parts:147 city = parts[0]148 if len(city) <= 80 and not re.search(r"\d{3,}", city):149 out["city"] = city150 return out151152153# ------------------------------------------------------------------------------------------------------------ dates154155DATE_IN_TEXT_RE = re.compile(156 r"(\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)?)|"157 r"((?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4})|"158 r"(\d{1,2}(?:st|nd|rd|th)?\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?,?\s+\d{4})|"159 r"(\d{1,2}[/.]\d{1,2}[/.]\d{4})", re.IGNORECASE)160DATE_IN_URL_RE = re.compile(r"/((?:19|20)\d{2})/(0?[1-9]|1[0-2])(?:/(0?[1-9]|[12]\d|3[01]))?(?:/|$)")161162163def parse_date(value: Any) -> datetime | None:164 """Best-effort aware datetime (UTC) from ISO strings, epoch seconds/ms, common textual dates. None when unparseable."""165 if value is None or value == "":166 return None167 if isinstance(value, datetime):168 return value if value.tzinfo else value.replace(tzinfo=UTC)169 if isinstance(value, int | float):170 v = float(value)171 if v > 1e12:172 v /= 1000.0173 if 0 < v < 4102444800:174 return datetime.fromtimestamp(v, tz=UTC)175 return None176 s = str(value).strip()177 if not s or len(s) > 60:178 return None179 if s.isdigit():180 return parse_date(int(s))181 try:182 dt = dtparser.parse(s, fuzzy=False, dayfirst=False)183 except (ValueError, OverflowError, TypeError):184 try:185 dt = dtparser.parse(s, fuzzy=True)186 except (ValueError, OverflowError, TypeError):187 return None188 if dt.year < 1990 or dt.year > 2100:189 return None190 return dt if dt.tzinfo else dt.replace(tzinfo=UTC)191192193def date_from_text(text: str | None) -> datetime | None:194 if not text:195 return None196 m = DATE_IN_TEXT_RE.search(text)197 return parse_date(m.group(0)) if m else None198199200def date_from_url(url: str | None) -> datetime | None:201 if not url:202 return None203 m = DATE_IN_URL_RE.search(urlparse(url).path)204 if not m:205 return None206 y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3) or 1)207 try:208 return datetime(y, mo, d, tzinfo=UTC)209 except ValueError:210 return None211212213# ------------------------------------------------------------------------------------------------------------ jobs214215SENIORITY_RULES: list[tuple[str, re.Pattern[str]]] = [216 ("intern", re.compile(r"\b(intern(ship)?|stagiaire|werkstudent|working student|apprentice|co-op)\b", re.IGNORECASE)),217 ("c_level", re.compile(r"\b(chief\b|\bc[a-z]o\b|vp\b|vice president|evp|svp|head of|director)", re.IGNORECASE)),218 ("principal", re.compile(r"\b(principal|distinguished|fellow)\b", re.IGNORECASE)),219 ("staff", re.compile(r"\bstaff\b", re.IGNORECASE)),220 ("lead", re.compile(r"\b(lead|manager|leiter)\b", re.IGNORECASE)),221 ("senior", re.compile(r"\b(senior|sr\.?|iii|iv)\b", re.IGNORECASE)),222 ("junior", re.compile(r"\b(junior|jr\.?|entry[- ]level|graduate|associate|i)\b", re.IGNORECASE)),223]224ENGINEERING_RE = re.compile(r"\b(engineer|engineering|developer|software|sre|devops|architect|data scientist|machine learning|backend|frontend|"225 r"full[- ]stack|qa|security|infrastructure|platform|embedded|firmware|ios|android|ingénieur|entwickler)\b", re.IGNORECASE)226EMPLOYMENT_TYPES = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time", "permanent": "full_time", "part-time": "part_time",227 "part time": "part_time", "contract": "contract", "contractor": "contract", "temporary": "temporary", "temp": "temporary",228 "intern": "internship", "internship": "internship", "freelance": "contract", "apprenticeship": "internship", "seasonal": "temporary",229 "fixed-term": "contract", "fixed term": "contract", "cdi": "full_time", "cdd": "contract"}230231232def seniority_guess(title: str | None) -> str | None:233 if not title:234 return None235 for label, pat in SENIORITY_RULES:236 if pat.search(title):237 return label238 return None239240241def employment_type_norm(value: str | None) -> str | None:242 if not value:243 return None244 v = normalize_whitespace(str(value)).lower().replace("_", "-")245 if v in EMPLOYMENT_TYPES:246 return EMPLOYMENT_TYPES[v]247 for k, out in EMPLOYMENT_TYPES.items():248 if k in v:249 return out250 return v[:30]251252253def is_engineering(title: str | None) -> bool:254 return bool(title and ENGINEERING_RE.search(title))255256257def job_fingerprint(title: str, location_text: str | None, external_id: str | None, url: str | None) -> str:258 ident = (external_id or "").strip() or (url or "").split("?")[0].rstrip("/")259 return stable_hash(normalized_text(title), normalized_text(location_text or ""), ident.lower())260261262def strip_html(s: str | None) -> str:263 if not s:264 return ""265 from selectolax.lexbor import LexborHTMLParser266267 try:268 return normalize_whitespace(LexborHTMLParser(f"<div>{s}</div>").text(separator=" "))269 except Exception: # noqa: BLE001270 return normalize_whitespace(re.sub(r"<[^>]+>", " ", s))271272273def description_hash(s: str | None) -> str | None:274 txt = normalized_text(strip_html(s)) if s else ""275 return hashlib.sha256(txt.encode("utf-8")).hexdigest()[:32] if txt else None276277278def finish_job(job: ExtractedJob) -> ExtractedJob:279 """Fill derived fields (seniority, remote/city/country from location text) without overriding vendor-provided values."""280 job.title = normalize_whitespace(job.title)[:300]281 if job.location_text:282 loc = parse_location(job.location_text)283 job.city = job.city or loc["city"]284 job.region = job.region or loc["region"]285 job.country = job.country or loc["country"]286 if job.remote is None:287 job.remote = loc["remote"]288 if job.remote is None and REMOTE_RE.search(job.title):289 job.remote = True290 job.seniority = job.seniority or seniority_guess(job.title)291 job.employment_type = employment_type_norm(job.employment_type)292 if job.country:293 job.country = job.country.upper()[:2]294 return job295296297def job_blocks(jobs: list[ExtractedJob], *, path: str = "Jobs") -> list[Block]:298 """One stable block per job (identity = external id / url), so the block diff mirrors the job delta exactly."""299 blocks: list[Block] = []300 seen: set[str] = set()301 for i, j in enumerate(jobs):302 ident = (j.external_id or j.url or job_fingerprint(j.title, j.location_text, None, None))303 key = f"job_listing:{hashlib.blake2b(str(ident).encode('utf-8'), digest_size=6).hexdigest()}"304 if key in seen:305 key = f"{key}#{i}"306 seen.add(key)307 text = " — ".join(x for x in (j.title, j.location_text, j.department) if x)308 blocks.append(Block(key=key, kind="job_listing", text=text, path=path, hash=hashlib.sha256(normalized_text(text).encode()).hexdigest()[:16],309 simhash=simhash(text), weight=BLOCK_WEIGHTS["job_listing"], order=i, attrs={"url": j.url, "external_id": j.external_id}))310 return blocks311312313def jobs_text(jobs: list[ExtractedJob], header: str) -> str:314 lines = [header] + [" — ".join(x for x in (j.title, j.location_text, j.department) if x) for j in jobs]315 return "\n".join(lines)316317318# ------------------------------------------------------------------------------------------------------------ JSON helpers319320321def load_json(result: FetchResult) -> Any:322 try:323 return result.json()324 except Exception as exc:325 raise ValueError(f"invalid JSON from {result.final_url}: {exc.__class__.__name__}") from exc326327328def dig(obj: Any, *path: str, default: Any = None) -> Any:329 cur = obj330 for p in path:331 if isinstance(cur, dict):332 cur = cur.get(p)333 else:334 return default335 if cur is None:336 return default337 return cur338339340def merged_result(first: FetchResult, payload: Any, *, pages: int) -> FetchResult:341 """Synthetic FetchResult holding the merged pages of a paginated public API (what we archive is what we saw)."""342 content = json.dumps(payload, ensure_ascii=False).encode("utf-8")343 res = FetchResult(url=first.url, final_url=first.final_url, status=first.status, headers=dict(first.headers), content=content,344 content_type="application/json; charset=utf-8", fetched_at=first.fetched_at, duration_ms=first.duration_ms,345 transport=first.transport, redirects=first.redirects)346 res.headers["x-companyatlas-pages"] = str(pages)347 return res348349350def text_of(v: Any) -> str | None:351 if v is None:352 return None353 if isinstance(v, dict):354 for k in ("name", "label", "text", "title", "value"):355 if v.get(k):356 return normalize_whitespace(str(v[k]))357 return None358 if isinstance(v, list):359 return ", ".join(x for x in (text_of(i) for i in v) if x) or None360 return normalize_whitespace(str(v)) or None361362363# ------------------------------------------------------------------------------------------------------------ ATS vendor map364365WORKDAY_RE = re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/(?:([a-z]{2}-[A-Z]{2})/)?([A-Za-z0-9_-]+)", re.IGNORECASE)366367368def ats_sensor_spec(vendor: str, token: str, board_url: str) -> tuple[str, str, dict[str, Any]] | None:369 """(api_url, connector_id, config) for a detected ATS board — the sensor URL *is* the public endpoint the board's page uses."""370 v = vendor.lower()371 cfg: dict[str, Any] = {"vendor": v, "token": token, "board_url": board_url}372 if v == "greenhouse":373 return f"https://boards-api.greenhouse.io/v1/boards/{token}/jobs?content=false", "greenhouse-v1", cfg374 if v == "lever":375 region = "eu." if "jobs.eu.lever.co" in board_url else ""376 return f"https://api.{region}lever.co/v0/postings/{token}?mode=json", "lever-v1", cfg377 if v == "ashby":378 return f"https://api.ashbyhq.com/posting-api/job-board/{token}", "ashby-v1", cfg379 if v == "smartrecruiters":380 return f"https://api.smartrecruiters.com/v1/companies/{token}/postings?limit=100", "smartrecruiters-v1", cfg381 if v == "workable":382 return f"https://apply.workable.com/api/v1/widget/accounts/{token}", "workable-v1", cfg383 if v == "recruitee":384 return f"https://{token}.recruitee.com/api/offers/", "recruitee-v1", cfg385 if v == "personio":386 return f"https://{token}.jobs.personio.de/xml", "personio-v1", cfg387 if v == "teamtailor":388 host = urlparse(board_url).hostname or f"{token}.teamtailor.com"389 return f"https://{host}/jobs.json", "teamtailor-v1", cfg390 if v == "workday":391 m = WORKDAY_RE.search(board_url)392 if not m:393 return None394 tenant, wd, _locale, site = m.group(1), m.group(2), m.group(3), m.group(4)395 if site.lower() in ("wday", "en-us", "job"):396 return None397 cfg.update({"tenant": tenant, "wd": wd, "site": site})398 return f"https://{tenant}.{wd}.myworkdayjobs.com/wday/cxs/{tenant}/{site}/jobs", "workday-v1", cfg399 return None400401402__all__ = [403 "CA_PROVINCES",404 "DATE_IN_URL_RE",405 "REMOTE_RE",406 "US_STATES",407 "WORKDAY_RE",408 "ats_sensor_spec",409 "country_code",410 "date_from_text",411 "date_from_url",412 "description_hash",413 "dig",414 "employment_type_norm",415 "finish_job",416 "is_engineering",417 "job_blocks",418 "job_fingerprint",419 "jobs_text",420 "load_json",421 "merged_result",422 "norm_name",423 "parse_date",424 "parse_location",425 "seniority_guess",426 "strip_html",427 "text_of",428]429