spb/company-atlas

1"""Precision rules for typed extractions (spec §107, "never fabricate"): deterministic validators and normalisers that reject the2navigation / call-to-action / cookie-consent / marketing noise the generic HTML connector picks up on real corporate pages, and3repair the classic confusions (person name ↔ title swapped, country used as a location name, truncated price text).45Shared by `connectors/generic_html.py` (applied while extracting) and `services/pipeline._drop_corrupt_entities` (last line of6defence for every connector) and by `scripts/audit_extractions.py` (measurement / purge on stored rows). Every rule is a small pure7function returning a `Verdict`, so the same code explains *why* a row is rejected. Multilingual where cheap (EN/FR/DE/NL/ES/IT/PT/JA).8No network, no LLM.9"""10from __future__ import annotations1112import re13import unicodedata14from collections.abc import Iterable15from dataclasses import dataclass16from datetime import datetime17from typing import Any1819from companyatlas.connectors._util import POSTAL_RE, country_code, date_from_url, parse_location20from companyatlas.sdk.models import (21    ExtractedJob,22    ExtractedLocation,23    ExtractedNewsItem,24    ExtractedPerson,25    ExtractedPlan,26    ExtractedProduct,27    Extraction,28)29from companyatlas.sdk.normalize import normalize_whitespace3031PRECISION_VERSION = "precision-v1"3233# ------------------------------------------------------------------------------------------------------------ limits (no magic numbers)3435MAX_JOB_TITLE_CHARS = 14036MIN_JOB_TITLE_CHARS = 437MIN_JOB_TITLE_WORDS = 238PROSE_MIN_WORDS = 4                    # a "title" with ≥ 4 words and function words reads like a sentence39PERSON_NAME_MIN_TOKENS, PERSON_NAME_MAX_TOKENS = 2, 540MIN_PERSON_NAME_CHARS, MAX_PERSON_NAME_CHARS = 4, 6041MAX_PERSON_TITLE_CHARS = 10042SENTENCE_TITLE_MIN_WORDS = 6           # a person "title" of ≥ 6 words ending with a full stop is a bio sentence43MAX_LOCATION_NAME_CHARS = 8044MAX_LOCATION_NAME_WORDS = 845MAX_CITY_CHARS, MAX_CITY_WORDS = 40, 446VENUE_MIN_WORDS = 3                    # "Hormuz Grand Hotel": ≥ 3 words in the city slot is a venue, not a city47MAX_PLAN_NAME_CHARS, MAX_PLAN_NAME_WORDS = 40, 548MAX_PRICE_TEXT_CHARS = 6049MAX_PRODUCT_NAME_CHARS, MAX_PRODUCT_NAME_WORDS = 80, 1050NEWS_MIN_WORDS, NEWS_MIN_CHARS = 3, 1551NEWS_MIN_CJK_CHARS = 6                 # CJK titles have no word boundaries: 6 characters already carry a headline52MAX_DEPARTMENT_CHARS, MAX_DEPARTMENT_WORDS = 40, 453CITY_NGRAM_MAX = 354YEAR_RE = re.compile(r"^(?:19|20)\d{2}$")555657@dataclass(slots=True, frozen=True)58class Verdict:59    ok: bool60    reason: str | None = None6162    @staticmethod63    def accept() -> Verdict:64        return Verdict(True, None)6566    @staticmethod67    def reject(reason: str) -> Verdict:68        return Verdict(False, reason)697071# ------------------------------------------------------------------------------------------------------------ text helpers7273_WORD_RE = re.compile(r"[^\W_]+", re.UNICODE)74_CJK_RE = re.compile(r"[぀-ヿ㐀-鿿]")75TERMINAL_PUNCT = ".!?:;,"76TRAILING_MARKS_RE = re.compile(r"[\s→➔➡›»>\-–—|·•]+$")   # arrows, chevrons, dashes, bullets77ELLIPSIS_RE = re.compile(r"(?:\.\.\.|…)\s*$")78PROSE_FUNCTION_WORDS_RE = re.compile(79    r"\b(?:who|that|which|want|wants|with|for|your|you|we|our|are|is|the|die|dat|willen|voor|met|jouw|onze|wij|zijn|qui|que|pour|avec|votre|nous|"80    r"sont|est|les|der|das|mit|für|und|ihre|wir|sind|para|con|nuestro|somos|per|che|nostro|siamo|com|nosso|somos)\b", re.IGNORECASE)81SENTENCE_VERB_RE = re.compile(82    r"\b(?:allows?|enables?|helps?|uses?|provides?|lets?|makes?|gives?|ensures?|collects?|stores?|can|will|would|should|permet|permettent|utilise|"83    r"erlaubt|ermöglicht|verwendet|gebruikt|permite|consente|permitem)\b", re.IGNORECASE)84MARKETING_VERB_RE = re.compile(85    r"^(?:win|get|start|discover|unlock|boost|grow|try|choose|compare|see|learn|find|build|join|save|upgrade|buy|sign up|request|book|schedule|"86    r"contact|talk|let'?s|ready|why|how|what|explore|meet|introducing|welcome|enjoy|make|take|achieve|transform|power|scale|drive|accelerate|"87    r"supercharge|level up|bring|stay|experience|gagnez|obtenez|découvrez|commencez|essayez|choisissez|comparez|rejoignez|débloquez|profitez|"88    r"gewinnen|holen|starten|entdecken|testen|wählen|vergleichen|jetzt|ontdek|start|kies|vergelijk|probeer|gana|obtén|empieza|descubre|prueba|"89    r"elige|compara|scopri|inizia|prova|scegli|confronta|ottieni|ganhe|obtenha|comece|descubra|experimente|escolha)\b", re.IGNORECASE)909192def word_count(text: str) -> int:93    return len(_WORD_RE.findall(text))949596def key_of(text: str | None) -> str:97    """Accent-insensitive, lower-case, single-spaced key (keeps CJK) used for stoplists and the city table."""98    if not text:99        return ""100    nfkd = unicodedata.normalize("NFKD", text)101    stripped = "".join(ch for ch in nfkd if not unicodedata.combining(ch))102    return " ".join(_WORD_RE.findall(stripped.lower()))103104105def strip_trailing_marks(text: str) -> str:106    return TRAILING_MARKS_RE.sub("", normalize_whitespace(text)).strip()107108109def has_terminal_punct(text: str) -> bool:110    t = text.rstrip("®™ ")111    return bool(t) and t[-1] in TERMINAL_PUNCT112113114def is_prose(text: str) -> bool:115    """A label that reads like a sentence: several words with function words, a sentence verb, or a full stop at the end."""116    words = word_count(text)117    if words < PROSE_MIN_WORDS:118        return False119    if text.rstrip().endswith((".", "!", "?")):120        return True121    return bool(SENTENCE_VERB_RE.search(text)) or len(PROSE_FUNCTION_WORDS_RE.findall(text)) >= 2122123124def is_slogan(text: str) -> bool:125    return bool(MARKETING_VERB_RE.match(text)) and word_count(text) >= PROSE_MIN_WORDS126127128def _phrase_re(phrases: Iterable[str]) -> re.Pattern[str]:129    alts = sorted({p.strip().lower() for p in phrases if p.strip()}, key=len, reverse=True)130    return re.compile(r"^(?:" + "|".join(re.escape(a) for a in alts) + r")$", re.IGNORECASE)131132133# ------------------------------------------------------------------------------------------------------------ role vocabulary (people)134135ROLE_RULES: list[tuple[str, re.Pattern[str]]] = [136    ("founder", re.compile(r"\b(co-?founder|founder|fondat(eur|rice)|gründer(in)?|fondator[ei]|fundador(a)?|oprichter|medeoprichter)\b", re.IGNORECASE)),137    ("ceo", re.compile(r"\b(chief executive( officer)?|ceo|pdg|président-directeur|geschäftsführer(in)?|managing director|directeur général|"138                       r"algemeen directeur|consejero delegado|amministratore delegato|diretor executivo)\b", re.IGNORECASE)),139    ("cfo", re.compile(r"\b(chief financial( officer)?|cfo|finanzvorstand|directeur financier|financieel directeur|director financiero|direttore finanziario)\b", re.IGNORECASE)),140    ("cto", re.compile(r"\b(chief technology( officer)?|chief technical( officer)?|cto|directeur technique)\b", re.IGNORECASE)),141    ("coo", re.compile(r"\b(chief operating( officer)?|coo|directeur des opérations)\b", re.IGNORECASE)),142    ("chair", re.compile(r"\b(chair(man|woman|person)?|executive chair|présidente? du conseil|vorsitzende[rn]?|voorzitter|presidente del consejo|"143                         r"presidente del consiglio)\b", re.IGNORECASE)),144    ("president", re.compile(r"\b(?<!vice )(?<!vice-)president(?! of)|président(?!e? du conseil)\b", re.IGNORECASE)),145    ("board", re.compile(r"\b(board member|member of the (supervisory |advisory )?board|non-executive director|independent director|"146                         r"administrat(eur|rice)|aufsichtsrat|conseil d'administration|director(?= \(board)|trustee|bestuurslid|consejer[oa]|consigliere)\b", re.IGNORECASE)),147    ("vp", re.compile(r"\b(vice[- ]president|vp|evp|svp|avp|vice-président(e)?)\b", re.IGNORECASE)),148    ("head", re.compile(r"\b(head of|head,|general manager|gm|leiter(in)?|directeur|directrice|director|managing partner|partner|responsable|hoofd|"149                        r"direttore|diretor(a)?|directora)\b", re.IGNORECASE)),150]151# Japanese titles have no word boundaries: plain substring rules, most specific first.152ROLE_RULES_JA: list[tuple[str, re.Pattern[str]]] = [153    ("founder", re.compile(r"創業者|共同創業者|ファウンダー")),154    ("cfo", re.compile(r"最高財務責任者|CFO")),155    ("cto", re.compile(r"最高技術責任者|CTO")),156    ("coo", re.compile(r"最高執行責任者|COO")),157    ("ceo", re.compile(r"代表取締役|最高経営責任者|社長|CEO")),158    ("chair", re.compile(r"取締役会長|会長|議長")),159    ("vp", re.compile(r"副社長|執行役員|バイスプレジデント")),160    ("board", re.compile(r"社外取締役|取締役|監査役|理事")),161    ("head", re.compile(r"本部長|事業部長|部長|統括|責任者|マネージャー|ディレクター")),162]163CHIEF_RE = re.compile(r"\b(chief\b|c[a-z]{1,3}o\b)|最高.{1,8}責任者", re.IGNORECASE)164EXEC_CATEGORIES = frozenset({"ceo", "cfo", "cto", "coo", "founder", "president", "chair"})165TITLE_WORD_RE = re.compile(r"\b(officer|manager|engineer|lead|counsel|scientist|architect|analyst|controller|secretary|treasurer|advisor|adviser|"166                           r"strategist|evangelist|designer|emeritus|emerita|executive|principal|fellow|associate|specialist|consultant|"167                           r"ingénieur|responsable|chargée?|gérant(e)?|associée?|manager|leiter(in)?|mitglied|vorstand|directeur|directrice|"168                           r"adviseur|bestuurder|gerente|socio|sócio|membro|miembro)\b", re.IGNORECASE)169TITLE_WORD_JA_RE = re.compile(r"取締役|執行役員|部長|社長|会長|役員|担当|責任者|マネージャー|ディレクター|エンジニア|顧問|監査役")170171172def role_category(title: str | None) -> tuple[str, bool]:173    """(category, is_executive) for a person's title — 'other' when nothing in the vocabulary matches."""174    if not title:175        return "other", False176    for cat, pat in ROLE_RULES:177        if pat.search(title):178            return cat, cat in EXEC_CATEGORIES or bool(CHIEF_RE.search(title))179    for cat, pat in ROLE_RULES_JA:180        if pat.search(title):181            return cat, cat in EXEC_CATEGORIES or bool(CHIEF_RE.search(title))182    return "other", bool(CHIEF_RE.search(title))183184185def looks_like_role_title(text: str) -> bool:186    t = normalize_whitespace(text)187    if not (2 <= len(t) <= MAX_PERSON_TITLE_CHARS):188        return False189    return role_category(t)[0] != "other" or bool(TITLE_WORD_RE.search(t)) or bool(TITLE_WORD_JA_RE.search(t)) or bool(CHIEF_RE.search(t))190191192# ------------------------------------------------------------------------------------------------------------ people193194_UP, _LOW = r"A-ZÀ-ÝĀ-Ž", r"a-zà-ÿā-ž'’"195NAME_TOKEN = (rf"(?:[{_UP}](?:['’][{_UP}])?[{_LOW}]*(?:[{_UP}][{_LOW}]+)?(?:-[{_UP}{_LOW}][{_LOW}]*)*|"     # Jane · O'Neill · McGrath · García-López196              rf"[{_UP}]\.|"                                                                                # initial197              rf"(?:de|van|von|der|den|ter|ten|da|di|do|dos|das|le|la|los|las|du|del|della|bin|ibn|al|el|y|e|of|op)\b|"198              rf"(?:de|van|von|da|di|le|la|du|d|l)[{_UP}][{_LOW}]+)")                                        # deSouza · DiCaprio · LeBlanc199NAME_SUFFIX = r"(?:,?\s+(?:Jr\.?|Sr\.?|II|III|IV|PhD|MD|MBA|CPA|Esq\.?))?"200NAME_RE = re.compile(rf"^(?:(?:Dr|Prof|Mr|Mrs|Ms|Sir|Dame|Hon)\.?\s+)?{NAME_TOKEN}(?:\s+{NAME_TOKEN}){{{PERSON_NAME_MIN_TOKENS - 1},{PERSON_NAME_MAX_TOKENS - 1}}}"201                     rf"{NAME_SUFFIX}$")202NAME_SUFFIX_TAIL_RE = re.compile(r"^\s*(?:Jr\.?|Sr\.?|II|III|IV|PhD|MD|MBA|CPA|Esq\.?)\s*$", re.IGNORECASE)203ABBREVIATION_END_RE = re.compile(r"\b(?:Jr|Sr|[A-Z])\.$")204JA_NAME_RE = re.compile(r"^[一-鿿぀-ヿ]{1,5}(?:[\s　・]+[一-鿿぀-ヿ]{1,5})?$")205JA_NOT_NAME_RE = re.compile(r"代表|取締役|社長|会長|役員|部長|執行|責任者|最高|担当|マネージャー|ディレクター|チーム|株式会社|有限会社|会社|事業|本部|営業|経営|"206                            r"お問い合わせ|プロフィール|詳細|一覧|紹介|概要|採用|ニュース|ホーム")207NOT_NAME_RE = re.compile(208    r"\b(team|our|meet|leadership|board|executive|management|contact|about|officer|director|directors|president|manager|head|chief|founder|"209    r"partner|partners|group|company|global|senior|vice|read|more|learn|view|profile|bio|linkedin|email|join|careers|news|press|the|and|of|for|"210    r"at|in|on|to|with|chair|chairman|emeritus|emerita|vp|advisors?|advisory|committee|members?|staff|people|employees|équipe|direction|"211    r"conseil|kontakt|vorstand|geschäftsführung|équipe de direction|equipo|dirección|squadra|direzione|diretoria|equipe)\b", re.IGNORECASE)212GENERIC_PERSON_NAMES_RE = _phrase_re([213    "contact", "contact us", "team", "our team", "the team", "meet the team", "board", "board of directors", "directors", "leadership", "our leadership",214    "leadership team", "senior leadership", "executive team", "executive committee", "management", "management team", "our people", "advisors",215    "advisory board", "about us", "kontakt", "unser team", "vorstand", "geschäftsführung", "aufsichtsrat", "équipe", "notre équipe", "direction",216    "conseil d'administration", "comité de direction", "equipo", "nuestro equipo", "dirección", "consejo", "squadra", "direzione", "equipe",217    "diretoria", "役員", "経営陣", "チーム", "お問い合わせ", "会社概要",218])219PERSON_TITLE_STOPLIST_RE = _phrase_re([220    "contact", "contact us", "read more", "read bio", "view bio", "full bio", "bio", "biography", "profile", "view profile", "linkedin", "twitter", "x",221    "email", "e-mail", "phone", "download", "download photo", "download bio", "website", "more", "learn more", "details", "en savoir plus", "voir le profil",222    "profil", "kontakt", "lebenslauf", "mehr erfahren", "biografie", "biografía", "ver perfil", "leggi la biografia", "meer info", "lees meer",223    "プロフィール", "詳細を見る", "詳細",224])225226227def _strip_name(text: str) -> str:228    """Trim punctuation around a name but keep a final abbreviation dot ("Edgar S. Woolard, Jr.")."""229    t = normalize_whitespace(text).strip(" ,:;-–—|")230    return t if ABBREVIATION_END_RE.search(t) else t.rstrip(".").strip()231232233def looks_like_person_name(text: str) -> bool:234    """2–5 capitalised tokens (particles, initials, Mc/Mac, glued particles allowed), no digits, no role vocabulary; a peerage or235    post-nominal tail after a comma ("…, Baron Trotman") is ignored; or a short CJK name without role words."""236    t = _strip_name(text)237    if not t or any(ch.isdigit() for ch in t):238        return False239    if _CJK_RE.search(t):240        return bool(JA_NAME_RE.match(t)) and not JA_NOT_NAME_RE.search(t) and 2 <= len(t.replace(" ", "").replace("　", "")) <= 10241    head, _, tail = t.partition(",")242    core = t if (not tail or NAME_SUFFIX_TAIL_RE.match(tail)) else head.strip()243    if not (MIN_PERSON_NAME_CHARS <= len(core) <= MAX_PERSON_NAME_CHARS):244        return False245    if NOT_NAME_RE.search(core) or GENERIC_PERSON_NAMES_RE.match(core):246        return False247    return bool(NAME_RE.match(core))248249250def clean_person_title(title: str | None) -> str | None:251    """Stoplisted link labels (Contact, Read more, LinkedIn…), bio sentences and over-long strings are not titles."""252    if not title:253        return None254    t = strip_trailing_marks(title)255    if not t or PERSON_TITLE_STOPLIST_RE.match(t) or len(t) > MAX_PERSON_TITLE_CHARS:256        return None257    if word_count(t) >= SENTENCE_TITLE_MIN_WORDS and t.endswith((".", "!", "?")):258        return None259    if is_prose(t) and not looks_like_role_title(t):260        return None261    return t.rstrip(".") if not t.endswith("...") else t262263264def normalize_person(name: str, title: str | None) -> tuple[str, str | None] | None:265    """Validated (name, title) — swapped when the card put the title first — or None when the row is not a person."""266    n = _strip_name(name)267    t = normalize_whitespace(title).strip(" ,:;-–—|") if title else None      # keep a final "." — it marks a bio sentence268    if not looks_like_person_name(n):269        if t and looks_like_person_name(t) and looks_like_role_title(n):270            n, t = t, n271        else:272            return None273    return n, clean_person_title(t)274275276def person_verdict(name: str, title: str | None) -> Verdict:277    fixed = normalize_person(name, title)278    if fixed is None:279        if looks_like_role_title(name):280            return Verdict.reject("name is a role title")281        if GENERIC_PERSON_NAMES_RE.match(_strip_name(name)):282            return Verdict.reject("generic label, not a person")283        return Verdict.reject("not a person name")284    return Verdict.accept()285286287# ------------------------------------------------------------------------------------------------------------ jobs288289JOB_ROLE_RE = re.compile(290    r"\b(?:engineer|ingénieur|ingenieur|ingeniero|ingegnere|engenheiro|developer|développeur|entwickler|desarrollador|sviluppatore|desenvolvedor|"291    r"ontwikkelaar|programmer|programmeur|manager|gestionnaire|gerente|analyst|analyste|analista|designer|director|directeur|directrice|direktor|"292    r"diretor|direttore|specialist|spécialiste|spezialist|especialista|specialista|intern|internship|stagiaire|praktikant|praktikum|stagiair|stagista|"293    r"tirocinio|tirocinante|becario|estagiário|estágio|alternan(?:t|ce)|apprenti|apprentice|auszubildende[rn]?|ausbildung|azubi|werkstudent|"294    r"sales|consultant|consulente|consultor|berater|adviseur|technician|technicien|techniker|técnico|tecnico|monteur|nurse|infirmi(?:er|ère)|"295    r"krankenpfleger|verpleegkundige|enfermer[oa]|infermier[ea]|pflegefachkraft|pflegekraft|driver|chauffeur|fahrer|conductor|autista|motorista|officer|"296    r"lead|head|associate|coordinator|coordinat(?:eur|rice)|koordinator|coördinator|coordinador|coordinatore|coordenador|architect|architecte|"297    r"architekt|arquitecto|architetto|arquiteto|scientist|scientifique|wissenschaftler|científico|scienziato|cientista|accountant|comptable|"298    r"buchhalter|contable|contabile|contador|operator|opérateur|operador|operatore|mechanic|mécanicien|mechaniker|mecánico|meccanico|mecânico|"299    r"representative|représentant|assistant|assistent|asistente|assistente|administrator|administrateur|administrador|amministratore|advisor|"300    r"adviser|counsel|attorney|lawyer|avocat|jurist|recruiter|controller|supervisor|superviseur|planner|buyer|acheteur|einkäufer|trainee|graduate|"301    r"student|étudiant|studierende|estudiante|responsable|chargée?|chef de|leiter(?:in)?|mitarbeiter(?:in)?|referent(?:in)?|sachbearbeiter(?:in)?|"302    r"kaufmann|kauffrau|fachkraft|medewerker|teamleider|projectleider|projektleiter|accountmanager|jefe|addetto|impiegato|responsabile|executive|"303    r"principal|partner|paralegal|pharmacist|physician|therapist|teacher|professor|researcher|chercheur|forscher|electrician|électricien|elektriker|"304    r"plumber|welder|soudeur|schweißer|machinist|cook|chef|barista|cashier|clerk|agent|guard|cleaner|handler|installer|inspector|auditor|actuary|"305    r"underwriter|banker|trader|economist|strategist|producer|editor|writer|rédacteur|redakteur|translator|marketer|merchandiser|dispatcher|steward|"306    r"pilot|captain|technologist|veterinarian|dentist|dietitian|paramedic|caregiver|educator|industriemechaniker|softwareentwickler|elektroniker|"307    r"mechatroniker|pfleger|erzieher(?:in)?|verkäufer(?:in)?|vendeur|vendeuse|verkoper|commercial|comercial|venditore|vendedor)s?\b|"308    r"エンジニア|マネージャー|マネジャー|デザイナー|ディレクター|コンサルタント|スペシャリスト|アナリスト|セールス|営業|開発|正社員|契約社員|インターン|職|担当", re.IGNORECASE)309JOB_FORMAT_RE = re.compile(r"\(\s*(?:[mfwhvdx]|all genders|alle geschlechter|tous genres)(?:\s*/\s*[mfwhvdx*])*\s*\)|\b(?:full[- ]?time|part[- ]?time|vollzeit|"310                           r"teilzeit|temps plein|temps partiel|fulltime|parttime|cdi|cdd|freelance|\d{2,3}\s?%)\b", re.IGNORECASE)311GENDER_MARK_RE = re.compile(r"\s*\(\s*(?:[mfwhvdx](?:\s*/\s*[mfwhvdx*]){1,3}|all genders|alle geschlechter|tous genres|m/f/d|h/f)\s*\)", re.IGNORECASE)312APPLY_SUFFIX_RE = re.compile(r"\s*[-–—|·]\s*(?:apply(?: now)?|postuler|jetzt bewerben|bewerben|solliciteer(?: nu)?|candidati|candidatar|aplicar)\s*$", re.IGNORECASE)313BRACKET_ID_RE = re.compile(r"\s*[\[(]\s*(?:#\s*|(?:job\s*id|ref\.?|id|req)(?:\s*[:#]\s*|\s+))?([A-Z]{0,4}[-_]?\d{3,})\s*[\])]", re.IGNORECASE)314JOB_STOP_EXACT_RE = _phrase_re([315    "read more", "learn more", "more info", "more information", "more", "info", "details", "view details", "view job", "view all", "view all jobs",316    "view all openings", "view openings", "see all", "see all jobs", "see more", "see open roles", "search jobs", "search", "job search", "all jobs",317    "all openings", "all positions", "open positions", "open roles", "openings", "jobs", "careers", "career", "apply", "apply now", "apply here",318    "join us", "join our team", "join the team", "find out more", "discover more", "explore", "explore roles", "back", "back to top", "load more",319    "show more", "next", "previous", "home", "filter", "filters", "sort by", "share", "save", "print", "benefits", "culture", "our culture",320    "our values", "diversity", "faq", "hiring process", "talent community", "talent network", "job alerts", "job alert", "sign up for job alerts",321    "meer info", "meer informatie", "lees meer", "solliciteer", "solliciteer nu", "alle vacatures", "bekijk alle vacatures", "bekijk vacature",322    "vacatures", "ontdek meer", "meer weten", "en savoir plus", "lire la suite", "postuler", "postulez", "voir toutes les offres", "toutes les offres",323    "voir l'offre", "voir plus", "découvrir", "nos offres", "offres d'emploi", "mehr erfahren", "mehr infos", "jetzt bewerben", "bewerben",324    "alle stellen", "alle stellenangebote", "stellenangebote", "alle jobs", "offene stellen", "weiterlesen", "mehr anzeigen", "zur stelle", "karriere",325    "ver más", "leer más", "aplicar", "aplica ahora", "postúlate", "ver todas las ofertas", "todas las ofertas", "más información", "saber más",326    "empleo", "ofertas de empleo", "scopri di più", "leggi tutto", "candidati", "candidati ora", "tutte le posizioni", "tutte le offerte",327    "maggiori informazioni", "posizioni aperte", "lavora con noi", "saiba mais", "ler mais", "candidatar", "candidate-se", "ver todas as vagas",328    "todas as vagas", "mais informações", "vagas", "詳細を見る", "もっと見る", "応募する", "エントリー", "募集一覧", "採用情報", "募集要項",329])330JOB_STOP_PREFIX_RE = re.compile(r"^(?:early careers?|graduate programm?e|graduates?\b|students?\b|starters\b|careers? (?:at|bij|chez|bei)\b|jobs? (?:at|bij|chez|bei)\b|"331                                r"working at\b|life at\b|why (?:join|work|us)\b|join us\b|about us\b|who we are\b|what we do\b|meet our\b|our (?:story|teams?|culture|"332                                r"values|benefits|people)\b|werken bij\b|waarom\b|pourquoi\b|warum\b|karriere bei\b|travailler chez\b|arbeiten bei\b|"333                                r"trabajar en\b|lavorare in\b|trabalhar na\b)", re.IGNORECASE)334JOB_URL_STRONG_RE = re.compile(335    r"(?:/jobs?/[^/?#]+|/careers?/[^/?#]+/[^/?#]+|/positions?/|/openings?/|/vacanc(?:y|ies)/[^/?#]+|/vacatures?/[^/?#]+|/stellen(?:angebote?)?/[^/?#]+|"336    r"/stelle/|/offres?(?:-d-?emploi)?/[^/?#]+|/emplois?/[^/?#]+|/empleos?/[^/?#]+|/lavoro/[^/?#]+|/vagas?/[^/?#]+|/opportunit(?:y|ies)/[^/?#]+|"337    r"/\d{4,}(?:[/-]|$)|[?&](?:gh_jid|jobid|job_id|jid|reqid|req_id|id)=\d+|lever\.co/|greenhouse\.io/|ashbyhq\.com/|myworkdayjobs\.com/|"338    r"smartrecruiters\.com/|workable\.com/|recruitee\.com/|personio\.(?:de|com)/|teamtailor\.com/|bamboohr\.com/|jobvite\.com/|icims\.com/|"339    r"taleo\.net/|successfactors\.(?:com|eu)/|breezy\.hr/|applytojob\.com/|eightfold\.ai/|phenompeople\.com/|avature\.net/|csod\.com/|oraclecloud\.com/)",340    re.IGNORECASE)341KNOWN_LOCATION_WORDS_RE = re.compile(r"\b(?:remote|hybrid|on-?site|télétravail|homeoffice|home office|thuiswerken|worldwide|anywhere)\b", re.IGNORECASE)342343344def clean_job_title(title: str) -> tuple[str, str | None]:345    """Display title without gender markers, "- Apply" suffixes and bracketed ids; the id found (if any) is returned for the fingerprint."""346    t = strip_trailing_marks(title)347    found_id: str | None = None348    m = BRACKET_ID_RE.search(t)349    if m:350        found_id = m.group(1)351        t = (t[:m.start()] + " " + t[m.end():]).strip()352    t = GENDER_MARK_RE.sub("", t)353    t = APPLY_SUFFIX_RE.sub("", t)354    return normalize_whitespace(t).strip(" -–—|·,"), found_id355356357def job_url_is_joblike(url: str | None) -> bool:358    return bool(url) and bool(JOB_URL_STRONG_RE.search(url or ""))359360361def states_job_location(text: str | None) -> bool:362    if not text or len(text) > MAX_LOCATION_NAME_CHARS:363        return False364    p = parse_location(text)365    return bool(p["city"] or p["country"] or p["remote"] or p["region"]) or is_known_city(text) or bool(KNOWN_LOCATION_WORDS_RE.search(text))366367368def _department_like(text: str | None) -> bool:369    return bool(text) and len(text or "") <= MAX_DEPARTMENT_CHARS and word_count(text or "") <= MAX_DEPARTMENT_WORDS and not any(ch.isdigit() for ch in text or "")370371372def job_verdict(title: str, *, url: str | None = None, location: str | None = None, department: str | None = None) -> Verdict:373    """CTA / navigation anchors are not jobs; a job needs role vocabulary, a job-like URL or an explicit location/department cell."""374    raw = normalize_whitespace(title)375    if ELLIPSIS_RE.search(raw):376        return Verdict.reject("truncated title (ellipsis)")377    t, _ = clean_job_title(raw)378    if not (MIN_JOB_TITLE_CHARS <= len(t) <= MAX_JOB_TITLE_CHARS):379        return Verdict.reject("title length")380    if JOB_STOP_EXACT_RE.match(t) or JOB_STOP_PREFIX_RE.match(t):381        return Verdict.reject("call-to-action / navigation label")382    if word_count(t) < MIN_JOB_TITLE_WORDS and not _CJK_RE.search(t):383        return Verdict.reject("single-word title")384    if JOB_ROLE_RE.search(t) or JOB_FORMAT_RE.search(raw):385        return Verdict.accept()386    if is_prose(t):387        return Verdict.reject("reads like a sentence")388    if job_url_is_joblike(url) or states_job_location(location) or _department_like(department):389        return Verdict.accept()390    return Verdict.reject("no job-like signal (vocabulary, url, location)")391392393def refine_job(job: ExtractedJob) -> ExtractedJob | None:394    """Apply `job_verdict`, clean the display title and keep a bracketed id as `external_id` (so the fingerprint keeps it)."""395    if not job_verdict(job.title, url=job.url, location=job.location_text, department=job.department).ok:396        return None397    clean, found_id = clean_job_title(job.title)398    if clean:399        job.title = clean400    if found_id and not job.external_id:401        job.external_id = found_id402    return job403404405# ------------------------------------------------------------------------------------------------------------ locations406407NAV_COOKIE_RE = re.compile(408    r"\b(?:performance|analytics|analytical|marketing|functional|functionality|necessary|essential|preferences?|strictly|targeting|advertising|"409    r"advertisement|statistics?|statistik(?:en)?|statistiques?|statistieken|estadisticas?|statistiche|tracking|cookies?|consent|privacy|privacybeleid|"410    r"datenschutz|confidentialite|privacidad|gdpr|contact us|contact|contactez|contacto|contatti|kontakt|careers?|jobs?|sitemap|terms|legal|"411    r"mentions legales|impressum|accessibility|newsletter|subscribe|login|log in|sign in|sign up|register|search|menu|language|select|filter|"412    r"view all|see all|all locations|find (?:a|an|your)|more|back|home|about us|faq|help|support|settings|einstellungen|notwendig|funktional|"413    r"necessaires?|fonctionnels?|preferences|noodzakelijk|functioneel|voorkeuren|necesarias?|funcionales?|preferencias|necessari|funzionali|"414    r"preferenze|publicidad|pubblicita|werbung|publicite|social media|session|third party|unclassified|uncategori[sz]ed|others?)\b", re.IGNORECASE)415LOCATION_KIND_WORD_RE = re.compile(416    r"\b(?:headquarters|head office|hq|office|offices|bureau|büro|kantoor|oficina|ufficio|escritório|store|shop|boutique|showroom|plant|factory|usine|"417    r"werk|fabrik|fabriek|fábrica|fabbrica|warehouse|entrepôt|lager|magazijn|almacén|lab|laboratory|laboratoire|labor|campus|branch|agence|filiale|"418    r"succursale|niederlassung|vestiging|sede|siège|hauptsitz|data ?cent(?:er|re)|distribution cent(?:er|re))\b|本社|支社|営業所|工場|オフィス|拠点", re.IGNORECASE)419VENUE_RE = re.compile(r"\b(?:hotel|tower|towers|building|centre|center|plaza|mall|park|campus|house|hall|street|avenue|road|floor|level|suite|"420                      r"hôtel|gebäude|gebouw|edificio|torre|palazzo)\b", re.IGNORECASE)421STREET_RE = re.compile(422    r"\b\d{1,5}[a-z]?\s+[^\n,]{2,40}\b(?:street|st\.?|avenue|ave\.?|road|rd\.?|boulevard|blvd\.?|drive|dr\.?|lane|ln\.?|way|place|pl\.?|square|plaza|"423    r"court|ct\.?|parkway|highway|straße|strasse|str\.|allee|platz|weg|rue|avenida|calle|via|piazza)\b"424    r"|\b(?:rue|avenue|boulevard|via|calle|avenida|carrer|rua|praça)\s+[^\n,]{2,40}\s\d{1,5}\b"425    r"|\b(?:[^\s,\d]+\s){0,2}[^\s,]{3,40}(?:straat|weg|laan|plein|kade|zijde|gracht|singel|markt|dijk|straße|strasse|str\.|allee|platz|gasse|ring|damm|ufer|"426    r"vej|gade|gatan|vägen|katu|tie)\s+\d{1,5}[a-z]?\b", re.IGNORECASE)427428# ~400 major cities and business hubs with common exonyms/endonyms; keys are accent-insensitive (see `key_of`).429_CITIES = """430new york, los angeles, chicago, houston, phoenix, philadelphia, san antonio, san diego, dallas, san jose, austin, jacksonville, fort worth, columbus,431charlotte, san francisco, indianapolis, seattle, denver, washington, boston, nashville, detroit, portland, las vegas, memphis, louisville, baltimore,432milwaukee, albuquerque, tucson, fresno, sacramento, kansas city, atlanta, miami, oakland, minneapolis, cleveland, raleigh, omaha, tampa, orlando,433pittsburgh, cincinnati, st. louis, salt lake city, richmond, new orleans, buffalo, hartford, providence, durham, boulder, palo alto, mountain view,434menlo park, redmond, bellevue, cambridge, princeton, stamford, irvine, santa clara, sunnyvale, cupertino, redwood city, san mateo, arlington, reston,435mclean, plano, scottsdale, tempe, ann arbor, madison, des moines, boise, honolulu, anchorage, charleston, savannah, oklahoma city, tulsa, el paso,436long beach, colorado springs, newark, jersey city, brooklyn, manhattan, santa monica, pasadena, burbank, anaheim, riverside, henderson, reno, spokane,437tacoma, rochester, syracuse, albany, wilmington, trenton, grand rapids, toledo, dayton, lexington, knoxville, chattanooga, huntsville, baton rouge,438little rock, tallahassee, st. petersburg, fort lauderdale, west palm beach, boca raton, greenville, columbia, greensboro, norfolk, alexandria, bethesda,439rockville, toronto, montréal, montreal, vancouver, calgary, edmonton, ottawa, winnipeg, québec, quebec city, hamilton, kitchener, waterloo, halifax,440victoria, saskatoon, regina, mississauga, brampton, markham, vaughan, burnaby, surrey, laval, gatineau, kelowna, oakville, burlington,441mexico city, ciudad de méxico, guadalajara, monterrey, puebla, tijuana, querétaro, cancún, mérida, são paulo, sao paulo, rio de janeiro, brasília, brasilia,442belo horizonte, curitiba, porto alegre, salvador, recife, fortaleza, campinas, florianópolis, manaus, buenos aires, córdoba, rosario, mendoza, santiago,443valparaíso, lima, bogotá, bogota, medellín, cali, cartagena, quito, guayaquil, caracas, montevideo, asunción, la paz, san josé, panama city,444ciudad de panamá, guatemala city, san salvador, tegucigalpa, managua, santo domingo, san juan, havana, kingston, nassau,445london, manchester, birmingham, leeds, glasgow, edinburgh, liverpool, bristol, sheffield, newcastle, nottingham, cardiff, belfast, leicester, coventry,446oxford, reading, milton keynes, brighton, southampton, portsmouth, aberdeen, dundee, york, bath, exeter, plymouth, norwich, swindon, slough, watford,447guildford, basingstoke, warrington, derby, sunderland, hull, bradford, dublin, cork, galway, limerick, waterford,448paris, marseille, lyon, toulouse, nice, nantes, strasbourg, montpellier, bordeaux, lille, rennes, reims, le havre, saint-étienne, toulon, grenoble,449dijon, angers, nîmes, villeurbanne, clermont-ferrand, le mans, aix-en-provence, brest, tours, amiens, limoges, annecy, perpignan, metz, besançon,450orléans, rouen, mulhouse, caen, nancy, boulogne-billancourt, issy-les-moulineaux, la défense, courbevoie, neuilly-sur-seine, levallois-perret, nanterre,451puteaux, saint-denis, versailles, sophia antipolis, cannes, monaco,452berlin, hamburg, münchen, munich, köln, cologne, frankfurt, frankfurt am main, stuttgart, düsseldorf, dortmund, essen, leipzig, bremen, dresden, hannover,453hanover, nürnberg, nuremberg, duisburg, bochum, wuppertal, bielefeld, bonn, münster, karlsruhe, mannheim, augsburg, wiesbaden, gelsenkirchen,454mönchengladbach, braunschweig, chemnitz, kiel, aachen, halle, magdeburg, freiburg, krefeld, lübeck, mainz, erfurt, oberhausen, rostock, kassel, hagen,455saarbrücken, potsdam, ludwigshafen, oldenburg, leverkusen, heidelberg, darmstadt, regensburg, ingolstadt, würzburg, ulm, wolfsburg, göttingen,456paderborn, heilbronn, erlangen, jena, walldorf, böblingen, sindelfingen, neckarsulm, herzogenaurach, gütersloh,457wien, vienna, graz, linz, salzburg, innsbruck, zürich, zurich, genève, geneva, genf, basel, bâle, bern, berne, lausanne, winterthur, luzern, lucerne,458st. gallen, lugano, zug, baar, vevey, neuchâtel,459amsterdam, rotterdam, den haag, the hague, utrecht, eindhoven, groningen, tilburg, almere, breda, nijmegen, arnhem, haarlem, amersfoort, enschede,460apeldoorn, 's-hertogenbosch, leiden, delft, maastricht, zwolle, hilversum, hoofddorp, schiphol, amstelveen, veldhoven, brussels, bruxelles, brussel,461antwerp, antwerpen, anvers, ghent, gent, gand, charleroi, liège, luik, bruges, brugge, namur, leuven, louvain, mechelen, mons, hasselt, kortrijk,462zaventem, diegem, luxembourg, esch-sur-alzette,463stockholm, göteborg, gothenburg, malmö, uppsala, västerås, linköping, lund, helsingborg, örebro, oslo, bergen, trondheim, stavanger, drammen,464copenhagen, københavn, aarhus, århus, odense, aalborg, helsinki, helsingfors, espoo, tampere, vantaa, oulu, turku, reykjavík, reykjavik, tallinn, riga,465vilnius, kaunas,466madrid, barcelona, valencia, sevilla, seville, zaragoza, málaga, murcia, palma, las palmas, bilbao, alicante, valladolid, vigo, gijón, a coruña,467granada, vitoria-gasteiz, san sebastián, donostia, pamplona, santander, lisboa, lisbon, porto, oporto, braga, coimbra, faro, funchal, roma, rome,468milano, milan, napoli, naples, torino, turin, palermo, genova, genoa, bologna, firenze, florence, bari, catania, venezia, venice, verona, messina,469padova, padua, trieste, brescia, parma, modena, reggio emilia, perugia, bergamo, vicenza, monza, athens, athina, thessaloniki, piraeus, nicosia,470limassol, valletta, ljubljana, zagreb, split, belgrade, beograd, sarajevo, skopje, tirana, podgorica,471warsaw, warszawa, kraków, krakow, cracow, łódź, lodz, wrocław, wroclaw, poznań, poznan, gdańsk, gdansk, szczecin, katowice, lublin, bydgoszcz, prague,472praha, brno, ostrava, plzeň, bratislava, košice, budapest, debrecen, szeged, bucharest, bucurești, cluj-napoca, timișoara, iași, sofia, plovdiv,473varna, kyiv, kiev, kharkiv, lviv, odesa, odessa, dnipro, minsk, moscow, moskva, saint petersburg, st. petersburg, novosibirsk, yekaterinburg, kazan,474chișinău, tbilisi, yerevan, baku,475istanbul, ankara, izmir, bursa, antalya, tel aviv, jerusalem, haifa, herzliya, petah tikva, beersheba, dubai, abu dhabi, sharjah, doha, riyadh, jeddah,476dammam, kuwait city, manama, muscat, amman, beirut, baghdad, tehran, cairo, giza, casablanca, rabat, marrakech, tangier, tunis, algiers, lagos, abuja,477nairobi, mombasa, accra, addis ababa, dar es salaam, kampala, kigali, johannesburg, cape town, durban, pretoria, gqeberha, luanda, maputo, lusaka,478harare, dakar, abidjan, kinshasa, douala, yaoundé, windhoek, gaborone, antananarivo, port louis, khartoum,479tokyo, 東京, osaka, 大阪, yokohama, 横浜, nagoya, 名古屋, sapporo, 札幌, fukuoka, 福岡, kobe, 神戸, kyoto, 京都, kawasaki, 川崎, saitama, hiroshima, 広島,480sendai, 仙台, chiba, 千葉, seoul, 서울, busan, 부산, incheon, 인천, daegu, daejeon, gwangju, suwon, pangyo, seongnam, beijing, 北京, peking, shanghai, 上海,481guangzhou, 广州, shenzhen, 深圳, chengdu, 成都, hangzhou, 杭州, wuhan, 武汉, xi'an, 西安, chongqing, 重庆, tianjin, 天津, nanjing, 南京, suzhou, 苏州,482qingdao, 青岛, dalian, 大连, xiamen, 厦门, shenyang, changsha, zhengzhou, dongguan, ningbo, hong kong, 香港, macau, macao, 澳门, taipei, 台北, 臺北,483taichung, kaohsiung, hsinchu, tainan, singapore, kuala lumpur, penang, george town, johor bahru, cyberjaya, petaling jaya, bangkok, กรุงเทพ,484chiang mai, jakarta, surabaya, bandung, denpasar, medan, manila, makati, quezon city, cebu, taguig, bonifacio global city, pasig, hanoi, hà nội,485ho chi minh city, hồ chí minh, saigon, da nang, phnom penh, yangon, vientiane, dhaka, chittagong, colombo, kathmandu, karachi, lahore, islamabad,486rawalpindi, faisalabad, mumbai, bombay, delhi, new delhi, bengaluru, bangalore, hyderabad, chennai, madras, kolkata, calcutta, pune, ahmedabad, jaipur,487surat, lucknow, kanpur, nagpur, indore, thane, bhopal, visakhapatnam, vadodara, coimbatore, kochi, cochin, gurgaon, gurugram, noida, chandigarh,488mysore, mysuru, thiruvananthapuram, trivandrum, bhubaneswar, mohali, navi mumbai, sydney, melbourne, brisbane, perth, adelaide, canberra,489gold coast, hobart, darwin, wollongong, geelong, auckland, wellington, christchurch, dunedin, tauranga, ulaanbaatar, almaty, astana, tashkent, bishkek490"""491CITY_KEYS: frozenset[str] = frozenset(key_of(c) for c in _CITIES.replace("\n", ",").split(",") if c.strip())492493494def is_known_city(text: str | None) -> bool:495    """Whole string or any 1–3-word n-gram of a short string names a city in the table."""496    if not text:497        return False498    k = key_of(text)499    if not k:500        return False501    if k in CITY_KEYS:502        return True503    if _CJK_RE.search(text):504        return any(city in k for city in CITY_KEYS if _CJK_RE.search(city))505    toks = k.split()506    if len(toks) > MAX_LOCATION_NAME_WORDS:507        return False508    for n in range(1, min(CITY_NGRAM_MAX, len(toks)) + 1):509        for i in range(len(toks) - n + 1):510            if " ".join(toks[i:i + n]) in CITY_KEYS:511                return True512    return False513514515CITY_DISTRICT_RE = re.compile(r"\s\d{1,2}$")          # "Dublin 2", "Paris 8" — a postal district after a known city516517518def has_postal_code(text: str | None) -> bool:519    if not text:520        return False521    return any(not YEAR_RE.match(m.group(0)) for m in POSTAL_RE.finditer(text))522523524def looks_like_city_value(text: str | None) -> bool:525    """A plausible `city` cell: short, capitalised words, no digits, no nav/cookie vocabulary, no sentence."""526    if not text:527        return False528    t = normalize_whitespace(text)529    if not (2 <= len(t) <= MAX_CITY_CHARS) or word_count(t) > MAX_CITY_WORDS:530        return False531    if any(ch.isdigit() for ch in t) and not (CITY_DISTRICT_RE.search(t) and is_known_city(CITY_DISTRICT_RE.sub("", t))):532        return False                                                                   # "Level 12" no · "Dublin 2" (postal district) yes533    if NAV_COOKIE_RE.search(key_of(t)) or is_prose(t):534        return False535    return t[0].isupper() or bool(_CJK_RE.search(t[0]))536537538def _looks_like_venue(text: str) -> bool:539    return word_count(text) >= VENUE_MIN_WORDS or bool(VENUE_RE.search(text))540541542def location_signal(*, name: str, city: str | None, region: str | None, country: str | None, kind: str | None, address: str | None) -> str | None:543    """The evidence that makes this a real place, or None: country / known city / postal code / street address / explicit kind label."""544    if country:545        return "country"546    if is_known_city(city) or is_known_city(region) or is_known_city(name):547        return "known city"548    if has_postal_code(address) or (has_postal_code(name) and "," in name):549        return "postal code"550    if STREET_RE.search(address or "") or STREET_RE.search(name):551        return "street address"552    if (kind and kind not in ("office", "other")) or LOCATION_KIND_WORD_RE.search(name):553        return "kind label"554    return None555556557def normalize_location(loc: ExtractedLocation) -> ExtractedLocation | None:558    """Validated copy of `loc` (city cell cleaned, country-as-name repaired) or None when it is not a place."""559    name = strip_trailing_marks(loc.name or "")560    if not name or len(name) > MAX_LOCATION_NAME_CHARS or word_count(name) > MAX_LOCATION_NAME_WORDS:561        return None562    if NAV_COOKIE_RE.search(key_of(name)) or is_prose(name):563        return None564    city = loc.city if looks_like_city_value(loc.city) else None565    region = loc.region566    country = loc.country567    name_country = country_code(name)568    if name_country and city and key_of(city) == key_of(name) and not is_known_city(name):569        city = None                                                                            # "Oman | Oman": a country is not its own city (Singapore is)570    if name_country and loc.city and (city is None or _looks_like_venue(loc.city)) and not NAV_COOKIE_RE.search(key_of(loc.city)) and not is_prose(loc.city) \571            and len(loc.city) <= MAX_LOCATION_NAME_CHARS and key_of(loc.city) != key_of(name):572        name, city, country = normalize_whitespace(loc.city), None, country or name_country   # "Oman | Hormuz Grand Hotel" → venue named, country kept573    if region and is_known_city(region) and looks_like_city_value(region) and (city is None or not is_known_city(city)):574        city, region = region, None                                                            # "3089 JH Rotterdam" parsed as a region575    if location_signal(name=name, city=city, region=region, country=country, kind=loc.kind, address=loc.address_text) is None:576        return None577    return ExtractedLocation(name=name, kind=loc.kind or "office", city=city, region=region, country=country, address_text=loc.address_text)578579580def location_verdict(loc: ExtractedLocation) -> Verdict:581    name = strip_trailing_marks(loc.name or "")582    if not name:583        return Verdict.reject("empty name")584    if len(name) > MAX_LOCATION_NAME_CHARS or word_count(name) > MAX_LOCATION_NAME_WORDS:585        return Verdict.reject("name too long")586    if NAV_COOKIE_RE.search(key_of(name)):587        return Verdict.reject("cookie/consent/navigation vocabulary")588    if is_prose(name):589        return Verdict.reject("name is a sentence")590    fixed = normalize_location(loc)591    if fixed is None:592        return Verdict.reject("no place evidence (country, city, postal code, street, kind)")593    return Verdict.accept()594595596# ------------------------------------------------------------------------------------------------------------ pricing plans597598PLAN_EYEBROW_RE = _phrase_re(["most popular", "popular", "best value", "recommended", "new", "beta", "coming soon", "limited offer", "best seller",599                              "le plus populaire", "beliebt", "am beliebtesten", "meest gekozen", "más popular", "più popolare", "mais popular", "人気",600                              # generic section headings that are not tiers601                              "plans", "plan", "pricing", "prices", "price", "plans & pricing", "plans and pricing", "our plans", "compare plans",602                              "choose your plan", "tarifs", "nos tarifs", "abonnements", "preise", "unsere preise", "tarife", "prijzen", "abonnementen",603                              "precios", "planes", "prezzi", "piani", "preços", "planos", "料金", "料金プラン", "プラン"])604PRICE_TOKEN_RE = re.compile(605    r"(?:(?:US\$|CA\$|C\$|A\$|NZ\$|HK\$|S\$|MX\$|R\$|USD|EUR|GBP|CAD|AUD|CHF|JPY|INR|SEK|NOK|DKK|PLN|BRL|CNY|RMB|SGD|[$€£¥₹₩₺])\s?"606    r"\d{1,3}(?:[,.\s]\d{3})*(?:[.,]\d{1,2})?|\d{1,3}(?:[,.\s]\d{3})*(?:[.,]\d{1,2})?\s?(?:€|£|USD|EUR|GBP|CHF|kr|zł|元|\$))", re.IGNORECASE)607PRICE_LEAD_RE = re.compile(r"(?:from|starting at|starts at|as low as|à partir de|ab|desde|a partire da|vanaf|only|just)\s*$", re.IGNORECASE)608PRICE_UNIT = (r"(?:month|mo|year|yr|annum|week|wk|day|hour|hr|user|seat|member|license|licence|agent|editor|contact|1,?000|1k|GB|TB|request|"609              r"transaction|call|minute|mois|an|année|monat|jahr|nutzer|benutzer|utilisateur|usuario|utente|mese|anno|mes|año|maand|jaar|gebruiker|"610              r"dag|tag|giorno|día|dia|jour)")611PRICE_TAIL_RE = re.compile(rf"^\s*(?:(?:/|per|a|an|each|every|par|pro|pour|por|al|je)\s*{PRICE_UNIT}\b(?:\s*(?:/|per|par|pro|pour|por)\s*{PRICE_UNIT}\b)?|"612                           rf"(?:monthly|annually|yearly|mensuel|annuel|monatlich|jährlich|one[- ]time|lifetime|forever))?"613                           r"(?:,?\s*billed (?:monthly|annually|yearly))?", re.IGNORECASE)614CONTACT_PHRASE_RE = re.compile(r"\b(contact (?:us|sales)|talk to (?:us|sales|an expert)|custom pricing|custom quote|get a quote|request (?:a )?(?:quote|demo)|"615                               r"let'?s talk|on request|upon request|sur devis|nous contacter|contactez-nous|auf anfrage|individuell|tailored|bespoke|call us)\b",616                               re.IGNORECASE)617FREE_TOKEN_RE = re.compile(r"^(free|gratuit|kostenlos|gratis|\$\s?0(?:\.00)?|0\s?€|€\s?0)\b", re.IGNORECASE)618619620def price_text_from(text: str) -> str | None:621    """The price phrase stated in `text` — currency + amount + period/unit, or the contact-sales / free wording — never a cut sentence."""622    t = normalize_whitespace(text)623    m = PRICE_TOKEN_RE.search(t)624    if m:625        lead = PRICE_LEAD_RE.search(t[:m.start()])626        start = lead.start() if lead else m.start()627        tail = PRICE_TAIL_RE.match(t[m.end():])628        end = m.end() + (tail.end() if tail else 0)629        return t[start:end].strip(" ,")[:MAX_PRICE_TEXT_CHARS]630    c = CONTACT_PHRASE_RE.search(t)631    if c:632        phrase = c.group(1)633        return phrase[0].upper() + phrase[1:]634    f = FREE_TOKEN_RE.match(t)635    if f:636        return f.group(1)637    return None638639640def plan_name_ok(name: str | None) -> bool:641    if not name:642        return False643    n = normalize_whitespace(name)644    if not (1 <= len(n) <= MAX_PLAN_NAME_CHARS) or word_count(n) > MAX_PLAN_NAME_WORDS:645        return False646    if has_terminal_punct(n) or "|" in n or "?" in n or PLAN_EYEBROW_RE.match(n) or re.search(r"\d+\s?%", n):647        return False648    if MARKETING_VERB_RE.match(n) and word_count(n) > 1:649        return False650    return not PRICE_TOKEN_RE.search(n)651652653def plan_verdict(plan: ExtractedPlan) -> Verdict:654    if not plan_name_ok(plan.plan_name):655        return Verdict.reject("plan name is a heading / marketing phrase")656    if plan.price is None and not plan.contact_sales and not (plan.price_text and FREE_TOKEN_RE.match(plan.price_text)):657        return Verdict.reject("no price, no contact-sales, no free tier")658    if plan.price_text and price_text_from(plan.price_text) is None:659        return Verdict.reject("price text does not state a price")660    return Verdict.accept()661662663# ------------------------------------------------------------------------------------------------------------ products664665PRODUCT_NAV_RE = _phrase_re([666    "overview", "products", "product", "all products", "our products", "solutions", "solution", "all solutions", "services", "our services", "features",667    "pricing", "resources", "support", "docs", "documentation", "blog", "contact", "contact us", "home", "learn more", "read more", "view all", "see all",668    "explore", "more", "get started", "sign up", "log in", "download", "compare", "industries", "platform", "use cases", "customers", "partners",669    "produits", "nos produits", "tous les produits", "en savoir plus", "produkte", "alle produkte", "unsere produkte", "lösungen", "mehr erfahren",670    "producten", "alle producten", "oplossingen", "meer info", "productos", "todos los productos", "soluciones", "ver más", "prodotti", "tutti i prodotti",671    "soluzioni", "scopri di più", "produtos", "todos os produtos", "soluções", "saiba mais", "製品", "製品一覧", "ソリューション", "サービス", "詳細", "もっと見る",672])673674675def product_verdict(name: str) -> Verdict:676    n = strip_trailing_marks(name)677    if not (2 <= len(n) <= MAX_PRODUCT_NAME_CHARS):678        return Verdict.reject("name length")679    if word_count(n) > MAX_PRODUCT_NAME_WORDS:680        return Verdict.reject("name too long")681    if PRODUCT_NAV_RE.match(n.rstrip("®™ ")):682        return Verdict.reject("navigation label")683    if has_terminal_punct(n) and word_count(n) >= 3:684        return Verdict.reject("name is a sentence")685    if is_slogan(n) or is_prose(n):686        return Verdict.reject("marketing slogan")687    return Verdict.accept()688689690# ------------------------------------------------------------------------------------------------------------ news691692NEWS_NOISE_RE = _phrase_re([693    "read more", "read more news", "learn more", "more", "more news", "continue reading", "view all", "see all", "all news", "all posts", "all articles",694    "next", "previous", "older", "newer", "older posts", "newer posts", "load more", "show more", "page", "news", "press", "press releases", "press release",695    "blog", "blog posts", "articles", "insights", "events", "media", "in the news", "case studies", "webinars", "whitepapers", "podcasts", "videos",696    "newsroom", "latest news", "category", "categories", "archive", "archives", "tags", "share", "rss", "subscribe", "newsletter", "actualités",697    "toutes les actualités", "communiqués de presse", "lire la suite", "en savoir plus", "alle news", "pressemitteilungen", "aktuelles", "weiterlesen",698    "mehr erfahren", "nieuws", "alle berichten", "persberichten", "lees meer", "noticias", "todas las noticias", "notas de prensa", "leer más", "notizie",699    "comunicati stampa", "leggi tutto", "notícias", "ler mais", "ニュース", "お知らせ", "プレスリリース", "一覧", "もっと見る",700])701PAGINATION_RE = re.compile(r"^(?:page\s*)?\d{1,4}$|^(?:[«»‹›<>]|\.\.\.|…)+$", re.IGNORECASE)702703704def news_verdict(title: str, *, url: str | None = None, published_at: datetime | None = None) -> Verdict:705    t = strip_trailing_marks(title)706    if not t or PAGINATION_RE.match(t):707        return Verdict.reject("pagination")708    if NEWS_NOISE_RE.match(t):709        return Verdict.reject("navigation / category label")710    confirmed = published_at is not None or date_from_url(url) is not None711    long_enough = word_count(t) >= NEWS_MIN_WORDS or len(t) >= NEWS_MIN_CHARS or (bool(_CJK_RE.search(t)) and len(t) >= NEWS_MIN_CJK_CHARS)712    if not confirmed and not long_enough:713        return Verdict.reject("too short without a date")714    return Verdict.accept()715716717# ------------------------------------------------------------------------------------------------------------ whole extraction718719HTML_JOB_CONNECTORS = frozenset({"generic-html-v1"})720721722def apply_precision(ex: Extraction, *, html_jobs: bool) -> dict[str, int]:723    """Filter/normalise every typed list in place; returns how many items each list lost. Job rules only for HTML-scraped listings."""724    dropped: dict[str, int] = {}725    if html_jobs:726        kept_jobs = [j for j in (refine_job(j) for j in ex.jobs) if j is not None]727        dropped["jobs"] = len(ex.jobs) - len(kept_jobs)728        ex.jobs = kept_jobs729    people: list[ExtractedPerson] = []730    for p in ex.people:731        fixed = normalize_person(p.name, p.title)732        if fixed is None:733            continue734        p.name, p.title = fixed735        p.role_category, p.is_executive = role_category(p.title)736        people.append(p)737    dropped["people"] = len(ex.people) - len(people)738    ex.people = people739    products: list[ExtractedProduct] = [p for p in ex.products if product_verdict(p.name).ok]740    dropped["products"] = len(ex.products) - len(products)741    ex.products = products742    plans: list[ExtractedPlan] = [p for p in ex.plans if plan_verdict(p).ok]743    dropped["plans"] = len(ex.plans) - len(plans)744    ex.plans = plans745    locations: list[ExtractedLocation] = [loc for loc in (normalize_location(loc) for loc in ex.locations) if loc is not None]746    dropped["locations"] = len(ex.locations) - len(locations)747    ex.locations = locations748    news: list[ExtractedNewsItem] = [n for n in ex.news if news_verdict(n.title, url=n.url, published_at=n.published_at).ok]749    dropped["news"] = len(ex.news) - len(news)750    ex.news = news751    return {k: v for k, v in dropped.items() if v}752753754def describe(verdict: Verdict) -> dict[str, Any]:755    return {"ok": verdict.ok, "reason": verdict.reason}756757758__all__ = [759    "CHIEF_RE", "CITY_KEYS", "EXEC_CATEGORIES", "HTML_JOB_CONNECTORS", "NAV_COOKIE_RE", "PRECISION_VERSION", "ROLE_RULES", "STREET_RE", "Verdict",760    "apply_precision", "clean_job_title", "clean_person_title", "describe", "has_postal_code", "is_known_city", "is_prose", "is_slogan", "job_url_is_joblike",761    "job_verdict", "key_of", "location_signal", "location_verdict", "looks_like_city_value", "looks_like_person_name", "looks_like_role_title", "news_verdict",762    "normalize_location", "normalize_person", "person_verdict", "plan_name_ok", "plan_verdict", "price_text_from", "product_verdict", "refine_job",763    "role_category", "states_job_location", "strip_trailing_marks", "word_count",764]765