"""Precision rules for typed extractions (spec §107, "never fabricate"): deterministic validators and normalisers that reject the navigation / call-to-action / cookie-consent / marketing noise the generic HTML connector picks up on real corporate pages, and repair the classic confusions (person name ↔ title swapped, country used as a location name, truncated price text). Shared by `connectors/generic_html.py` (applied while extracting) and `services/pipeline._drop_corrupt_entities` (last line of defence for every connector) and by `scripts/audit_extractions.py` (measurement / purge on stored rows). Every rule is a small pure function returning a `Verdict`, so the same code explains *why* a row is rejected. Multilingual where cheap (EN/FR/DE/NL/ES/IT/PT/JA). No network, no LLM. """ from __future__ import annotations import re import unicodedata from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime from typing import Any from companyatlas.connectors._util import POSTAL_RE, country_code, date_from_url, parse_location from companyatlas.sdk.models import ( ExtractedJob, ExtractedLocation, ExtractedNewsItem, ExtractedPerson, ExtractedPlan, ExtractedProduct, Extraction, ) from companyatlas.sdk.normalize import normalize_whitespace PRECISION_VERSION = "precision-v1" # ------------------------------------------------------------------------------------------------------------ limits (no magic numbers) MAX_JOB_TITLE_CHARS = 140 MIN_JOB_TITLE_CHARS = 4 MIN_JOB_TITLE_WORDS = 2 PROSE_MIN_WORDS = 4 # a "title" with ≥ 4 words and function words reads like a sentence PERSON_NAME_MIN_TOKENS, PERSON_NAME_MAX_TOKENS = 2, 5 MIN_PERSON_NAME_CHARS, MAX_PERSON_NAME_CHARS = 4, 60 MAX_PERSON_TITLE_CHARS = 100 SENTENCE_TITLE_MIN_WORDS = 6 # a person "title" of ≥ 6 words ending with a full stop is a bio sentence MAX_LOCATION_NAME_CHARS = 80 MAX_LOCATION_NAME_WORDS = 8 MAX_CITY_CHARS, MAX_CITY_WORDS = 40, 4 VENUE_MIN_WORDS = 3 # "Hormuz Grand Hotel": ≥ 3 words in the city slot is a venue, not a city MAX_PLAN_NAME_CHARS, MAX_PLAN_NAME_WORDS = 40, 5 MAX_PRICE_TEXT_CHARS = 60 MAX_PRODUCT_NAME_CHARS, MAX_PRODUCT_NAME_WORDS = 80, 10 NEWS_MIN_WORDS, NEWS_MIN_CHARS = 3, 15 NEWS_MIN_CJK_CHARS = 6 # CJK titles have no word boundaries: 6 characters already carry a headline MAX_DEPARTMENT_CHARS, MAX_DEPARTMENT_WORDS = 40, 4 CITY_NGRAM_MAX = 3 YEAR_RE = re.compile(r"^(?:19|20)\d{2}$") @dataclass(slots=True, frozen=True) class Verdict: ok: bool reason: str | None = None @staticmethod def accept() -> Verdict: return Verdict(True, None) @staticmethod def reject(reason: str) -> Verdict: return Verdict(False, reason) # ------------------------------------------------------------------------------------------------------------ text helpers _WORD_RE = re.compile(r"[^\W_]+", re.UNICODE) _CJK_RE = re.compile(r"[぀-ヿ㐀-鿿]") TERMINAL_PUNCT = ".!?:;," TRAILING_MARKS_RE = re.compile(r"[\s→➔➡›»>\-–—|·•]+$") # arrows, chevrons, dashes, bullets ELLIPSIS_RE = re.compile(r"(?:\.\.\.|…)\s*$") PROSE_FUNCTION_WORDS_RE = re.compile( r"\b(?:who|that|which|want|wants|with|for|your|you|we|our|are|is|the|die|dat|willen|voor|met|jouw|onze|wij|zijn|qui|que|pour|avec|votre|nous|" r"sont|est|les|der|das|mit|für|und|ihre|wir|sind|para|con|nuestro|somos|per|che|nostro|siamo|com|nosso|somos)\b", re.IGNORECASE) SENTENCE_VERB_RE = re.compile( r"\b(?:allows?|enables?|helps?|uses?|provides?|lets?|makes?|gives?|ensures?|collects?|stores?|can|will|would|should|permet|permettent|utilise|" r"erlaubt|ermöglicht|verwendet|gebruikt|permite|consente|permitem)\b", re.IGNORECASE) MARKETING_VERB_RE = re.compile( r"^(?:win|get|start|discover|unlock|boost|grow|try|choose|compare|see|learn|find|build|join|save|upgrade|buy|sign up|request|book|schedule|" r"contact|talk|let'?s|ready|why|how|what|explore|meet|introducing|welcome|enjoy|make|take|achieve|transform|power|scale|drive|accelerate|" r"supercharge|level up|bring|stay|experience|gagnez|obtenez|découvrez|commencez|essayez|choisissez|comparez|rejoignez|débloquez|profitez|" r"gewinnen|holen|starten|entdecken|testen|wählen|vergleichen|jetzt|ontdek|start|kies|vergelijk|probeer|gana|obtén|empieza|descubre|prueba|" r"elige|compara|scopri|inizia|prova|scegli|confronta|ottieni|ganhe|obtenha|comece|descubra|experimente|escolha)\b", re.IGNORECASE) def word_count(text: str) -> int: return len(_WORD_RE.findall(text)) def key_of(text: str | None) -> str: """Accent-insensitive, lower-case, single-spaced key (keeps CJK) used for stoplists and the city table.""" if not text: return "" nfkd = unicodedata.normalize("NFKD", text) stripped = "".join(ch for ch in nfkd if not unicodedata.combining(ch)) return " ".join(_WORD_RE.findall(stripped.lower())) def strip_trailing_marks(text: str) -> str: return TRAILING_MARKS_RE.sub("", normalize_whitespace(text)).strip() def has_terminal_punct(text: str) -> bool: t = text.rstrip("®™ ") return bool(t) and t[-1] in TERMINAL_PUNCT def is_prose(text: str) -> bool: """A label that reads like a sentence: several words with function words, a sentence verb, or a full stop at the end.""" words = word_count(text) if words < PROSE_MIN_WORDS: return False if text.rstrip().endswith((".", "!", "?")): return True return bool(SENTENCE_VERB_RE.search(text)) or len(PROSE_FUNCTION_WORDS_RE.findall(text)) >= 2 def is_slogan(text: str) -> bool: return bool(MARKETING_VERB_RE.match(text)) and word_count(text) >= PROSE_MIN_WORDS def _phrase_re(phrases: Iterable[str]) -> re.Pattern[str]: alts = sorted({p.strip().lower() for p in phrases if p.strip()}, key=len, reverse=True) return re.compile(r"^(?:" + "|".join(re.escape(a) for a in alts) + r")$", re.IGNORECASE) # ------------------------------------------------------------------------------------------------------------ role vocabulary (people) ROLE_RULES: list[tuple[str, re.Pattern[str]]] = [ ("founder", re.compile(r"\b(co-?founder|founder|fondat(eur|rice)|gründer(in)?|fondator[ei]|fundador(a)?|oprichter|medeoprichter)\b", re.IGNORECASE)), ("ceo", re.compile(r"\b(chief executive( officer)?|ceo|pdg|président-directeur|geschäftsführer(in)?|managing director|directeur général|" r"algemeen directeur|consejero delegado|amministratore delegato|diretor executivo)\b", re.IGNORECASE)), ("cfo", re.compile(r"\b(chief financial( officer)?|cfo|finanzvorstand|directeur financier|financieel directeur|director financiero|direttore finanziario)\b", re.IGNORECASE)), ("cto", re.compile(r"\b(chief technology( officer)?|chief technical( officer)?|cto|directeur technique)\b", re.IGNORECASE)), ("coo", re.compile(r"\b(chief operating( officer)?|coo|directeur des opérations)\b", re.IGNORECASE)), ("chair", re.compile(r"\b(chair(man|woman|person)?|executive chair|présidente? du conseil|vorsitzende[rn]?|voorzitter|presidente del consejo|" r"presidente del consiglio)\b", re.IGNORECASE)), ("president", re.compile(r"\b(? tuple[str, bool]: """(category, is_executive) for a person's title — 'other' when nothing in the vocabulary matches.""" if not title: return "other", False for cat, pat in ROLE_RULES: if pat.search(title): return cat, cat in EXEC_CATEGORIES or bool(CHIEF_RE.search(title)) for cat, pat in ROLE_RULES_JA: if pat.search(title): return cat, cat in EXEC_CATEGORIES or bool(CHIEF_RE.search(title)) return "other", bool(CHIEF_RE.search(title)) def looks_like_role_title(text: str) -> bool: t = normalize_whitespace(text) if not (2 <= len(t) <= MAX_PERSON_TITLE_CHARS): return False return role_category(t)[0] != "other" or bool(TITLE_WORD_RE.search(t)) or bool(TITLE_WORD_JA_RE.search(t)) or bool(CHIEF_RE.search(t)) # ------------------------------------------------------------------------------------------------------------ people _UP, _LOW = r"A-ZÀ-ÝĀ-Ž", r"a-zà-ÿā-ž'’" NAME_TOKEN = (rf"(?:[{_UP}](?:['’][{_UP}])?[{_LOW}]*(?:[{_UP}][{_LOW}]+)?(?:-[{_UP}{_LOW}][{_LOW}]*)*|" # Jane · O'Neill · McGrath · García-López rf"[{_UP}]\.|" # initial rf"(?:de|van|von|der|den|ter|ten|da|di|do|dos|das|le|la|los|las|du|del|della|bin|ibn|al|el|y|e|of|op)\b|" rf"(?:de|van|von|da|di|le|la|du|d|l)[{_UP}][{_LOW}]+)") # deSouza · DiCaprio · LeBlanc NAME_SUFFIX = r"(?:,?\s+(?:Jr\.?|Sr\.?|II|III|IV|PhD|MD|MBA|CPA|Esq\.?))?" NAME_RE = re.compile(rf"^(?:(?:Dr|Prof|Mr|Mrs|Ms|Sir|Dame|Hon)\.?\s+)?{NAME_TOKEN}(?:\s+{NAME_TOKEN}){{{PERSON_NAME_MIN_TOKENS - 1},{PERSON_NAME_MAX_TOKENS - 1}}}" rf"{NAME_SUFFIX}$") NAME_SUFFIX_TAIL_RE = re.compile(r"^\s*(?:Jr\.?|Sr\.?|II|III|IV|PhD|MD|MBA|CPA|Esq\.?)\s*$", re.IGNORECASE) ABBREVIATION_END_RE = re.compile(r"\b(?:Jr|Sr|[A-Z])\.$") JA_NAME_RE = re.compile(r"^[一-鿿぀-ヿ]{1,5}(?:[\s　・]+[一-鿿぀-ヿ]{1,5})?$") JA_NOT_NAME_RE = re.compile(r"代表|取締役|社長|会長|役員|部長|執行|責任者|最高|担当|マネージャー|ディレクター|チーム|株式会社|有限会社|会社|事業|本部|営業|経営|" r"お問い合わせ|プロフィール|詳細|一覧|紹介|概要|採用|ニュース|ホーム") NOT_NAME_RE = re.compile( r"\b(team|our|meet|leadership|board|executive|management|contact|about|officer|director|directors|president|manager|head|chief|founder|" r"partner|partners|group|company|global|senior|vice|read|more|learn|view|profile|bio|linkedin|email|join|careers|news|press|the|and|of|for|" r"at|in|on|to|with|chair|chairman|emeritus|emerita|vp|advisors?|advisory|committee|members?|staff|people|employees|équipe|direction|" r"conseil|kontakt|vorstand|geschäftsführung|équipe de direction|equipo|dirección|squadra|direzione|diretoria|equipe)\b", re.IGNORECASE) GENERIC_PERSON_NAMES_RE = _phrase_re([ "contact", "contact us", "team", "our team", "the team", "meet the team", "board", "board of directors", "directors", "leadership", "our leadership", "leadership team", "senior leadership", "executive team", "executive committee", "management", "management team", "our people", "advisors", "advisory board", "about us", "kontakt", "unser team", "vorstand", "geschäftsführung", "aufsichtsrat", "équipe", "notre équipe", "direction", "conseil d'administration", "comité de direction", "equipo", "nuestro equipo", "dirección", "consejo", "squadra", "direzione", "equipe", "diretoria", "役員", "経営陣", "チーム", "お問い合わせ", "会社概要", ]) PERSON_TITLE_STOPLIST_RE = _phrase_re([ "contact", "contact us", "read more", "read bio", "view bio", "full bio", "bio", "biography", "profile", "view profile", "linkedin", "twitter", "x", "email", "e-mail", "phone", "download", "download photo", "download bio", "website", "more", "learn more", "details", "en savoir plus", "voir le profil", "profil", "kontakt", "lebenslauf", "mehr erfahren", "biografie", "biografía", "ver perfil", "leggi la biografia", "meer info", "lees meer", "プロフィール", "詳細を見る", "詳細", ]) def _strip_name(text: str) -> str: """Trim punctuation around a name but keep a final abbreviation dot ("Edgar S. Woolard, Jr.").""" t = normalize_whitespace(text).strip(" ,:;-–—|") return t if ABBREVIATION_END_RE.search(t) else t.rstrip(".").strip() def looks_like_person_name(text: str) -> bool: """2–5 capitalised tokens (particles, initials, Mc/Mac, glued particles allowed), no digits, no role vocabulary; a peerage or post-nominal tail after a comma ("…, Baron Trotman") is ignored; or a short CJK name without role words.""" t = _strip_name(text) if not t or any(ch.isdigit() for ch in t): return False if _CJK_RE.search(t): return bool(JA_NAME_RE.match(t)) and not JA_NOT_NAME_RE.search(t) and 2 <= len(t.replace(" ", "").replace("　", "")) <= 10 head, _, tail = t.partition(",") core = t if (not tail or NAME_SUFFIX_TAIL_RE.match(tail)) else head.strip() if not (MIN_PERSON_NAME_CHARS <= len(core) <= MAX_PERSON_NAME_CHARS): return False if NOT_NAME_RE.search(core) or GENERIC_PERSON_NAMES_RE.match(core): return False return bool(NAME_RE.match(core)) def clean_person_title(title: str | None) -> str | None: """Stoplisted link labels (Contact, Read more, LinkedIn…), bio sentences and over-long strings are not titles.""" if not title: return None t = strip_trailing_marks(title) if not t or PERSON_TITLE_STOPLIST_RE.match(t) or len(t) > MAX_PERSON_TITLE_CHARS: return None if word_count(t) >= SENTENCE_TITLE_MIN_WORDS and t.endswith((".", "!", "?")): return None if is_prose(t) and not looks_like_role_title(t): return None return t.rstrip(".") if not t.endswith("...") else t def normalize_person(name: str, title: str | None) -> tuple[str, str | None] | None: """Validated (name, title) — swapped when the card put the title first — or None when the row is not a person.""" n = _strip_name(name) t = normalize_whitespace(title).strip(" ,:;-–—|") if title else None # keep a final "." — it marks a bio sentence if not looks_like_person_name(n): if t and looks_like_person_name(t) and looks_like_role_title(n): n, t = t, n else: return None return n, clean_person_title(t) def person_verdict(name: str, title: str | None) -> Verdict: fixed = normalize_person(name, title) if fixed is None: if looks_like_role_title(name): return Verdict.reject("name is a role title") if GENERIC_PERSON_NAMES_RE.match(_strip_name(name)): return Verdict.reject("generic label, not a person") return Verdict.reject("not a person name") return Verdict.accept() # ------------------------------------------------------------------------------------------------------------ jobs JOB_ROLE_RE = re.compile( r"\b(?:engineer|ingénieur|ingenieur|ingeniero|ingegnere|engenheiro|developer|développeur|entwickler|desarrollador|sviluppatore|desenvolvedor|" r"ontwikkelaar|programmer|programmeur|manager|gestionnaire|gerente|analyst|analyste|analista|designer|director|directeur|directrice|direktor|" r"diretor|direttore|specialist|spécialiste|spezialist|especialista|specialista|intern|internship|stagiaire|praktikant|praktikum|stagiair|stagista|" r"tirocinio|tirocinante|becario|estagiário|estágio|alternan(?:t|ce)|apprenti|apprentice|auszubildende[rn]?|ausbildung|azubi|werkstudent|" r"sales|consultant|consulente|consultor|berater|adviseur|technician|technicien|techniker|técnico|tecnico|monteur|nurse|infirmi(?:er|ère)|" r"krankenpfleger|verpleegkundige|enfermer[oa]|infermier[ea]|pflegefachkraft|pflegekraft|driver|chauffeur|fahrer|conductor|autista|motorista|officer|" r"lead|head|associate|coordinator|coordinat(?:eur|rice)|koordinator|coördinator|coordinador|coordinatore|coordenador|architect|architecte|" r"architekt|arquitecto|architetto|arquiteto|scientist|scientifique|wissenschaftler|científico|scienziato|cientista|accountant|comptable|" r"buchhalter|contable|contabile|contador|operator|opérateur|operador|operatore|mechanic|mécanicien|mechaniker|mecánico|meccanico|mecânico|" r"representative|représentant|assistant|assistent|asistente|assistente|administrator|administrateur|administrador|amministratore|advisor|" r"adviser|counsel|attorney|lawyer|avocat|jurist|recruiter|controller|supervisor|superviseur|planner|buyer|acheteur|einkäufer|trainee|graduate|" r"student|étudiant|studierende|estudiante|responsable|chargée?|chef de|leiter(?:in)?|mitarbeiter(?:in)?|referent(?:in)?|sachbearbeiter(?:in)?|" r"kaufmann|kauffrau|fachkraft|medewerker|teamleider|projectleider|projektleiter|accountmanager|jefe|addetto|impiegato|responsabile|executive|" r"principal|partner|paralegal|pharmacist|physician|therapist|teacher|professor|researcher|chercheur|forscher|electrician|électricien|elektriker|" r"plumber|welder|soudeur|schweißer|machinist|cook|chef|barista|cashier|clerk|agent|guard|cleaner|handler|installer|inspector|auditor|actuary|" r"underwriter|banker|trader|economist|strategist|producer|editor|writer|rédacteur|redakteur|translator|marketer|merchandiser|dispatcher|steward|" r"pilot|captain|technologist|veterinarian|dentist|dietitian|paramedic|caregiver|educator|industriemechaniker|softwareentwickler|elektroniker|" r"mechatroniker|pfleger|erzieher(?:in)?|verkäufer(?:in)?|vendeur|vendeuse|verkoper|commercial|comercial|venditore|vendedor)s?\b|" r"エンジニア|マネージャー|マネジャー|デザイナー|ディレクター|コンサルタント|スペシャリスト|アナリスト|セールス|営業|開発|正社員|契約社員|インターン|職|担当", re.IGNORECASE) JOB_FORMAT_RE = re.compile(r"\(\s*(?:[mfwhvdx]|all genders|alle geschlechter|tous genres)(?:\s*/\s*[mfwhvdx*])*\s*\)|\b(?:full[- ]?time|part[- ]?time|vollzeit|" r"teilzeit|temps plein|temps partiel|fulltime|parttime|cdi|cdd|freelance|\d{2,3}\s?%)\b", re.IGNORECASE) GENDER_MARK_RE = re.compile(r"\s*\(\s*(?:[mfwhvdx](?:\s*/\s*[mfwhvdx*]){1,3}|all genders|alle geschlechter|tous genres|m/f/d|h/f)\s*\)", re.IGNORECASE) APPLY_SUFFIX_RE = re.compile(r"\s*[-–—|·]\s*(?:apply(?: now)?|postuler|jetzt bewerben|bewerben|solliciteer(?: nu)?|candidati|candidatar|aplicar)\s*$", re.IGNORECASE) BRACKET_ID_RE = re.compile(r"\s*[\[(]\s*(?:#\s*|(?:job\s*id|ref\.?|id|req)(?:\s*[:#]\s*|\s+))?([A-Z]{0,4}[-_]?\d{3,})\s*[\])]", re.IGNORECASE) JOB_STOP_EXACT_RE = _phrase_re([ "read more", "learn more", "more info", "more information", "more", "info", "details", "view details", "view job", "view all", "view all jobs", "view all openings", "view openings", "see all", "see all jobs", "see more", "see open roles", "search jobs", "search", "job search", "all jobs", "all openings", "all positions", "open positions", "open roles", "openings", "jobs", "careers", "career", "apply", "apply now", "apply here", "join us", "join our team", "join the team", "find out more", "discover more", "explore", "explore roles", "back", "back to top", "load more", "show more", "next", "previous", "home", "filter", "filters", "sort by", "share", "save", "print", "benefits", "culture", "our culture", "our values", "diversity", "faq", "hiring process", "talent community", "talent network", "job alerts", "job alert", "sign up for job alerts", "meer info", "meer informatie", "lees meer", "solliciteer", "solliciteer nu", "alle vacatures", "bekijk alle vacatures", "bekijk vacature", "vacatures", "ontdek meer", "meer weten", "en savoir plus", "lire la suite", "postuler", "postulez", "voir toutes les offres", "toutes les offres", "voir l'offre", "voir plus", "découvrir", "nos offres", "offres d'emploi", "mehr erfahren", "mehr infos", "jetzt bewerben", "bewerben", "alle stellen", "alle stellenangebote", "stellenangebote", "alle jobs", "offene stellen", "weiterlesen", "mehr anzeigen", "zur stelle", "karriere", "ver más", "leer más", "aplicar", "aplica ahora", "postúlate", "ver todas las ofertas", "todas las ofertas", "más información", "saber más", "empleo", "ofertas de empleo", "scopri di più", "leggi tutto", "candidati", "candidati ora", "tutte le posizioni", "tutte le offerte", "maggiori informazioni", "posizioni aperte", "lavora con noi", "saiba mais", "ler mais", "candidatar", "candidate-se", "ver todas as vagas", "todas as vagas", "mais informações", "vagas", "詳細を見る", "もっと見る", "応募する", "エントリー", "募集一覧", "採用情報", "募集要項", ]) JOB_STOP_PREFIX_RE = re.compile(r"^(?:early careers?|graduate programm?e|graduates?\b|students?\b|starters\b|careers? (?:at|bij|chez|bei)\b|jobs? (?:at|bij|chez|bei)\b|" r"working at\b|life at\b|why (?:join|work|us)\b|join us\b|about us\b|who we are\b|what we do\b|meet our\b|our (?:story|teams?|culture|" r"values|benefits|people)\b|werken bij\b|waarom\b|pourquoi\b|warum\b|karriere bei\b|travailler chez\b|arbeiten bei\b|" r"trabajar en\b|lavorare in\b|trabalhar na\b)", re.IGNORECASE) JOB_URL_STRONG_RE = re.compile( r"(?:/jobs?/[^/?#]+|/careers?/[^/?#]+/[^/?#]+|/positions?/|/openings?/|/vacanc(?:y|ies)/[^/?#]+|/vacatures?/[^/?#]+|/stellen(?:angebote?)?/[^/?#]+|" r"/stelle/|/offres?(?:-d-?emploi)?/[^/?#]+|/emplois?/[^/?#]+|/empleos?/[^/?#]+|/lavoro/[^/?#]+|/vagas?/[^/?#]+|/opportunit(?:y|ies)/[^/?#]+|" r"/\d{4,}(?:[/-]|$)|[?&](?:gh_jid|jobid|job_id|jid|reqid|req_id|id)=\d+|lever\.co/|greenhouse\.io/|ashbyhq\.com/|myworkdayjobs\.com/|" r"smartrecruiters\.com/|workable\.com/|recruitee\.com/|personio\.(?:de|com)/|teamtailor\.com/|bamboohr\.com/|jobvite\.com/|icims\.com/|" r"taleo\.net/|successfactors\.(?:com|eu)/|breezy\.hr/|applytojob\.com/|eightfold\.ai/|phenompeople\.com/|avature\.net/|csod\.com/|oraclecloud\.com/)", re.IGNORECASE) KNOWN_LOCATION_WORDS_RE = re.compile(r"\b(?:remote|hybrid|on-?site|télétravail|homeoffice|home office|thuiswerken|worldwide|anywhere)\b", re.IGNORECASE) def clean_job_title(title: str) -> tuple[str, str | None]: """Display title without gender markers, "- Apply" suffixes and bracketed ids; the id found (if any) is returned for the fingerprint.""" t = strip_trailing_marks(title) found_id: str | None = None m = BRACKET_ID_RE.search(t) if m: found_id = m.group(1) t = (t[:m.start()] + " " + t[m.end():]).strip() t = GENDER_MARK_RE.sub("", t) t = APPLY_SUFFIX_RE.sub("", t) return normalize_whitespace(t).strip(" -–—|·,"), found_id def job_url_is_joblike(url: str | None) -> bool: return bool(url) and bool(JOB_URL_STRONG_RE.search(url or "")) def states_job_location(text: str | None) -> bool: if not text or len(text) > MAX_LOCATION_NAME_CHARS: return False p = parse_location(text) return bool(p["city"] or p["country"] or p["remote"] or p["region"]) or is_known_city(text) or bool(KNOWN_LOCATION_WORDS_RE.search(text)) def _department_like(text: str | None) -> bool: return bool(text) and len(text or "") <= MAX_DEPARTMENT_CHARS and word_count(text or "") <= MAX_DEPARTMENT_WORDS and not any(ch.isdigit() for ch in text or "") def job_verdict(title: str, *, url: str | None = None, location: str | None = None, department: str | None = None) -> Verdict: """CTA / navigation anchors are not jobs; a job needs role vocabulary, a job-like URL or an explicit location/department cell.""" raw = normalize_whitespace(title) if ELLIPSIS_RE.search(raw): return Verdict.reject("truncated title (ellipsis)") t, _ = clean_job_title(raw) if not (MIN_JOB_TITLE_CHARS <= len(t) <= MAX_JOB_TITLE_CHARS): return Verdict.reject("title length") if JOB_STOP_EXACT_RE.match(t) or JOB_STOP_PREFIX_RE.match(t): return Verdict.reject("call-to-action / navigation label") if word_count(t) < MIN_JOB_TITLE_WORDS and not _CJK_RE.search(t): return Verdict.reject("single-word title") if JOB_ROLE_RE.search(t) or JOB_FORMAT_RE.search(raw): return Verdict.accept() if is_prose(t): return Verdict.reject("reads like a sentence") if job_url_is_joblike(url) or states_job_location(location) or _department_like(department): return Verdict.accept() return Verdict.reject("no job-like signal (vocabulary, url, location)") def refine_job(job: ExtractedJob) -> ExtractedJob | None: """Apply `job_verdict`, clean the display title and keep a bracketed id as `external_id` (so the fingerprint keeps it).""" if not job_verdict(job.title, url=job.url, location=job.location_text, department=job.department).ok: return None clean, found_id = clean_job_title(job.title) if clean: job.title = clean if found_id and not job.external_id: job.external_id = found_id return job # ------------------------------------------------------------------------------------------------------------ locations NAV_COOKIE_RE = re.compile( r"\b(?:performance|analytics|analytical|marketing|functional|functionality|necessary|essential|preferences?|strictly|targeting|advertising|" r"advertisement|statistics?|statistik(?:en)?|statistiques?|statistieken|estadisticas?|statistiche|tracking|cookies?|consent|privacy|privacybeleid|" r"datenschutz|confidentialite|privacidad|gdpr|contact us|contact|contactez|contacto|contatti|kontakt|careers?|jobs?|sitemap|terms|legal|" r"mentions legales|impressum|accessibility|newsletter|subscribe|login|log in|sign in|sign up|register|search|menu|language|select|filter|" r"view all|see all|all locations|find (?:a|an|your)|more|back|home|about us|faq|help|support|settings|einstellungen|notwendig|funktional|" r"necessaires?|fonctionnels?|preferences|noodzakelijk|functioneel|voorkeuren|necesarias?|funcionales?|preferencias|necessari|funzionali|" r"preferenze|publicidad|pubblicita|werbung|publicite|social media|session|third party|unclassified|uncategori[sz]ed|others?)\b", re.IGNORECASE) LOCATION_KIND_WORD_RE = re.compile( r"\b(?:headquarters|head office|hq|office|offices|bureau|büro|kantoor|oficina|ufficio|escritório|store|shop|boutique|showroom|plant|factory|usine|" r"werk|fabrik|fabriek|fábrica|fabbrica|warehouse|entrepôt|lager|magazijn|almacén|lab|laboratory|laboratoire|labor|campus|branch|agence|filiale|" r"succursale|niederlassung|vestiging|sede|siège|hauptsitz|data ?cent(?:er|re)|distribution cent(?:er|re))\b|本社|支社|営業所|工場|オフィス|拠点", re.IGNORECASE) VENUE_RE = re.compile(r"\b(?:hotel|tower|towers|building|centre|center|plaza|mall|park|campus|house|hall|street|avenue|road|floor|level|suite|" r"hôtel|gebäude|gebouw|edificio|torre|palazzo)\b", re.IGNORECASE) STREET_RE = re.compile( r"\b\d{1,5}[a-z]?\s+[^\n,]{2,40}\b(?:street|st\.?|avenue|ave\.?|road|rd\.?|boulevard|blvd\.?|drive|dr\.?|lane|ln\.?|way|place|pl\.?|square|plaza|" r"court|ct\.?|parkway|highway|straße|strasse|str\.|allee|platz|weg|rue|avenida|calle|via|piazza)\b" r"|\b(?:rue|avenue|boulevard|via|calle|avenida|carrer|rua|praça)\s+[^\n,]{2,40}\s\d{1,5}\b" r"|\b(?:[^\s,\d]+\s){0,2}[^\s,]{3,40}(?:straat|weg|laan|plein|kade|zijde|gracht|singel|markt|dijk|straße|strasse|str\.|allee|platz|gasse|ring|damm|ufer|" r"vej|gade|gatan|vägen|katu|tie)\s+\d{1,5}[a-z]?\b", re.IGNORECASE) # ~400 major cities and business hubs with common exonyms/endonyms; keys are accent-insensitive (see `key_of`). _CITIES = """ new york, los angeles, chicago, houston, phoenix, philadelphia, san antonio, san diego, dallas, san jose, austin, jacksonville, fort worth, columbus, charlotte, san francisco, indianapolis, seattle, denver, washington, boston, nashville, detroit, portland, las vegas, memphis, louisville, baltimore, milwaukee, albuquerque, tucson, fresno, sacramento, kansas city, atlanta, miami, oakland, minneapolis, cleveland, raleigh, omaha, tampa, orlando, pittsburgh, cincinnati, st. louis, salt lake city, richmond, new orleans, buffalo, hartford, providence, durham, boulder, palo alto, mountain view, menlo park, redmond, bellevue, cambridge, princeton, stamford, irvine, santa clara, sunnyvale, cupertino, redwood city, san mateo, arlington, reston, mclean, plano, scottsdale, tempe, ann arbor, madison, des moines, boise, honolulu, anchorage, charleston, savannah, oklahoma city, tulsa, el paso, long beach, colorado springs, newark, jersey city, brooklyn, manhattan, santa monica, pasadena, burbank, anaheim, riverside, henderson, reno, spokane, tacoma, rochester, syracuse, albany, wilmington, trenton, grand rapids, toledo, dayton, lexington, knoxville, chattanooga, huntsville, baton rouge, little rock, tallahassee, st. petersburg, fort lauderdale, west palm beach, boca raton, greenville, columbia, greensboro, norfolk, alexandria, bethesda, rockville, toronto, montréal, montreal, vancouver, calgary, edmonton, ottawa, winnipeg, québec, quebec city, hamilton, kitchener, waterloo, halifax, victoria, saskatoon, regina, mississauga, brampton, markham, vaughan, burnaby, surrey, laval, gatineau, kelowna, oakville, burlington, mexico city, ciudad de méxico, guadalajara, monterrey, puebla, tijuana, querétaro, cancún, mérida, são paulo, sao paulo, rio de janeiro, brasília, brasilia, belo horizonte, curitiba, porto alegre, salvador, recife, fortaleza, campinas, florianópolis, manaus, buenos aires, córdoba, rosario, mendoza, santiago, valparaíso, lima, bogotá, bogota, medellín, cali, cartagena, quito, guayaquil, caracas, montevideo, asunción, la paz, san josé, panama city, ciudad de panamá, guatemala city, san salvador, tegucigalpa, managua, santo domingo, san juan, havana, kingston, nassau, london, manchester, birmingham, leeds, glasgow, edinburgh, liverpool, bristol, sheffield, newcastle, nottingham, cardiff, belfast, leicester, coventry, oxford, reading, milton keynes, brighton, southampton, portsmouth, aberdeen, dundee, york, bath, exeter, plymouth, norwich, swindon, slough, watford, guildford, basingstoke, warrington, derby, sunderland, hull, bradford, dublin, cork, galway, limerick, waterford, paris, marseille, lyon, toulouse, nice, nantes, strasbourg, montpellier, bordeaux, lille, rennes, reims, le havre, saint-étienne, toulon, grenoble, dijon, angers, nîmes, villeurbanne, clermont-ferrand, le mans, aix-en-provence, brest, tours, amiens, limoges, annecy, perpignan, metz, besançon, orléans, rouen, mulhouse, caen, nancy, boulogne-billancourt, issy-les-moulineaux, la défense, courbevoie, neuilly-sur-seine, levallois-perret, nanterre, puteaux, saint-denis, versailles, sophia antipolis, cannes, monaco, berlin, hamburg, münchen, munich, köln, cologne, frankfurt, frankfurt am main, stuttgart, düsseldorf, dortmund, essen, leipzig, bremen, dresden, hannover, hanover, nürnberg, nuremberg, duisburg, bochum, wuppertal, bielefeld, bonn, münster, karlsruhe, mannheim, augsburg, wiesbaden, gelsenkirchen, mönchengladbach, braunschweig, chemnitz, kiel, aachen, halle, magdeburg, freiburg, krefeld, lübeck, mainz, erfurt, oberhausen, rostock, kassel, hagen, saarbrücken, potsdam, ludwigshafen, oldenburg, leverkusen, heidelberg, darmstadt, regensburg, ingolstadt, würzburg, ulm, wolfsburg, göttingen, paderborn, heilbronn, erlangen, jena, walldorf, böblingen, sindelfingen, neckarsulm, herzogenaurach, gütersloh, wien, vienna, graz, linz, salzburg, innsbruck, zürich, zurich, genève, geneva, genf, basel, bâle, bern, berne, lausanne, winterthur, luzern, lucerne, st. gallen, lugano, zug, baar, vevey, neuchâtel, amsterdam, rotterdam, den haag, the hague, utrecht, eindhoven, groningen, tilburg, almere, breda, nijmegen, arnhem, haarlem, amersfoort, enschede, apeldoorn, 's-hertogenbosch, leiden, delft, maastricht, zwolle, hilversum, hoofddorp, schiphol, amstelveen, veldhoven, brussels, bruxelles, brussel, antwerp, antwerpen, anvers, ghent, gent, gand, charleroi, liège, luik, bruges, brugge, namur, leuven, louvain, mechelen, mons, hasselt, kortrijk, zaventem, diegem, luxembourg, esch-sur-alzette, stockholm, göteborg, gothenburg, malmö, uppsala, västerås, linköping, lund, helsingborg, örebro, oslo, bergen, trondheim, stavanger, drammen, copenhagen, københavn, aarhus, århus, odense, aalborg, helsinki, helsingfors, espoo, tampere, vantaa, oulu, turku, reykjavík, reykjavik, tallinn, riga, vilnius, kaunas, madrid, barcelona, valencia, sevilla, seville, zaragoza, málaga, murcia, palma, las palmas, bilbao, alicante, valladolid, vigo, gijón, a coruña, granada, vitoria-gasteiz, san sebastián, donostia, pamplona, santander, lisboa, lisbon, porto, oporto, braga, coimbra, faro, funchal, roma, rome, milano, milan, napoli, naples, torino, turin, palermo, genova, genoa, bologna, firenze, florence, bari, catania, venezia, venice, verona, messina, padova, padua, trieste, brescia, parma, modena, reggio emilia, perugia, bergamo, vicenza, monza, athens, athina, thessaloniki, piraeus, nicosia, limassol, valletta, ljubljana, zagreb, split, belgrade, beograd, sarajevo, skopje, tirana, podgorica, warsaw, warszawa, kraków, krakow, cracow, łódź, lodz, wrocław, wroclaw, poznań, poznan, gdańsk, gdansk, szczecin, katowice, lublin, bydgoszcz, prague, praha, brno, ostrava, plzeň, bratislava, košice, budapest, debrecen, szeged, bucharest, bucurești, cluj-napoca, timișoara, iași, sofia, plovdiv, varna, kyiv, kiev, kharkiv, lviv, odesa, odessa, dnipro, minsk, moscow, moskva, saint petersburg, st. petersburg, novosibirsk, yekaterinburg, kazan, chișinău, tbilisi, yerevan, baku, istanbul, ankara, izmir, bursa, antalya, tel aviv, jerusalem, haifa, herzliya, petah tikva, beersheba, dubai, abu dhabi, sharjah, doha, riyadh, jeddah, dammam, kuwait city, manama, muscat, amman, beirut, baghdad, tehran, cairo, giza, casablanca, rabat, marrakech, tangier, tunis, algiers, lagos, abuja, nairobi, mombasa, accra, addis ababa, dar es salaam, kampala, kigali, johannesburg, cape town, durban, pretoria, gqeberha, luanda, maputo, lusaka, harare, dakar, abidjan, kinshasa, douala, yaoundé, windhoek, gaborone, antananarivo, port louis, khartoum, tokyo, 東京, osaka, 大阪, yokohama, 横浜, nagoya, 名古屋, sapporo, 札幌, fukuoka, 福岡, kobe, 神戸, kyoto, 京都, kawasaki, 川崎, saitama, hiroshima, 広島, sendai, 仙台, chiba, 千葉, seoul, 서울, busan, 부산, incheon, 인천, daegu, daejeon, gwangju, suwon, pangyo, seongnam, beijing, 北京, peking, shanghai, 上海, guangzhou, 广州, shenzhen, 深圳, chengdu, 成都, hangzhou, 杭州, wuhan, 武汉, xi'an, 西安, chongqing, 重庆, tianjin, 天津, nanjing, 南京, suzhou, 苏州, qingdao, 青岛, dalian, 大连, xiamen, 厦门, shenyang, changsha, zhengzhou, dongguan, ningbo, hong kong, 香港, macau, macao, 澳门, taipei, 台北, 臺北, taichung, kaohsiung, hsinchu, tainan, singapore, kuala lumpur, penang, george town, johor bahru, cyberjaya, petaling jaya, bangkok, กรุงเทพ, chiang mai, jakarta, surabaya, bandung, denpasar, medan, manila, makati, quezon city, cebu, taguig, bonifacio global city, pasig, hanoi, hà nội, ho chi minh city, hồ chí minh, saigon, da nang, phnom penh, yangon, vientiane, dhaka, chittagong, colombo, kathmandu, karachi, lahore, islamabad, rawalpindi, faisalabad, mumbai, bombay, delhi, new delhi, bengaluru, bangalore, hyderabad, chennai, madras, kolkata, calcutta, pune, ahmedabad, jaipur, surat, lucknow, kanpur, nagpur, indore, thane, bhopal, visakhapatnam, vadodara, coimbatore, kochi, cochin, gurgaon, gurugram, noida, chandigarh, mysore, mysuru, thiruvananthapuram, trivandrum, bhubaneswar, mohali, navi mumbai, sydney, melbourne, brisbane, perth, adelaide, canberra, gold coast, hobart, darwin, wollongong, geelong, auckland, wellington, christchurch, dunedin, tauranga, ulaanbaatar, almaty, astana, tashkent, bishkek """ CITY_KEYS: frozenset[str] = frozenset(key_of(c) for c in _CITIES.replace("\n", ",").split(",") if c.strip()) def is_known_city(text: str | None) -> bool: """Whole string or any 1–3-word n-gram of a short string names a city in the table.""" if not text: return False k = key_of(text) if not k: return False if k in CITY_KEYS: return True if _CJK_RE.search(text): return any(city in k for city in CITY_KEYS if _CJK_RE.search(city)) toks = k.split() if len(toks) > MAX_LOCATION_NAME_WORDS: return False for n in range(1, min(CITY_NGRAM_MAX, len(toks)) + 1): for i in range(len(toks) - n + 1): if " ".join(toks[i:i + n]) in CITY_KEYS: return True return False CITY_DISTRICT_RE = re.compile(r"\s\d{1,2}$") # "Dublin 2", "Paris 8" — a postal district after a known city def has_postal_code(text: str | None) -> bool: if not text: return False return any(not YEAR_RE.match(m.group(0)) for m in POSTAL_RE.finditer(text)) def looks_like_city_value(text: str | None) -> bool: """A plausible `city` cell: short, capitalised words, no digits, no nav/cookie vocabulary, no sentence.""" if not text: return False t = normalize_whitespace(text) if not (2 <= len(t) <= MAX_CITY_CHARS) or word_count(t) > MAX_CITY_WORDS: return False if any(ch.isdigit() for ch in t) and not (CITY_DISTRICT_RE.search(t) and is_known_city(CITY_DISTRICT_RE.sub("", t))): return False # "Level 12" no · "Dublin 2" (postal district) yes if NAV_COOKIE_RE.search(key_of(t)) or is_prose(t): return False return t[0].isupper() or bool(_CJK_RE.search(t[0])) def _looks_like_venue(text: str) -> bool: return word_count(text) >= VENUE_MIN_WORDS or bool(VENUE_RE.search(text)) def location_signal(*, name: str, city: str | None, region: str | None, country: str | None, kind: str | None, address: str | None) -> str | None: """The evidence that makes this a real place, or None: country / known city / postal code / street address / explicit kind label.""" if country: return "country" if is_known_city(city) or is_known_city(region) or is_known_city(name): return "known city" if has_postal_code(address) or (has_postal_code(name) and "," in name): return "postal code" if STREET_RE.search(address or "") or STREET_RE.search(name): return "street address" if (kind and kind not in ("office", "other")) or LOCATION_KIND_WORD_RE.search(name): return "kind label" return None def normalize_location(loc: ExtractedLocation) -> ExtractedLocation | None: """Validated copy of `loc` (city cell cleaned, country-as-name repaired) or None when it is not a place.""" name = strip_trailing_marks(loc.name or "") if not name or len(name) > MAX_LOCATION_NAME_CHARS or word_count(name) > MAX_LOCATION_NAME_WORDS: return None if NAV_COOKIE_RE.search(key_of(name)) or is_prose(name): return None city = loc.city if looks_like_city_value(loc.city) else None region = loc.region country = loc.country name_country = country_code(name) if name_country and city and key_of(city) == key_of(name) and not is_known_city(name): city = None # "Oman | Oman": a country is not its own city (Singapore is) if name_country and loc.city and (city is None or _looks_like_venue(loc.city)) and not NAV_COOKIE_RE.search(key_of(loc.city)) and not is_prose(loc.city) \ and len(loc.city) <= MAX_LOCATION_NAME_CHARS and key_of(loc.city) != key_of(name): name, city, country = normalize_whitespace(loc.city), None, country or name_country # "Oman | Hormuz Grand Hotel" → venue named, country kept if region and is_known_city(region) and looks_like_city_value(region) and (city is None or not is_known_city(city)): city, region = region, None # "3089 JH Rotterdam" parsed as a region if location_signal(name=name, city=city, region=region, country=country, kind=loc.kind, address=loc.address_text) is None: return None return ExtractedLocation(name=name, kind=loc.kind or "office", city=city, region=region, country=country, address_text=loc.address_text) def location_verdict(loc: ExtractedLocation) -> Verdict: name = strip_trailing_marks(loc.name or "") if not name: return Verdict.reject("empty name") if len(name) > MAX_LOCATION_NAME_CHARS or word_count(name) > MAX_LOCATION_NAME_WORDS: return Verdict.reject("name too long") if NAV_COOKIE_RE.search(key_of(name)): return Verdict.reject("cookie/consent/navigation vocabulary") if is_prose(name): return Verdict.reject("name is a sentence") fixed = normalize_location(loc) if fixed is None: return Verdict.reject("no place evidence (country, city, postal code, street, kind)") return Verdict.accept() # ------------------------------------------------------------------------------------------------------------ pricing plans PLAN_EYEBROW_RE = _phrase_re(["most popular", "popular", "best value", "recommended", "new", "beta", "coming soon", "limited offer", "best seller", "le plus populaire", "beliebt", "am beliebtesten", "meest gekozen", "más popular", "più popolare", "mais popular", "人気", # generic section headings that are not tiers "plans", "plan", "pricing", "prices", "price", "plans & pricing", "plans and pricing", "our plans", "compare plans", "choose your plan", "tarifs", "nos tarifs", "abonnements", "preise", "unsere preise", "tarife", "prijzen", "abonnementen", "precios", "planes", "prezzi", "piani", "preços", "planos", "料金", "料金プラン", "プラン"]) PRICE_TOKEN_RE = re.compile( r"(?:(?:US\$|CA\$|C\$|A\$|NZ\$|HK\$|S\$|MX\$|R\$|USD|EUR|GBP|CAD|AUD|CHF|JPY|INR|SEK|NOK|DKK|PLN|BRL|CNY|RMB|SGD|[$€£¥₹₩₺])\s?" r"\d{1,3}(?:[,.\s]\d{3})*(?:[.,]\d{1,2})?|\d{1,3}(?:[,.\s]\d{3})*(?:[.,]\d{1,2})?\s?(?:€|£|USD|EUR|GBP|CHF|kr|zł|元|\$))", re.IGNORECASE) PRICE_LEAD_RE = re.compile(r"(?:from|starting at|starts at|as low as|à partir de|ab|desde|a partire da|vanaf|only|just)\s*$", re.IGNORECASE) PRICE_UNIT = (r"(?:month|mo|year|yr|annum|week|wk|day|hour|hr|user|seat|member|license|licence|agent|editor|contact|1,?000|1k|GB|TB|request|" r"transaction|call|minute|mois|an|année|monat|jahr|nutzer|benutzer|utilisateur|usuario|utente|mese|anno|mes|año|maand|jaar|gebruiker|" r"dag|tag|giorno|día|dia|jour)") PRICE_TAIL_RE = re.compile(rf"^\s*(?:(?:/|per|a|an|each|every|par|pro|pour|por|al|je)\s*{PRICE_UNIT}\b(?:\s*(?:/|per|par|pro|pour|por)\s*{PRICE_UNIT}\b)?|" rf"(?:monthly|annually|yearly|mensuel|annuel|monatlich|jährlich|one[- ]time|lifetime|forever))?" r"(?:,?\s*billed (?:monthly|annually|yearly))?", re.IGNORECASE) CONTACT_PHRASE_RE = re.compile(r"\b(contact (?:us|sales)|talk to (?:us|sales|an expert)|custom pricing|custom quote|get a quote|request (?:a )?(?:quote|demo)|" r"let'?s talk|on request|upon request|sur devis|nous contacter|contactez-nous|auf anfrage|individuell|tailored|bespoke|call us)\b", re.IGNORECASE) FREE_TOKEN_RE = re.compile(r"^(free|gratuit|kostenlos|gratis|\$\s?0(?:\.00)?|0\s?€|€\s?0)\b", re.IGNORECASE) def price_text_from(text: str) -> str | None: """The price phrase stated in `text` — currency + amount + period/unit, or the contact-sales / free wording — never a cut sentence.""" t = normalize_whitespace(text) m = PRICE_TOKEN_RE.search(t) if m: lead = PRICE_LEAD_RE.search(t[:m.start()]) start = lead.start() if lead else m.start() tail = PRICE_TAIL_RE.match(t[m.end():]) end = m.end() + (tail.end() if tail else 0) return t[start:end].strip(" ,")[:MAX_PRICE_TEXT_CHARS] c = CONTACT_PHRASE_RE.search(t) if c: phrase = c.group(1) return phrase[0].upper() + phrase[1:] f = FREE_TOKEN_RE.match(t) if f: return f.group(1) return None def plan_name_ok(name: str | None) -> bool: if not name: return False n = normalize_whitespace(name) if not (1 <= len(n) <= MAX_PLAN_NAME_CHARS) or word_count(n) > MAX_PLAN_NAME_WORDS: return False if has_terminal_punct(n) or "|" in n or "?" in n or PLAN_EYEBROW_RE.match(n) or re.search(r"\d+\s?%", n): return False if MARKETING_VERB_RE.match(n) and word_count(n) > 1: return False return not PRICE_TOKEN_RE.search(n) def plan_verdict(plan: ExtractedPlan) -> Verdict: if not plan_name_ok(plan.plan_name): return Verdict.reject("plan name is a heading / marketing phrase") if plan.price is None and not plan.contact_sales and not (plan.price_text and FREE_TOKEN_RE.match(plan.price_text)): return Verdict.reject("no price, no contact-sales, no free tier") if plan.price_text and price_text_from(plan.price_text) is None: return Verdict.reject("price text does not state a price") return Verdict.accept() # ------------------------------------------------------------------------------------------------------------ products PRODUCT_NAV_RE = _phrase_re([ "overview", "products", "product", "all products", "our products", "solutions", "solution", "all solutions", "services", "our services", "features", "pricing", "resources", "support", "docs", "documentation", "blog", "contact", "contact us", "home", "learn more", "read more", "view all", "see all", "explore", "more", "get started", "sign up", "log in", "download", "compare", "industries", "platform", "use cases", "customers", "partners", "produits", "nos produits", "tous les produits", "en savoir plus", "produkte", "alle produkte", "unsere produkte", "lösungen", "mehr erfahren", "producten", "alle producten", "oplossingen", "meer info", "productos", "todos los productos", "soluciones", "ver más", "prodotti", "tutti i prodotti", "soluzioni", "scopri di più", "produtos", "todos os produtos", "soluções", "saiba mais", "製品", "製品一覧", "ソリューション", "サービス", "詳細", "もっと見る", ]) def product_verdict(name: str) -> Verdict: n = strip_trailing_marks(name) if not (2 <= len(n) <= MAX_PRODUCT_NAME_CHARS): return Verdict.reject("name length") if word_count(n) > MAX_PRODUCT_NAME_WORDS: return Verdict.reject("name too long") if PRODUCT_NAV_RE.match(n.rstrip("®™ ")): return Verdict.reject("navigation label") if has_terminal_punct(n) and word_count(n) >= 3: return Verdict.reject("name is a sentence") if is_slogan(n) or is_prose(n): return Verdict.reject("marketing slogan") return Verdict.accept() # ------------------------------------------------------------------------------------------------------------ news NEWS_NOISE_RE = _phrase_re([ "read more", "read more news", "learn more", "more", "more news", "continue reading", "view all", "see all", "all news", "all posts", "all articles", "next", "previous", "older", "newer", "older posts", "newer posts", "load more", "show more", "page", "news", "press", "press releases", "press release", "blog", "blog posts", "articles", "insights", "events", "media", "in the news", "case studies", "webinars", "whitepapers", "podcasts", "videos", "newsroom", "latest news", "category", "categories", "archive", "archives", "tags", "share", "rss", "subscribe", "newsletter", "actualités", "toutes les actualités", "communiqués de presse", "lire la suite", "en savoir plus", "alle news", "pressemitteilungen", "aktuelles", "weiterlesen", "mehr erfahren", "nieuws", "alle berichten", "persberichten", "lees meer", "noticias", "todas las noticias", "notas de prensa", "leer más", "notizie", "comunicati stampa", "leggi tutto", "notícias", "ler mais", "ニュース", "お知らせ", "プレスリリース", "一覧", "もっと見る", ]) PAGINATION_RE = re.compile(r"^(?:page\s*)?\d{1,4}$|^(?:[«»‹›<>]|\.\.\.|…)+$", re.IGNORECASE) def news_verdict(title: str, *, url: str | None = None, published_at: datetime | None = None) -> Verdict: t = strip_trailing_marks(title) if not t or PAGINATION_RE.match(t): return Verdict.reject("pagination") if NEWS_NOISE_RE.match(t): return Verdict.reject("navigation / category label") confirmed = published_at is not None or date_from_url(url) is not None long_enough = word_count(t) >= NEWS_MIN_WORDS or len(t) >= NEWS_MIN_CHARS or (bool(_CJK_RE.search(t)) and len(t) >= NEWS_MIN_CJK_CHARS) if not confirmed and not long_enough: return Verdict.reject("too short without a date") return Verdict.accept() # ------------------------------------------------------------------------------------------------------------ whole extraction HTML_JOB_CONNECTORS = frozenset({"generic-html-v1"}) def apply_precision(ex: Extraction, *, html_jobs: bool) -> dict[str, int]: """Filter/normalise every typed list in place; returns how many items each list lost. Job rules only for HTML-scraped listings.""" dropped: dict[str, int] = {} if html_jobs: kept_jobs = [j for j in (refine_job(j) for j in ex.jobs) if j is not None] dropped["jobs"] = len(ex.jobs) - len(kept_jobs) ex.jobs = kept_jobs people: list[ExtractedPerson] = [] for p in ex.people: fixed = normalize_person(p.name, p.title) if fixed is None: continue p.name, p.title = fixed p.role_category, p.is_executive = role_category(p.title) people.append(p) dropped["people"] = len(ex.people) - len(people) ex.people = people products: list[ExtractedProduct] = [p for p in ex.products if product_verdict(p.name).ok] dropped["products"] = len(ex.products) - len(products) ex.products = products plans: list[ExtractedPlan] = [p for p in ex.plans if plan_verdict(p).ok] dropped["plans"] = len(ex.plans) - len(plans) ex.plans = plans locations: list[ExtractedLocation] = [loc for loc in (normalize_location(loc) for loc in ex.locations) if loc is not None] dropped["locations"] = len(ex.locations) - len(locations) ex.locations = locations news: list[ExtractedNewsItem] = [n for n in ex.news if news_verdict(n.title, url=n.url, published_at=n.published_at).ok] dropped["news"] = len(ex.news) - len(news) ex.news = news return {k: v for k, v in dropped.items() if v} def describe(verdict: Verdict) -> dict[str, Any]: return {"ok": verdict.ok, "reason": verdict.reason} __all__ = [ "CHIEF_RE", "CITY_KEYS", "EXEC_CATEGORIES", "HTML_JOB_CONNECTORS", "NAV_COOKIE_RE", "PRECISION_VERSION", "ROLE_RULES", "STREET_RE", "Verdict", "apply_precision", "clean_job_title", "clean_person_title", "describe", "has_postal_code", "is_known_city", "is_prose", "is_slogan", "job_url_is_joblike", "job_verdict", "key_of", "location_signal", "location_verdict", "looks_like_city_value", "looks_like_person_name", "looks_like_role_title", "news_verdict", "normalize_location", "normalize_person", "person_verdict", "plan_name_ok", "plan_verdict", "price_text_from", "product_verdict", "refine_job", "role_category", "states_job_location", "strip_trailing_marks", "word_count", ]