spb/forma-ka Public
Python 65.1%
TypeScript 17.9%
CSS 16.4%
HTML 0.5%
1# -----------------------------------------------------------------------------2# Forma-Ka — Agrégateur de formations (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/afi.py : connecteur AFI par Edgenda (afiexpertise.com)5# Firme de formation TI, bureautique et leadership (Québec, Montréal,6# classe virtuelle) — ~470 formations au calendrier public.7# Site Gatsby : chaque fiche possède un JSON statique complet à8# /page-data/fr/formation/<slug>/page-data.json — nom, description HTML,9# plan de cours en tableau (public concerné, certification, prérequis,10# objectifs, méthode pédagogique, contenu par modules), sessions datées11# avec villes et horaires, prix PAR JOUR (le prix affiché sur le site est12# RegularPrice × nb de jours, « Sur demande » quand aucune session),13# thématique, éditeur, formateurs. Liste découverte via sitemap-0.xml.14# Fiches en cache, rafraîchies chaque semaine (clé « AAAA-WSS »).15# -----------------------------------------------------------------------------16from __future__ import annotations1718import datetime19import re2021from bs4 import BeautifulSoup2223from ..schema import Formation, clean_text24from .base import BaseConnector2526BASE = "https://www.afiexpertise.com"27SITEMAP_URL = f"{BASE}/sitemap-0.xml"2829_COURSE_LOC_RE = re.compile(30 r"<loc>https://www\.afiexpertise\.com(/fr/formation/([^<]+))</loc>")313233def _cell_text(td) -> str:34 return clean_text(td.get_text(" ")) if td is not None else ""353637def _cell_items(td) -> list[str]:38 items = [clean_text(li.get_text(" ")) for li in td.find_all("li")]39 return [i for i in dict.fromkeys(items) if i]404142class AfiConnector(BaseConnector):43 source_id = "afi"44 request_delay = 0.545 limit: int | None = None # borne optionnelle (tests/débogage)4647 def fetch(self) -> list[Formation]:48 # 1) Liste des fiches françaises depuis le sitemap49 xml = self.get(SITEMAP_URL).text50 slugs = list(dict.fromkeys(m.group(2).strip("/")51 for m in _COURSE_LOC_RE.finditer(xml)))52 if self.limit:53 slugs = slugs[: self.limit]5455 # 2) JSON statique Gatsby par formation — cache hebdomadaire56 week = datetime.date.today().strftime("%G-W%V")57 out: list[Formation] = []58 for slug in slugs:59 payload = self.detail(slug, week,60 lambda s=slug: self._fetch_detail(s))61 if not payload:62 continue63 f = Formation(64 source=self.source_id,65 external_id=slug,66 url=f"{BASE}/fr/formation/{slug}",67 training_type="Formation continue",68 language="fr",69 )70 for k, v in payload.items():71 if hasattr(f, k) and v not in (None, "", []):72 setattr(f, k, v)73 out.append(f)74 return out7576 # -- fiche ----------------------------------------------------------------77 def _fetch_detail(self, slug: str) -> dict:78 url = f"{BASE}/page-data/fr/formation/{slug}/page-data.json"79 try:80 data = self.get(url).json()81 except Exception:82 return {}83 result = (data.get("result") or {})84 course = ((result.get("data") or {}).get("course") or {})85 if not course:86 return {}87 payload: dict = {}88 details: dict = {}8990 payload["title"] = clean_text(91 (course.get("Name") or {}).get("fr") or "")92 desc = BeautifulSoup(93 (course.get("Description") or {}).get("fr") or "", "html.parser")94 paragraphs = [clean_text(p.get_text(" "))95 for p in desc.find_all(["p", "li"])]96 payload["description"] = "\n\n".join(97 dict.fromkeys(p for p in paragraphs if p)) or clean_text(98 desc.get_text(" "))99 if not payload["description"]:100 payload["description"] = clean_text(101 (course.get("MetaDescription") or {}).get("fr") or "")102103 # plan de cours : tableau à deux colonnes (libellé -> contenu)104 plan = BeautifulSoup((course.get("Plan") or {}).get("fr") or "",105 "html.parser")106 program: list[str] = []107 for tr in plan.find_all("tr"):108 tds = tr.find_all("td")109 if len(tds) < 2:110 continue111 label = clean_text(tds[0].get_text(" ")).lower()112 cell = tds[1]113 if "public" in label:114 payload["audience"] = _cell_text(cell)115 elif "prérequis" in label or "prealable" in label:116 payload["prerequisites"] = _cell_text(cell)117 elif "objectif" in label:118 payload["objectives"] = _cell_items(cell) or \119 [t for t in (_cell_text(cell),) if t]120 elif "certification" in label:121 details["certification"] = _cell_text(cell)122 elif "méthode" in label:123 details["methode_pedagogique"] = _cell_text(cell)124 elif "contenu" in label:125 # titres de modules (<p>) quand ils structurent le contenu,126 # sinon puces (<li>)127 ptexts = [clean_text(p.get_text(" "))128 for p in cell.find_all("p")]129 ptexts = [p for p in dict.fromkeys(ptexts) if p]130 litexts = _cell_items(cell)131 program = ptexts if len(ptexts) >= 2 else litexts or ptexts132 if program:133 payload["program"] = program134135 # durée en jours + prix affiché = prix/jour × jours (min. 1 jour)136 days = course.get("DurationInDays")137 if days:138 payload["duration"] = (f"{days:g} jour"139 if days <= 1 else f"{days:g} jours")140 payload["duration_hours"] = days * 7.0141 sessions_raw = course.get("Sessions") or []142 price = course.get("RegularPrice")143 if price is not None and sessions_raw:144 total = price * days if days and days >= 1 else price145 payload["price"] = float(total)146 payload["price_label"] = f"{total:g} $ + tx"147 pref = course.get("PreferentialPrice")148 if pref is not None:149 details["prix_preferentiel"] = (pref * days150 if days and days >= 1 else pref)151 elif not sessions_raw:152 payload["price_label"] = "Sur demande"153154 # sessions datées (première journée de chaque cohorte) + villes155 sessions, cities, langs = [], [], []156 for s in sessions_raw:157 dates = s.get("Dates") or []158 if dates:159 start = str(dates[0])[:10]160 if re.match(r"20\d{2}-\d{2}-\d{2}", start):161 sessions.append(start)162 if s.get("City"):163 cities.append(s["City"])164 if s.get("Language"):165 langs.append(s["Language"])166 sessions = sorted(set(sessions))167 if sessions:168 payload["sessions"] = sessions169 payload["start_date"] = sessions[0]170 cities = list(dict.fromkeys(cities))171 modes = ["en ligne" if c.lower() == "classe virtuelle" else "présentiel"172 for c in cities]173 modes = list(dict.fromkeys(modes))174 if not modes and course.get("IsVirtual"):175 modes = ["en ligne"]176 if modes:177 payload["mode"] = modes[0] if len(modes) == 1 else "hybride"178 details["modes_offerts"] = modes179 real_cities = [c for c in cities if c.lower() != "classe virtuelle"]180 if real_cities:181 payload["city"] = real_cities[0]182 if langs:183 payload["language"] = "/".join(dict.fromkeys(langs))184185 # contexte de page : thématique (catégorie) et éditeur (Microsoft…)186 ctx = result.get("pageContext") or {}187 theme = ((ctx.get("theme") or {}).get("name") or {}).get("fr", "")188 if theme:189 payload["category"] = theme190 editor = ((ctx.get("editor") or {}).get("name") or {}).get("fr", "")191 if editor:192 payload["tags"] = [editor]193194 teachers = [clean_text(f"{t.get('FirstName', '')} {t.get('LastName', '')}")195 for t in course.get("Teachers") or []]196 teachers = [t for t in dict.fromkeys(teachers) if t]197 if teachers:198 payload["instructor"] = ", ".join(teachers)199 if course.get("IsNew"):200 details["nouveau"] = True201 if details:202 payload["details"] = details203 return payload204