spb/forma-ka Public
Python 65.1%
TypeScript 17.9%
CSS 16.4%
HTML 0.5%
1# -----------------------------------------------------------------------------2# Forma-Ka — Agrégateur de formations (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/ulaval_distance.py : connecteur Formation à distance — Université5# Laval (distance.ulaval.ca). ~1 700 cours crédités offerts à distance,6# hybrides ou comodaux (tous cycles, toutes facultés).7# Site rendu serveur, très propre :8# - liste : /etudes/cours — un lien .list__link par cours avec sigle + titre9# (.oe-title), cycle (.oe-cycle) et formule d'enseignement (aria-label)10# - fiche : Description (h2 + paragraphes), Responsables (faculté,11# département), bloc .credits-and-cycles, section Horaire (.sessions :12# sessions offertes, formule, section-groupe, NRC, enseignants) et13# programmes auxquels le cours contribue. Fiches en cache, rafraîchies14# chaque semaine (clé ISO « AAAA-WSS »).15# -----------------------------------------------------------------------------16from __future__ import annotations1718import datetime19import re2021from bs4 import BeautifulSoup2223from ..schema import Formation, clean_text2425from .base import BaseConnector2627BASE = "https://www.distance.ulaval.ca"28LIST_URL = f"{BASE}/etudes/cours"2930_CODE_TITLE_RE = re.compile(r"^([A-Z]{2,4}-\d{4}[A-Z]?)\s+(.*)$")31_FORMULE_RE = re.compile(r"formules? d'enseignement\s*:\s*(.+)", re.I)3233# formule d'enseignement ULaval -> mode canonique Forma-Ka34_MODE_MAP = [35 (re.compile(r"hybride|comodal", re.I), "hybride"),36 (re.compile(r"distance|en ligne", re.I), "en ligne"),37 (re.compile(r"pr[ée]sentiel|en classe", re.I), "présentiel"),38]394041def _mode_from(formule: str) -> str:42 for rx, canon in _MODE_MAP:43 if rx.search(formule or ""):44 return canon45 return ""464748class UlavalDistanceConnector(BaseConnector):49 source_id = "ulaval_distance"50 request_delay = 0.55152 def fetch(self) -> list[Formation]:53 html = self.fetch_html(LIST_URL)54 soup = BeautifulSoup(html, "html.parser")5556 # 1) Liste complète (une seule page rendue serveur, ~1 700 cours)57 week = datetime.date.today().strftime("%G-W%V")58 out: list[Formation] = []59 seen: set[str] = set()60 for a in soup.select("a.list__link[href]"):61 title_el = a.find(class_="oe-title")62 if title_el is None:63 continue64 raw_title = clean_text(title_el.get_text(" "))65 m = _CODE_TITLE_RE.match(raw_title)66 code = m.group(1) if m else ""67 title = m.group(2) if m else raw_title68 href = a["href"]69 url = href if href.startswith("http") else BASE + href70 external_id = code or url.rstrip("/").rsplit("/", 1)[-1]71 if external_id in seen:72 continue73 seen.add(external_id)7475 f = Formation(76 source=self.source_id,77 external_id=external_id,78 url=url,79 title=title,80 training_type="Cours universitaire",81 language="fr",82 code=code,83 )84 cycle_el = a.find(class_="oe-cycle")85 if cycle_el:86 f.details["cycle"] = clean_text(cycle_el.get_text(" "))87 formula_el = a.find(class_="oe-teaching-formulas")88 fm = _FORMULE_RE.search(formula_el.get("aria-label", "")) \89 if formula_el else None90 if fm:91 formule = clean_text(fm.group(1))92 f.details["formule_enseignement"] = formule93 f.mode = _mode_from(formule)9495 # 2) Fiche détaillée — cache hebdomadaire96 key = f"{week}:{raw_title}"97 payload = self.detail(external_id, key,98 lambda u=url: self._fetch_detail(u))99 for k, v in (payload or {}).items():100 if k == "details":101 f.details = {**f.details, **v}102 elif hasattr(f, k) and v not in (None, "", []):103 setattr(f, k, v)104 out.append(f)105 return out106107 # -- fiche ----------------------------------------------------------------108 def _fetch_detail(self, url: str) -> dict:109 html = self.fetch_html(url)110 soup = BeautifulSoup(html, "html.parser")111 payload: dict = {}112 details: dict = {}113114 # description : paragraphes qui suivent le h2 « Description »115 h2 = soup.find("h2", string=re.compile(r"^\s*Description\s*$", re.I))116 if h2:117 paras = []118 for sib in h2.find_next_siblings():119 if sib.name in ("h1", "h2", "h3"):120 break121 if sib.name == "p":122 t = clean_text(sib.get_text(" "))123 # on écarte le lien « Consulter la description officielle… »124 if t and not t.lower().startswith("consulter la description"):125 paras.append(t)126 if paras:127 payload["description"] = "\n\n".join(paras)128129 # crédits et cycle (bloc latéral)130 cred = soup.select_one(".credits-and-cycles .credits")131 if cred:132 n = clean_text(cred.find("p").get_text(" ")) if cred.find("p") else ""133 if n:134 if n.isdigit():135 payload["credits"] = f"{n} crédit" + ("s" if int(n) > 1 else "")136 else:137 payload["credits"] = n138 cyc = soup.select_one(".credits-and-cycles .cycles")139 if cyc:140 details["cycle"] = clean_text(cyc.get_text(" "))141142 # responsables : faculté (-> catégorie) et département143 resp = soup.find("h2", string=re.compile(r"^\s*Responsables?\s*$", re.I))144 if resp:145 block = resp.find_parent(["section", "div"])146 txt = clean_text(block.get_text(" ")) if block else ""147 m = re.search(r"Faculté\s*:\s*([^:]+?)(?:\s+Courriel|\s+Département|$)", txt)148 if m:149 payload["category"] = clean_text(m.group(1))150 m = re.search(r"Département\s*:\s*([^:]+?)(?:\s+Courriel|$)", txt)151 if m:152 details["departement"] = clean_text(m.group(1))153154 # horaire : sessions offertes, formules, sections-groupes, enseignants155 sched = soup.select_one(".schedule .sessions")156 if sched:157 sessions: list[dict] = []158 instructors: list[str] = []159 current: dict | None = None160 for el in sched.find_all(["h3", "div"], recursive=False):161 if el.name == "h3":162 current = {"session": clean_text(el.get_text(" ")),163 "sections": []}164 sessions.append(current)165 elif current is not None and "session" in (el.get("class") or []):166 sec: dict = {}167 formule = el.select_one(".teaching-formulas .text")168 if formule:169 sec["formule"] = clean_text(formule.get_text(" "))170 sigle = el.select_one(".sigle-tag")171 if sigle:172 sec["groupe"] = clean_text(sigle.get_text(" "))173 nrc = el.select_one(".c-tag")174 if nrc:175 sec["nrc"] = clean_text(nrc.get_text(" "))176 ens = el.find("p")177 if ens:178 names = clean_text(ens.get_text(" "))179 names = re.sub(r"^Enseignant\(e\)s?\s*:\s*", "", names)180 if names:181 sec["enseignants"] = names182 instructors.append(names)183 current["sections"].append(sec)184 if sessions:185 details["sessions_offertes"] = sessions186 payload["schedule_label"] = ", ".join(s["session"] for s in sessions)187 if instructors:188 payload["instructor"] = ", ".join(dict.fromkeys(instructors))189190 # programmes auxquels le cours contribue191 contrib = soup.find("h2", string=re.compile(r"contributoire", re.I))192 if contrib:193 block = contrib.find_parent(["section", "div"])194 progs = [clean_text(a.get_text(" ")) for a in block.find_all("a")] \195 if block else []196 progs = [p for p in progs if p]197 if progs:198 details["programmes_lies"] = progs199200 if details:201 payload["details"] = details202 return payload203