# ----------------------------------------------------------------------------- # Forma-Ka — Agrégateur de formations (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/ulaval_distance.py : connecteur Formation à distance — Université # Laval (distance.ulaval.ca). ~1 700 cours crédités offerts à distance, # hybrides ou comodaux (tous cycles, toutes facultés). # Site rendu serveur, très propre : # - liste : /etudes/cours — un lien .list__link par cours avec sigle + titre # (.oe-title), cycle (.oe-cycle) et formule d'enseignement (aria-label) # - fiche : Description (h2 + paragraphes), Responsables (faculté, # département), bloc .credits-and-cycles, section Horaire (.sessions : # sessions offertes, formule, section-groupe, NRC, enseignants) et # programmes auxquels le cours contribue. Fiches en cache, rafraîchies # chaque semaine (clé ISO « AAAA-WSS »). # ----------------------------------------------------------------------------- from __future__ import annotations import datetime import re from bs4 import BeautifulSoup from ..schema import Formation, clean_text from .base import BaseConnector BASE = "https://www.distance.ulaval.ca" LIST_URL = f"{BASE}/etudes/cours" _CODE_TITLE_RE = re.compile(r"^([A-Z]{2,4}-\d{4}[A-Z]?)\s+(.*)$") _FORMULE_RE = re.compile(r"formules? d'enseignement\s*:\s*(.+)", re.I) # formule d'enseignement ULaval -> mode canonique Forma-Ka _MODE_MAP = [ (re.compile(r"hybride|comodal", re.I), "hybride"), (re.compile(r"distance|en ligne", re.I), "en ligne"), (re.compile(r"pr[ée]sentiel|en classe", re.I), "présentiel"), ] def _mode_from(formule: str) -> str: for rx, canon in _MODE_MAP: if rx.search(formule or ""): return canon return "" class UlavalDistanceConnector(BaseConnector): source_id = "ulaval_distance" request_delay = 0.5 def fetch(self) -> list[Formation]: html = self.fetch_html(LIST_URL) soup = BeautifulSoup(html, "html.parser") # 1) Liste complète (une seule page rendue serveur, ~1 700 cours) week = datetime.date.today().strftime("%G-W%V") out: list[Formation] = [] seen: set[str] = set() for a in soup.select("a.list__link[href]"): title_el = a.find(class_="oe-title") if title_el is None: continue raw_title = clean_text(title_el.get_text(" ")) m = _CODE_TITLE_RE.match(raw_title) code = m.group(1) if m else "" title = m.group(2) if m else raw_title href = a["href"] url = href if href.startswith("http") else BASE + href external_id = code or url.rstrip("/").rsplit("/", 1)[-1] if external_id in seen: continue seen.add(external_id) f = Formation( source=self.source_id, external_id=external_id, url=url, title=title, training_type="Cours universitaire", language="fr", code=code, ) cycle_el = a.find(class_="oe-cycle") if cycle_el: f.details["cycle"] = clean_text(cycle_el.get_text(" ")) formula_el = a.find(class_="oe-teaching-formulas") fm = _FORMULE_RE.search(formula_el.get("aria-label", "")) \ if formula_el else None if fm: formule = clean_text(fm.group(1)) f.details["formule_enseignement"] = formule f.mode = _mode_from(formule) # 2) Fiche détaillée — cache hebdomadaire key = f"{week}:{raw_title}" payload = self.detail(external_id, key, lambda u=url: self._fetch_detail(u)) for k, v in (payload or {}).items(): if k == "details": f.details = {**f.details, **v} elif hasattr(f, k) and v not in (None, "", []): setattr(f, k, v) out.append(f) return out # -- fiche ---------------------------------------------------------------- def _fetch_detail(self, url: str) -> dict: html = self.fetch_html(url) soup = BeautifulSoup(html, "html.parser") payload: dict = {} details: dict = {} # description : paragraphes qui suivent le h2 « Description » h2 = soup.find("h2", string=re.compile(r"^\s*Description\s*$", re.I)) if h2: paras = [] for sib in h2.find_next_siblings(): if sib.name in ("h1", "h2", "h3"): break if sib.name == "p": t = clean_text(sib.get_text(" ")) # on écarte le lien « Consulter la description officielle… » if t and not t.lower().startswith("consulter la description"): paras.append(t) if paras: payload["description"] = "\n\n".join(paras) # crédits et cycle (bloc latéral) cred = soup.select_one(".credits-and-cycles .credits") if cred: n = clean_text(cred.find("p").get_text(" ")) if cred.find("p") else "" if n: if n.isdigit(): payload["credits"] = f"{n} crédit" + ("s" if int(n) > 1 else "") else: payload["credits"] = n cyc = soup.select_one(".credits-and-cycles .cycles") if cyc: details["cycle"] = clean_text(cyc.get_text(" ")) # responsables : faculté (-> catégorie) et département resp = soup.find("h2", string=re.compile(r"^\s*Responsables?\s*$", re.I)) if resp: block = resp.find_parent(["section", "div"]) txt = clean_text(block.get_text(" ")) if block else "" m = re.search(r"Faculté\s*:\s*([^:]+?)(?:\s+Courriel|\s+Département|$)", txt) if m: payload["category"] = clean_text(m.group(1)) m = re.search(r"Département\s*:\s*([^:]+?)(?:\s+Courriel|$)", txt) if m: details["departement"] = clean_text(m.group(1)) # horaire : sessions offertes, formules, sections-groupes, enseignants sched = soup.select_one(".schedule .sessions") if sched: sessions: list[dict] = [] instructors: list[str] = [] current: dict | None = None for el in sched.find_all(["h3", "div"], recursive=False): if el.name == "h3": current = {"session": clean_text(el.get_text(" ")), "sections": []} sessions.append(current) elif current is not None and "session" in (el.get("class") or []): sec: dict = {} formule = el.select_one(".teaching-formulas .text") if formule: sec["formule"] = clean_text(formule.get_text(" ")) sigle = el.select_one(".sigle-tag") if sigle: sec["groupe"] = clean_text(sigle.get_text(" ")) nrc = el.select_one(".c-tag") if nrc: sec["nrc"] = clean_text(nrc.get_text(" ")) ens = el.find("p") if ens: names = clean_text(ens.get_text(" ")) names = re.sub(r"^Enseignant\(e\)s?\s*:\s*", "", names) if names: sec["enseignants"] = names instructors.append(names) current["sections"].append(sec) if sessions: details["sessions_offertes"] = sessions payload["schedule_label"] = ", ".join(s["session"] for s in sessions) if instructors: payload["instructor"] = ", ".join(dict.fromkeys(instructors)) # programmes auxquels le cours contribue contrib = soup.find("h2", string=re.compile(r"contributoire", re.I)) if contrib: block = contrib.find_parent(["section", "div"]) progs = [clean_text(a.get_text(" ")) for a in block.find_all("a")] \ if block else [] progs = [p for p in progs if p] if progs: details["programmes_lies"] = progs if details: payload["details"] = details return payload