Documents partis : découverte élargie (QS sous-domaines, PCQ blog), rendu Firecrawl pour les sites JS, archives pré-2022, retry JSON LLM

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 18 days ago (Sep 6, 2026) parent f3bb070

2 changed files +87 −31

modified engine/qc26/connectors/parties/documents.py +78 −29

		`@@ -69,16 +69,19 @@ SITES: dict[str, PartySite] = {`
69	69	re.compile(r"(\.(jpg\|jpeg\|png\|gif\|svg\|webp\|mp4)\|/en/\|/page/\d\|wp-json\|/feed\|#)", re.I),
70	70	("https://pq.org/", "https://pq.org/nouvelles/", "https://pq.org/programme/", "https://pq.org/engagements/")),
71	71	"qs": PartySite("qs", "https://quebecsolidaire.net/",
72		− ("programme", "plateforme 2026", "engagements", "cadre financier", "communiqué"),
73		− re.compile(r"quebecsolidaire\.net/(?!.*(don\|boutique\|adhesion\|nous-joindre\|recherche\|evenement))", re.I),
74		− re.compile(r"(\.(jpg\|jpeg\|png\|gif\|svg\|webp\|mp4)\|/en/\|/page/\d\|#)", re.I),
75		− ("https://quebecsolidaire.net/", "https://quebecsolidaire.net/nouvelles", "https://quebecsolidaire.net/programme", "https://quebecsolidaire.net/engagements")),
	72	+ ("programme", "plateforme 2026", "engagements", "priorités", "plan", "communiqué"),
	73	+ re.compile(r"(quebecsolidaire\.net/(nouvelle/\|programme\|plateforme\|engagement\|priorit\|plan\|declaration\|sites/\|files/)\|appuyez\.quebecsolidaire\.net/\|signez\.quebecsolidaire\.net/\|api-wp\.quebecsolidaire\.net/wp-content/uploads/202[2-9])", re.I),
	74	+ re.compile(r"(\.(jpg\|jpeg\|png\|gif\|svg\|webp\|mp4)\|/en/\|/page/\d\|#\|offre-demploi\|investiture\|candidat\|lancement-de-campagne\|donnez\|don\b)", re.I),
	75	+ ("https://quebecsolidaire.net/nouvelles", "https://site.quebecsolidaire.net/nouvelles", "https://appuyez.quebecsolidaire.net/priorites",
	76	+ "https://appuyez.quebecsolidaire.net/priorites/logement", "https://appuyez.quebecsolidaire.net/priorites/inflation",
	77	+ "https://appuyez.quebecsolidaire.net/priorites/environnement", "https://signez.quebecsolidaire.net/plan-habitation-2026")),
76	78	"pcq": PartySite("pcq", "https://conservateur.quebec/",
77		− ("plateforme 2026", "programme", "engagements", "cadre financier", "communiqué"),
78		− re.compile(r"conservateur\.quebec/(?!.*(don\|boutique\|adhesion\|nous-joindre\|recherche\|candidat))", re.I),
79		− re.compile(r"(\.(jpg\|jpeg\|png\|gif\|svg\|webp\|mp4)\|/en/\|/page/\d\|wp-json\|/feed\|#)", re.I),
80		− ("https://conservateur.quebec/", "https://conservateur.quebec/nouvelles/", "https://conservateur.quebec/plateforme/", "https://conservateur.quebec/programme/")),
	79	+ ("plateforme 2026", "programme", "engagements", "plan", "communiqué"),
	80	+ re.compile(r"conservateur\.quebec/(blog/ressources/\|actualites\|plateforme\|programme\|engagement\|wp-content/uploads/202[2-9])", re.I),
	81	+ re.compile(r"(\.(jpg\|jpeg\|png\|gif\|svg\|webp\|mp4)\|/en/\|/page/\d\|wp-json\|/feed\|#\|adhesion\|don\b\|rencontre-communautaire\|video-entrevue\|lettre-a-\|citizenship)", re.I),
	82	+ ("https://conservateur.quebec/actualites", "https://conservateur.quebec/blog/ressources", "https://conservateur.quebec/")),
81	83	}
	84	+OLD_YEAR_RE = re.compile(r"/(19\d\d\|200\d\|201\d\|202[01])/") # documents datés d'avant 2022 dans l'URL : archives, non analysés
82	85
83	86	DOC_KEYWORDS = {
84	87	"plateforme": re.compile(r"(plateforme\|platform\|programme\|program\|projet-de-societe\|engagements\|promesses\|priorites\|vision)", re.I),
		`@@ -100,37 +103,63 @@ def classify_doc_type(url: str, title: str \| None) -> str:`
100	103	return "page"
101	104
102	105
103		−def discover(site: PartySite, max_pages: int = 60) -> list[tuple[str, str \| None]]:
104		− """URL candidates (PDF et pages de fond) sur le site officiel."""
	106	+CONTENT_RE = re.compile(r"(plateforme\|programme\|engagement\|priorit\|cadre\|vision\|propos\|promesse\|plan\|mesure\|projet\|nouvelles?/\|actualit\|communique\|blog/ressources\|declaration\|\.pdf)", re.I)
	107	+
	108	+
	109	+def _page_links(url: str, site: PartySite) -> tuple[list[str], str]:
	110	+ """Liens d'une page (HTML direct ; Firecrawl si le site est rendu en JavaScript). Retourne (liens, texte)."""
	111	+ html, text = "", ""
	112	+ try:
	113	+ r = http().get(url)
	114	+ if r.status_code < 400:
	115	+ html = r.text
	116	+ text = html_to_text(html)
	117	+ except Exception as e: # noqa: BLE001
	118	+ log.debug("seed %s : %s", url, e)
	119	+ if len(text) < 400 and settings.firecrawl_api_key:
	120	+ try:
	121	+ html, _ = firecrawl_scrape(url)
	122	+ except Exception as e: # noqa: BLE001
	123	+ log.debug("seed firecrawl %s : %s", url, e)
	124	+ base = site.site.split("/", 3)[0] + "//" + url.split("/")[2]
	125	+ links = []
	126	+ for m in re.finditer(r'href=["\']([^"\'#]+)["\']', html):
	127	+ u = m.group(1)
	128	+ if u.startswith("/"):
	129	+ u = base + u
	130	+ if u.startswith("http"):
	131	+ links.append(u.split("#")[0].rstrip("/"))
	132	+ return links, text
	133	+
	134	+
	135	+def discover(site: PartySite, max_pages: int = 80) -> list[tuple[str, str \| None]]:
	136	+ """URL candidates (PDF et pages de fond) sur le site officiel : Firecrawl map + pages d'amorçage."""
105	137	urls: dict[str, str \| None] = {}
	138	+
	139	+ def add(u: str) -> None:
	140	+ if site.include.search(u) and not site.exclude.search(u) and CONTENT_RE.search(u):
	141	+ urls.setdefault(u.split("#")[0].rstrip("/"), None)
106	142	for q in site.searches:
107	143	for u in firecrawl_map(site.site, q, 80):
108		− if site.include.search(u) and not site.exclude.search(u):
109		− urls.setdefault(u.split("#")[0], None)
	144	+ add(u)
110	145	for u in firecrawl_map(site.site, None, 400):
111		− if u.lower().split("?")[0].endswith(".pdf") and site.include.search(u):
112		− urls.setdefault(u.split("#")[0], None)
	146	+ add(u)
113	147	for sp in site.seed_pages:
114		− try:
115		− r = http().get(sp)
116		− if r.status_code >= 400:
117		− continue
118		− for m in re.finditer(r'href=["\']([^"\']+)["\']', r.text):
119		− u = m.group(1)
120		− if u.startswith("/"):
121		− u = site.site.split("/", 3)[0] + "//" + site.site.split("/")[2] + u
122		− if u.lower().split("?")[0].endswith(".pdf") and site.include.search(u):
123		− urls.setdefault(u.split("#")[0], None)
124		− except Exception as e: # noqa: BLE001
125		− log.debug("seed %s : %s", sp, e)
	148	+ links, _ = _page_links(sp, site)
	149	+ add(sp)
	150	+ for u in links:
	151	+ add(u)
126	152	pdfs = [u for u in urls if u.lower().split("?")[0].endswith(".pdf")]
127		− pages = [u for u in urls if u not in pdfs and re.search(r"(plateforme\|programme\|engagement\|priorit\|cadre\|vision\|propos\|promesse\|plan\|mesure\|projet\|nouvelles?/\|actualit\|communique)", u, re.I)]
128		− # priorité : PDF, puis pages « de fond »
129		− return [(u, None) for u in pdfs[:30]] + [(u, None) for u in pages[:max_pages]]
	153	+ pages = [u for u in urls if u not in pdfs]
	154	+ # priorité : PDF récents, puis pages « de fond » ; les documents datés d'avant 2022 passent en dernier (archives)
	155	+ recent_pdfs = [u for u in pdfs if not OLD_YEAR_RE.search(u)]
	156	+ old_pdfs = [u for u in pdfs if OLD_YEAR_RE.search(u)]
	157	+ return [(u, None) for u in recent_pdfs[:40]] + [(u, None) for u in pages[:max_pages]] + [(u, None) for u in old_pdfs[:20]]
130	158
131	159
132	160	def fetch_version(s: Session, doc: PartyDocument) -> DocumentVersion \| None:
133	161	"""Télécharge, hache, archive ; None si la version courante est identique."""
	162	+ rendered_md: str \| None = None
134	163	try:
135	164	b, r = fetch_bytes(doc.url)
136	165	ctype = (r.headers.get("content-type") or "").split(";")[0].lower()
		`@@ -144,6 +173,20 @@ def fetch_version(s: Session, doc: PartyDocument) -> DocumentVersion \| None:`
144	173	if not is_pdf:
145	174	# pour les pages HTML, hacher le texte principal (les scripts/dates changent à chaque chargement)
146	175	text = html_to_text(b.decode("utf-8", errors="replace"))
	176	+ if len(text) < 400 and settings.firecrawl_api_key:
	177	+ # site rendu en JavaScript (ex. Québec solidaire) : rendu Firecrawl, texte = markdown
	178	+ try:
	179	+ import httpx as _hx
	180	+ rr = _hx.post("https://api.firecrawl.dev/v1/scrape", timeout=120, headers={"Authorization": f"Bearer {settings.firecrawl_api_key}"},
	181	+ json={"url": doc.url, "formats": ["markdown", "html"], "onlyMainContent": True})
	182	+ rr.raise_for_status()
	183	+ data = rr.json().get("data", {})
	184	+ rendered_md = (data.get("markdown") or "").strip()
	185	+ if len(rendered_md) >= 400:
	186	+ text = rendered_md
	187	+ b = (data.get("html") or rendered_md).encode()
	188	+ except Exception as e: # noqa: BLE001
	189	+ log.debug("firecrawl %s : %s", doc.url, e)
147	190	hash_src = text.encode()
148	191	else:
149	192	text, hash_src = "", b
		`@@ -226,6 +269,12 @@ def analyze_version(s: Session, doc: PartyDocument, v: DocumentVersion, max_chun`
226	269	doc.status = "ignored"
227	270	v.analyzed_at = utcnow()
228	271	return 0
	272	+ if OLD_YEAR_RE.search(doc.url):
	273	+ # document d'archive (daté d'avant 2022 dans l'URL) : conservé et versionné, mais pas de propositions 2026
	274	+ doc.status = "archive"
	275	+ doc.doc_type = "archive"
	276	+ v.analyzed_at = utcnow()
	277	+ return 0
229	278	s.query(DocumentChunk).filter(DocumentChunk.version_id == v.id).delete()
230	279	s.query(PolicyProposal).filter(PolicyProposal.document_id == doc.id).delete()
231	280	chunks = chunk_text(text)

modified engine/qc26/llm.py +9 −2

		`@@ -50,7 +50,7 @@ def _extract_json(text: str) -> Any:`
50	50	raise
51	51
52	52
53		−def complete_json(system: str, user: str, *, effort: str = "low", max_tokens: int = 12000) -> tuple[Any, dict]:
	53	+def complete_json(system: str, user: str, *, effort: str = "low", max_tokens: int = 12000, _retry: bool = False) -> tuple[Any, dict]:
54	54	"""Appel Claude → JSON parsé + métadonnées d'usage. Lève en cas d'échec."""
55	55	if not available():
56	56	raise RuntimeError("LLM désactivé ou clé absente")
		`@@ -78,4 +78,11 @@ def complete_json(system: str, user: str, *, effort: str = "low", max_tokens: in`
78	78	"request_id": getattr(msg, "_request_id", None), "stop_reason": msg.stop_reason}
79	79	if msg.stop_reason == "max_tokens":
80	80	log.warning("réponse LLM tronquée (max_tokens=%d)", max_tokens)
81		− return _extract_json(text), usage
	81	+ try:
	82	+ return _extract_json(text), usage
	83	+ except (json.JSONDecodeError, ValueError):
	84	+ if _retry:
	85	+ raise
	86	+ log.warning("JSON invalide reçu du modèle — nouvelle tentative")
	87	+ return complete_json(system, user + "\n\nRAPPEL : réponds strictement avec un seul objet JSON valide (guillemets doubles, pas de virgule finale, pas de commentaire).",
	88	+ effort=effort, max_tokens=max_tokens, _retry=True)
82	89