Guichet-Emplois : reprise sur coupure WAF (retry + acquis partiel) ; fixtures cascades/cssdm
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
7 changed files +12,893 −4
modified
jobka/connectors/guichet_emplois.py
+23 −4
@@ -24,6 +24,9 @@ from __future__ import annotations | ||
| 24 | 24 | import html as _html |
| 25 | 25 | import os |
| 26 | 26 | import re |
| 27 | +import time | |
| 28 | + | |
| 29 | +import requests | |
| 27 | 30 | |
| 28 | 31 | from ..schema import JobPosting, clean_html |
| 29 | 32 | from .base import BaseConnector |
@@ -75,9 +78,22 @@ class GuichetEmploisConnector(BaseConnector): | ||
| 75 | 78 | details_used = 0 |
| 76 | 79 | seen: set[str] = set() |
| 77 | 80 | for page in range(1, MAX_PAGES + 1): |
| 78 | − html = self.get(f"{BASE}/jobsearch/rechercheemplois", | |
| 79 | − params={"fprov": "QC", "sort": "D", | |
| 80 | − "fage": DAYS, "page": str(page)}).text | |
| 81 | + html = "" | |
| 82 | + for attempt in (1, 2): | |
| 83 | + try: | |
| 84 | + html = self.get(f"{BASE}/jobsearch/rechercheemplois", | |
| 85 | + params={"fprov": "QC", "sort": "D", | |
| 86 | + "fage": DAYS, | |
| 87 | + "page": str(page)}).text | |
| 88 | + break | |
| 89 | + except (requests.ConnectionError, requests.Timeout): | |
| 90 | + # le WAF du Guichet coupe parfois la connexion : une | |
| 91 | + # reprise après pause, sinon on garde l'acquis partiel | |
| 92 | + if attempt == 2: | |
| 93 | + if out: | |
| 94 | + return out | |
| 95 | + raise | |
| 96 | + time.sleep(25) | |
| 81 | 97 | items = _ITEM_RE.findall(html) |
| 82 | 98 | if not items: |
| 83 | 99 | break |
@@ -115,7 +131,10 @@ class GuichetEmploisConnector(BaseConnector): | ||
| 115 | 131 | fresh[0] = True |
| 116 | 132 | return self._fetch_detail(j) |
| 117 | 133 | |
| 118 | − d = self.detail(jid, key, _fn) | |
| 134 | + try: | |
| 135 | + d = self.detail(jid, key, _fn) | |
| 136 | + except (requests.ConnectionError, requests.Timeout): | |
| 137 | + d = {} # détail repris à la prochaine synchro | |
| 119 | 138 | if fresh[0]: |
| 120 | 139 | details_used += 1 |
| 121 | 140 | job.description = d.get("description", "") |
added
tests/fixtures/cascades/0bd8019141bff72510da.txt
+9841 −0
Large diff (9,842 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cascades/expected.json
+1377 −0
Large diff (1,378 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cascades/index.json
+9 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cssdm/2e4b3d263118d85271af.html
+1307 −0
Large diff (1,308 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cssdm/expected.json
+327 −0
Large diff (328 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cssdm/index.json
+9 −0
Large diff (11 lines) is not loaded by default to keep the page fast.Load diff