Guichet-Emplois : reprise sur coupure WAF (retry + acquis partiel) ; fixtures cascades/cssdm
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
7 changed files +12,893 −4
modified
jobka/connectors/guichet_emplois.py
+23 −4
@@ -24,6 +24,9 @@ from __future__ import annotations | |||||
| 24 | import html as _html | 24 | import html as _html | ||
| 25 | import os | 25 | import os | ||
| 26 | import re | 26 | import re | ||
| 27 | +import time | ||||
| 28 | + | ||||
| 29 | +import requests | ||||
| 27 | 30 | ||||
| 28 | from ..schema import JobPosting, clean_html | 31 | from ..schema import JobPosting, clean_html | ||
| 29 | from .base import BaseConnector | 32 | from .base import BaseConnector | ||
@@ -75,9 +78,22 @@ class GuichetEmploisConnector(BaseConnector): | |||||
| 75 | details_used = 0 | 78 | details_used = 0 | ||
| 76 | seen: set[str] = set() | 79 | seen: set[str] = set() | ||
| 77 | for page in range(1, MAX_PAGES + 1): | 80 | for page in range(1, MAX_PAGES + 1): | ||
| 78 | − html = self.get(f"{BASE}/jobsearch/rechercheemplois", | 81 | + html = "" | ||
| 79 | − params={"fprov": "QC", "sort": "D", | 82 | + for attempt in (1, 2): | ||
| 80 | − "fage": DAYS, "page": str(page)}).text | 83 | + try: | ||
| 84 | + html = self.get(f"{BASE}/jobsearch/rechercheemplois", | ||||
| 85 | + params={"fprov": "QC", "sort": "D", | ||||
| 86 | + "fage": DAYS, | ||||
| 87 | + "page": str(page)}).text | ||||
| 88 | + break | ||||
| 89 | + except (requests.ConnectionError, requests.Timeout): | ||||
| 90 | + # le WAF du Guichet coupe parfois la connexion : une | ||||
| 91 | + # reprise après pause, sinon on garde l'acquis partiel | ||||
| 92 | + if attempt == 2: | ||||
| 93 | + if out: | ||||
| 94 | + return out | ||||
| 95 | + raise | ||||
| 96 | + time.sleep(25) | ||||
| 81 | items = _ITEM_RE.findall(html) | 97 | items = _ITEM_RE.findall(html) | ||
| 82 | if not items: | 98 | if not items: | ||
| 83 | break | 99 | break | ||
@@ -115,7 +131,10 @@ class GuichetEmploisConnector(BaseConnector): | |||||
| 115 | fresh[0] = True | 131 | fresh[0] = True | ||
| 116 | return self._fetch_detail(j) | 132 | return self._fetch_detail(j) | ||
| 117 | 133 | ||||
| 118 | − d = self.detail(jid, key, _fn) | 134 | + try: | ||
| 135 | + d = self.detail(jid, key, _fn) | ||||
| 136 | + except (requests.ConnectionError, requests.Timeout): | ||||
| 137 | + d = {} # détail repris à la prochaine synchro | ||||
| 119 | if fresh[0]: | 138 | if fresh[0]: | ||
| 120 | details_used += 1 | 139 | details_used += 1 | ||
| 121 | job.description = d.get("description", "") | 140 | job.description = d.get("description", "") | ||
added
tests/fixtures/cascades/0bd8019141bff72510da.txt
+9841 −0
Large diff (9,842 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cascades/expected.json
+1377 −0
Large diff (1,378 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cascades/index.json
+9 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cssdm/2e4b3d263118d85271af.html
+1307 −0
Large diff (1,308 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cssdm/expected.json
+327 −0
Large diff (328 lines) is not loaded by default to keep the page fast.Load diff
added
tests/fixtures/cssdm/index.json
+9 −0
Large diff (11 lines) is not loaded by default to keep the page fast.Load diff