SPB Git forge
spboucher.ai ↗

spb/home-ka

Public
10commits 1branches 0releases
793.0 KBsize
maindefault branch
19 days agolast push
Python 49.6% TypeScript 25.5% CSS 24.1%

Firecrawl throughput backend: fc_scrape (retries, maxAge cache) + parallel detail fetching (fc_workers) + sitemap bot-wall fallback; Delta Media family unlocked (ebby, semonin, patterson-schwartz, watson) — all jsonld sources switched, max_pages 2500

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 27 days ago (Aug 28, 2026) parent f1f5354

2 changed files +101 −12

modified homeka/connectors/base.py +32 −0
@@ -103,6 +103,38 @@ class BaseConnector:
103 103 resp.raise_for_status()
104 104 return resp
105 105
106 + def fc_scrape(self, url: str, proxy: str = "auto", render: bool = False,
107 + wait_for: int = 0, retries: int = 2) -> str:
108 + """Raw HTML via Firecrawl — used as a THROUGHPUT backend (parallel
109 + fetching without hammering the source directly) and to reach
110 + bot-challenged sites. `maxAge` leans on Firecrawl's cache so weekly
111 + re-syncs stay cheap. Retries on 429/5xx with backoff."""
112 + key = os.environ.get("FIRECRAWL_API_KEY")
113 + if not key:
114 + raise RuntimeError("FIRECRAWL_API_KEY missing (see .env)")
115 + payload: dict = {"url": url, "timeout": 60000,
116 + "formats": ["html"] if render else ["rawHtml"],
117 + "proxy": proxy,
118 + "maxAge": 6 * 86400 * 1000}
119 + if wait_for:
120 + payload["waitFor"] = wait_for
121 + last = ""
122 + for attempt in range(retries + 1):
123 + try:
124 + resp = requests.post(
125 + FIRECRAWL_API, json=payload,
126 + headers={"Authorization": f"Bearer {key}"}, timeout=120)
127 + if resp.status_code in (429, 500, 502, 503):
128 + time.sleep(3 * (attempt + 1))
129 + continue
130 + resp.raise_for_status()
131 + data = resp.json().get("data") or {}
132 + return data.get("rawHtml") or data.get("html") or ""
133 + except requests.RequestException as exc:
134 + last = str(exc)
135 + time.sleep(2 * (attempt + 1))
136 + raise RuntimeError(f"firecrawl failed for {url}: {last}")
137 +
106 138 def get_rendered(self, url: str, wait_for: int = 0,
107 139 proxy: str | None = None) -> str:
108 140 """LAST RESORT — JS-rendered HTML via Firecrawl (FIRECRAWL_API_KEY)."""
modified homeka/connectors/json/jsonld_site.py +69 −12
@@ -131,8 +131,17 @@ class JSONLDSiteConnector(BaseConnector):
131 131 "utf-8", errors="replace")
132 132 else:
133 133 body = resp.text
134 + # bot wall on the sitemap itself → Firecrawl fallback
135 + if ("<loc" not in body and "<urlset" not in body
136 + and self.config.get("backend") == "firecrawl"):
137 + body = self.fc_scrape(sm)
134 138 except Exception:
135 − continue
139 + if self.config.get("backend") != "firecrawl":
140 + continue
141 + try:
142 + body = self.fc_scrape(sm)
143 + except Exception:
144 + continue
136 145 for loc in _LOC_RE.findall(body):
137 146 # nested sitemap: .xml/.gz files, or dynamic shard URLs
138 147 # (sitemapindex.aspx / sitemaplistings.ashx à la Weichert)
@@ -155,15 +164,28 @@ class JSONLDSiteConnector(BaseConnector):
155 164 add(u.split("#")[0])
156 165 return urls[:max_pages]
157 166
167 + # -- page fetch backends -------------------------------------------------
168 + def _page(self, url: str) -> str:
169 + """One listing page: Firecrawl backend (config backend=firecrawl —
170 + parallel-safe, bypasses bot walls, spares the source) or polite
171 + direct GET."""
172 + if self.config.get("backend") == "firecrawl":
173 + return self.fc_scrape(url,
174 + proxy=self.config.get("fc_proxy", "auto"),
175 + render=bool(self.config.get("fc_render")))
176 + return self.get(url).text
177 +
158 178 # -- fetch --------------------------------------------------------------
159 179 def fetch(self) -> list[Listing]:
160 − static = self.config.get("static") or {}
161 − ext_re = re.compile(self.config["external_id_regex"]) \
162 − if self.config.get("external_id_regex") else None
180 + cfg = self.config
181 + static = cfg.get("static") or {}
182 + ext_re = re.compile(cfg["external_id_regex"]) \
183 + if cfg.get("external_id_regex") else None
163 184 # weekly cache bucket: each detail page is re-fetched at most once a
164 185 # week (price/status refresh) and never re-fetched inside a cycle
165 186 week = int(time.time() // (7 * 86400))
166 − out: list[Listing] = []
187 + items: list[tuple[str, str]] = []
188 + seen_ext: set[str] = set()
167 189 for url in self._listing_urls():
168 190 ext = ""
169 191 if ext_re:
@@ -172,15 +194,50 @@ class JSONLDSiteConnector(BaseConnector):
172 194 ext = m.group(1)
173 195 if not ext:
174 196 ext = hashlib.sha1(url.encode()).hexdigest()[:16]
197 + if ext not in seen_ext:
198 + seen_ext.add(ext)
199 + items.append((url, ext))
175 200
176 − def _fetch(url=url):
177 − try:
178 − return {"ld": list(iter_ld(self.get(url).text))}
179 − except Exception:
180 − return {}
201 + from ... import db
202 + con = db.connect()
203 + payloads: dict[str, dict] = {}
204 + misses: list[tuple[str, str]] = []
205 + for url, ext in items:
206 + cached = db.get_cached_detail(con, self.source_id, ext,
207 + f"w{week}:{url}")
208 + if cached is not None:
209 + payloads[ext] = cached
210 + else:
211 + misses.append((url, ext))
181 212
182 − payload = self.detail(ext, f"w{week}:{url}", _fetch)
183 − lst = self._from_nodes(payload.get("ld") or [], url, ext, static)
213 + def _grab(url: str) -> dict:
214 + try:
215 + return {"ld": list(iter_ld(self._page(url)))}
216 + except Exception:
217 + return {}
218 +
219 + if cfg.get("backend") == "firecrawl" and misses:
220 + # Firecrawl absorbs the load — fetch in parallel, cache serially
221 + from concurrent.futures import ThreadPoolExecutor
222 + workers = int(cfg.get("fc_workers", 8))
223 + with ThreadPoolExecutor(max_workers=workers) as pool:
224 + results = pool.map(lambda t: _grab(t[0]), misses)
225 + for (url, ext), payload in zip(misses, results):
226 + db.put_cached_detail(con, self.source_id, ext,
227 + f"w{week}:{url}", payload)
228 + payloads[ext] = payload
229 + else:
230 + for url, ext in misses:
231 + payload = _grab(url)
232 + db.put_cached_detail(con, self.source_id, ext,
233 + f"w{week}:{url}", payload)
234 + payloads[ext] = payload
235 + con.close()
236 +
237 + out: list[Listing] = []
238 + for url, ext in items:
239 + lst = self._from_nodes(payloads.get(ext, {}).get("ld") or [],
240 + url, ext, static)
184 241 if lst is not None:
185 242 out.append(lst)
186 243 return out
187 244