SPB Git forge
spboucher.ai ↗

spb/trawls

Public
3commits 1branches 0releases
456.0 KBsize
maindefault branch
19 days agolast push
Python 76.2% JavaScript 11.3% CSS 6.3% HTML 5.9%

fix: cache deep copy (liens du crawl), fallback body quand le contenu principal est vide, plafond map en mode search, manifeste mld

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 19 days ago (Sep 5, 2026) parent 20da528

2 changed files +60 −20

modified CLAUDE.md +34 −18
@@ -88,41 +88,57 @@ trawls/
88 88
89 89 ```python
90 90 class ScrapeOptions(BaseModel):
91 − formats: list[Literal["markdown","html","raw_html","json","links","screenshot","chunks","metadata"]] = ["markdown"]
91 + formats: list[
92 + Literal["markdown", "html", "raw_html", "json", "links", "screenshot", "chunks", "metadata"]
93 + ] = ["markdown"]
92 94 only_main_content: bool = True
93 − include_tags: list[str] = [] # CSS, forcer l'inclusion
95 + include_tags: list[str] = [] # CSS, forcer l'inclusion
94 96 exclude_tags: list[str] = []
95 − wait_for: str | int | None = None # sélecteur CSS ou ms
97 + wait_for: str | int | None = None # sélecteur CSS ou ms
96 98 timeout_ms: int = 30_000
97 − mode: Literal["auto","http","browser","stealth"] = "auto"
99 + mode: Literal["auto", "http", "browser", "stealth"] = "auto"
98 100 headers: dict[str, str] = {}
99 101 cookies: list[Cookie] = []
100 102 proxy: str | None = None
101 − actions: list[BrowserAction] = [] # click/scroll/type/wait/press/evaluate avant extraction
102 − location: Location | None = None # pays, langues → headers + timezone
103 + actions: list[BrowserAction] = [] # click/scroll/type/wait/press/evaluate avant extraction
104 + location: Location | None = None # pays, langues → headers + timezone
103 105 remove_base64_images: bool = True
104 106 chunk: ChunkOptions | None = None
105 − extract: ExtractOptions | None = None # css: {champ: CssField} | llm: schema JSON + prompt
106 − cache: Literal["use","bypass","refresh"] = "use"
107 + extract: ExtractOptions | None = None # css: {champ: CssField} | llm: schema JSON + prompt
108 + cache: Literal["use", "bypass", "refresh"] = "use"
107 109 max_age_s: int = 86_400
108 110 citations: bool = False
109 111 verify_ssl: bool = True
110 112 respect_robots: bool = True
111 113
114 +
112 115 class PageResult(BaseModel):
113 − url: str; final_url: str
114 − status: Literal["ok","failed","skipped"]
116 + url: str
117 + final_url: str
118 + status: Literal["ok", "failed", "skipped"]
115 119 http_status: int | None
116 − fetch_mode_used: Literal["http","browser","stealth"] | None
117 − markdown: str | None; html: str | None; raw_html: str | None
118 − json_data: dict | None # {"data", "errors"} pour extract ; {"jsonld"} pour format json
119 − links: list[Link]; metadata: PageMetadata; chunks: list[Chunk] | None
120 − screenshot_url: str | None; error: ErrorInfo | None
121 − timings: Timings # ttfb, fetch, render, process, total (ms)
122 − fetched_at: datetime; depth: int; from_cache: bool
120 + fetch_mode_used: Literal["http", "browser", "stealth"] | None
121 + markdown: str | None
122 + html: str | None
123 + raw_html: str | None
124 + json_data: dict | None # {"data", "errors"} pour extract ; {"jsonld"} pour format json
125 + links: list[Link]
126 + metadata: PageMetadata
127 + chunks: list[Chunk] | None
128 + screenshot_url: str | None
129 + error: ErrorInfo | None
130 + timings: Timings # ttfb, fetch, render, process, total (ms)
131 + fetched_at: datetime
132 + depth: int
133 + from_cache: bool
134 +
123 135
124 136 class ErrorInfo(BaseModel):
125 − code: ErrorCode; message: str; retryable: bool; attempts: int; details: dict | None
137 + code: ErrorCode
138 + message: str
139 + retryable: bool
140 + attempts: int
141 + details: dict | None
126 142 ```
127 143
128 144 Règle : aucun `dict` non typé ne sort de l'API. Tout passe par un modèle.
modified trawls/processors/html_to_md/__init__.py +26 −2
@@ -49,8 +49,32 @@ def html_to_markdown(
49 49 )
50 50 hint = ident[:120]
51 51 conv = Converter(base_url=base_url, remove_base64_images=remove_base64_images)
52 − md = conv.block(root)
53 − md = postprocess(md)
52 + md = postprocess(conv.block(root))
53 + if only_main_content and len(md.split()) < 30:
54 + # Filet : le scoring n'a rien gardé (page d'accueil, app, portail de liens) → on repart du body
55 + # nettoyé sans readability, puis sans suppression de boilerplate si toujours vide.
56 + full = clean(
57 + html or "", exclude_selectors=exclude_tags, include_selectors=include_tags, boilerplate=True
58 + )
59 + body = full.body if full.body is not None else full.root
60 + if body is not None:
61 + alt = postprocess(
62 + Converter(base_url=base_url, remove_base64_images=remove_base64_images).block(body)
63 + )
64 + if len(alt.split()) < 30:
65 + raw = clean(
66 + html or "",
67 + exclude_selectors=exclude_tags,
68 + include_selectors=include_tags,
69 + boilerplate=False,
70 + )
71 + rb = raw.body if raw.body is not None else raw.root
72 + if rb is not None:
73 + alt = postprocess(
74 + Converter(base_url=base_url, remove_base64_images=remove_base64_images).block(rb)
75 + )
76 + if len(alt.split()) > len(md.split()):
77 + md, root, hint = alt, body, "body (fallback)"
54 78 if citations:
55 79 md = add_citations(md)
56 80 clean_html = root.html or ""
57 81