SPB Git
spboucher.ai

spb/trawls

Public

fix: cache deep copy (liens du crawl), fallback body quand le contenu principal est vide, plafond map en mode search, manifeste mld

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 5, 2026) parent 20da528 Browse files

2 changed files +60 −20

2 changed files
Unified Split
modified CLAUDE.md +34 −18
@@ -88,41 +88,57 @@ trawls/
88 88
89 ```python 89 ```python
90 class ScrapeOptions(BaseModel): 90 class ScrapeOptions(BaseModel):
91− formats: list[Literal["markdown","html","raw_html","json","links","screenshot","chunks","metadata"]] = ["markdown"] 91+ formats: list[
92+ Literal["markdown", "html", "raw_html", "json", "links", "screenshot", "chunks", "metadata"]
93+ ] = ["markdown"]
92 only_main_content: bool = True 94 only_main_content: bool = True
93− include_tags: list[str] = [] # CSS, forcer l'inclusion 95+ include_tags: list[str] = [] # CSS, forcer l'inclusion
94 exclude_tags: list[str] = [] 96 exclude_tags: list[str] = []
95− wait_for: str | int | None = None # sélecteur CSS ou ms 97+ wait_for: str | int | None = None # sélecteur CSS ou ms
96 timeout_ms: int = 30_000 98 timeout_ms: int = 30_000
97− mode: Literal["auto","http","browser","stealth"] = "auto" 99+ mode: Literal["auto", "http", "browser", "stealth"] = "auto"
98 headers: dict[str, str] = {} 100 headers: dict[str, str] = {}
99 cookies: list[Cookie] = [] 101 cookies: list[Cookie] = []
100 proxy: str | None = None 102 proxy: str | None = None
101− actions: list[BrowserAction] = [] # click/scroll/type/wait/press/evaluate avant extraction 103+ actions: list[BrowserAction] = [] # click/scroll/type/wait/press/evaluate avant extraction
102− location: Location | None = None # pays, langues → headers + timezone 104+ location: Location | None = None # pays, langues → headers + timezone
103 remove_base64_images: bool = True 105 remove_base64_images: bool = True
104 chunk: ChunkOptions | None = None 106 chunk: ChunkOptions | None = None
105− extract: ExtractOptions | None = None # css: {champ: CssField} | llm: schema JSON + prompt 107+ extract: ExtractOptions | None = None # css: {champ: CssField} | llm: schema JSON + prompt
106− cache: Literal["use","bypass","refresh"] = "use" 108+ cache: Literal["use", "bypass", "refresh"] = "use"
107 max_age_s: int = 86_400 109 max_age_s: int = 86_400
108 citations: bool = False 110 citations: bool = False
109 verify_ssl: bool = True 111 verify_ssl: bool = True
110 respect_robots: bool = True 112 respect_robots: bool = True
111 113
114+
112 class PageResult(BaseModel): 115 class PageResult(BaseModel):
113− url: str; final_url: str 116+ url: str
114− status: Literal["ok","failed","skipped"] 117+ final_url: str
118+ status: Literal["ok", "failed", "skipped"]
115 http_status: int | None 119 http_status: int | None
116− fetch_mode_used: Literal["http","browser","stealth"] | None 120+ fetch_mode_used: Literal["http", "browser", "stealth"] | None
117− markdown: str | None; html: str | None; raw_html: str | None 121+ markdown: str | None
118− json_data: dict | None # {"data", "errors"} pour extract ; {"jsonld"} pour format json 122+ html: str | None
119− links: list[Link]; metadata: PageMetadata; chunks: list[Chunk] | None 123+ raw_html: str | None
120− screenshot_url: str | None; error: ErrorInfo | None 124+ json_data: dict | None # {"data", "errors"} pour extract ; {"jsonld"} pour format json
121− timings: Timings # ttfb, fetch, render, process, total (ms) 125+ links: list[Link]
122− fetched_at: datetime; depth: int; from_cache: bool 126+ metadata: PageMetadata
127+ chunks: list[Chunk] | None
128+ screenshot_url: str | None
129+ error: ErrorInfo | None
130+ timings: Timings # ttfb, fetch, render, process, total (ms)
131+ fetched_at: datetime
132+ depth: int
133+ from_cache: bool
134+
123 135
124 class ErrorInfo(BaseModel): 136 class ErrorInfo(BaseModel):
125− code: ErrorCode; message: str; retryable: bool; attempts: int; details: dict | None 137+ code: ErrorCode
138+ message: str
139+ retryable: bool
140+ attempts: int
141+ details: dict | None
126 ``` 142 ```
127 143
128 Règle : aucun `dict` non typé ne sort de l'API. Tout passe par un modèle. 144 Règle : aucun `dict` non typé ne sort de l'API. Tout passe par un modèle.
modified trawls/processors/html_to_md/__init__.py +26 −2
@@ -49,8 +49,32 @@ def html_to_markdown(
49 ) 49 )
50 hint = ident[:120] 50 hint = ident[:120]
51 conv = Converter(base_url=base_url, remove_base64_images=remove_base64_images) 51 conv = Converter(base_url=base_url, remove_base64_images=remove_base64_images)
52− md = conv.block(root) 52+ md = postprocess(conv.block(root))
53− md = postprocess(md) 53+ if only_main_content and len(md.split()) < 30:
54+ # Filet : le scoring n'a rien gardé (page d'accueil, app, portail de liens) → on repart du body
55+ # nettoyé sans readability, puis sans suppression de boilerplate si toujours vide.
56+ full = clean(
57+ html or "", exclude_selectors=exclude_tags, include_selectors=include_tags, boilerplate=True
58+ )
59+ body = full.body if full.body is not None else full.root
60+ if body is not None:
61+ alt = postprocess(
62+ Converter(base_url=base_url, remove_base64_images=remove_base64_images).block(body)
63+ )
64+ if len(alt.split()) < 30:
65+ raw = clean(
66+ html or "",
67+ exclude_selectors=exclude_tags,
68+ include_selectors=include_tags,
69+ boilerplate=False,
70+ )
71+ rb = raw.body if raw.body is not None else raw.root
72+ if rb is not None:
73+ alt = postprocess(
74+ Converter(base_url=base_url, remove_base64_images=remove_base64_images).block(rb)
75+ )
76+ if len(alt.split()) > len(md.split()):
77+ md, root, hint = alt, body, "body (fallback)"
54 if citations: 78 if citations:
55 md = add_citations(md) 79 md = add_citations(md)
56 clean_html = root.html or "" 80 clean_html = root.html or ""
57 81
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere