feat: Trawls v0.1.0 — fetch auto (http→browser→stealth), html_to_md, PDF, map, extract CSS/LLM, chunker, jobs SQLite + SSE, API /v1, UI, CLI
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
67 changed files +7,933 −0
added
.env.example
+29 −0
@@ -0,0 +1,29 @@ | ||
| 1 | +# Trawls — configuration (préfixe TRAWLS_). Toutes les valeurs ont un défaut sûr. | |
| 2 | +TRAWLS_ENV=production | |
| 3 | +TRAWLS_PORT=8180 | |
| 4 | +TRAWLS_PUBLIC_URL=https://www.trawls.dev | |
| 5 | +TRAWLS_DATA_DIR=./data | |
| 6 | +TRAWLS_LOG_LEVEL=INFO | |
| 7 | + | |
| 8 | +# Auth : désactivée par défaut (self-host). ADMIN_KEY permet de créer des clés via /v1/keys. | |
| 9 | +TRAWLS_REQUIRE_AUTH=false | |
| 10 | +TRAWLS_ADMIN_KEY= | |
| 11 | +TRAWLS_RATE_LIMIT_PER_MINUTE=300 | |
| 12 | + | |
| 13 | +# Fetch | |
| 14 | +TRAWLS_BROWSER_POOL_SIZE=6 | |
| 15 | +TRAWLS_MAX_CONCURRENCY_PER_HOST=4 | |
| 16 | +TRAWLS_MAX_CONCURRENCY_GLOBAL=16 | |
| 17 | +TRAWLS_MAX_SIZE_MB=50 | |
| 18 | +TRAWLS_PROXY_URLS= | |
| 19 | +TRAWLS_ALLOW_PRIVATE_TARGETS=false | |
| 20 | + | |
| 21 | +# LLM (extraction llm) : OpenAI-compatible (Ollama/vLLM/OpenRouter) ou anthropic | |
| 22 | +TRAWLS_LLM_PROVIDER=openai | |
| 23 | +TRAWLS_LLM_BASE_URL=https://llm.maclustr.io/v1 | |
| 24 | +TRAWLS_LLM_API_KEY= | |
| 25 | +TRAWLS_LLM_MODEL=maclustr-general | |
| 26 | + | |
| 27 | +# Jobs | |
| 28 | +TRAWLS_WORKER_CONCURRENCY=4 | |
| 29 | +TRAWLS_RESULT_TTL_DAYS=7 | |
added
.gitignore
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +.venv/ | |
| 2 | +venv/ | |
| 3 | +__pycache__/ | |
| 4 | +*.pyc | |
| 5 | +.pytest_cache/ | |
| 6 | +.ruff_cache/ | |
| 7 | +.mypy_cache/ | |
| 8 | +data/ | |
| 9 | +*.db | |
| 10 | +*.db-wal | |
| 11 | +*.db-shm | |
| 12 | +.env | |
| 13 | +dist/ | |
| 14 | +build/ | |
| 15 | +*.egg-info/ | |
| 16 | +node_modules/ | |
| 17 | +.DS_Store | |
added
CLAUDE.md
+306 −0
@@ -0,0 +1,306 @@ | ||
| 1 | +# CLAUDE.md — Trawls (www.trawls.dev) | |
| 2 | + | |
| 3 | +> Moteur de crawling, d'extraction et de navigation web, construit from scratch. | |
| 4 | +> Objectif : dépasser crawl4ai et Firecrawl en robustesse, qualité de sortie et autonomie. | |
| 5 | +> Nom : **Trawls** — domaine **www.trawls.dev**. Un chalut (trawl) ratisse le fond et remonte tout : on cartographie et on extrait n'importe quel site. | |
| 6 | + | |
| 7 | +Ce fichier est la source de vérité pour Claude Code. Lis-le intégralement avant toute modification. En cas de doute entre deux approches, choisis celle qui respecte la section « Principes ». | |
| 8 | + | |
| 9 | +**État au 2026-09-05 (v0.1.0)** : phases 1 à 6 livrées et déployées sur MacLustr via `mld` (PM2 `trawls-api` + `trawls-ngrok` → https://www.trawls.dev). Écarts assumés par rapport à la cible ci-dessous, à résorber dans l'ordre : (a) persistance **SQLite + worker embarqué asyncio** au lieu de PostgreSQL + Redis/arq (principe « self-host first », zéro dépendance) — `TRAWLS_DATABASE_URL`/`TRAWLS_REDIS_URL` réservés ; (b) UI en **HTML/JS vanilla** servie par FastAPI (`trawls/web/static/`) au lieu de Next.js (`web/`) ; (c) blobs (screenshots) sur disque local `data/blobs/` au lieu de S3/MinIO ; (d) agent (phase 7) et SDK/benchmarks (phase 8) non commencés. Ne pas casser le contrat API existant en résorbant ces écarts. | |
| 10 | + | |
| 11 | +--- | |
| 12 | + | |
| 13 | +## 0. Principes (ordre de priorité) | |
| 14 | + | |
| 15 | +1. **Ne jamais crasher sur une page.** Une page qui échoue produit un `PageResult` avec `status=failed` et une erreur typée. Le crawl continue. | |
| 16 | +2. **Sortie LLM-ready par défaut.** Markdown propre, sans boilerplate, métadonnées complètes, chunks optionnels. La qualité du MD est le critère n°1 du produit. | |
| 17 | +3. **Escalade automatique.** HTTP pur d'abord (rapide, 90 % des cas), navigateur seulement si nécessaire, stealth seulement si bloqué. L'utilisateur ne choisit pas le mode sauf s'il le force. | |
| 18 | +4. **Déterministe avant probabiliste.** Extraction CSS/JSON-LD sans LLM quand c'est possible. Le LLM est un outil, pas une béquille. | |
| 19 | +5. **Self-host first.** Tout fonctionne sans clé API externe (Ollama pour le LLM local). Le SaaS est une couche au-dessus, jamais une dépendance. | |
| 20 | +6. **Observable.** Chaque job, chaque page, chaque retry est tracé. Pas de « ça a échoué on ne sait pas pourquoi ». | |
| 21 | +7. **Async partout, backpressure partout.** Aucun appel bloquant dans le chemin chaud. | |
| 22 | + | |
| 23 | +--- | |
| 24 | + | |
| 25 | +## 1. Stack technique | |
| 26 | + | |
| 27 | +| Couche | Choix | Raison / contraintes | | |
| 28 | +|---|---|---| | |
| 29 | +| Langage | Python 3.12+, `uv` pour les deps | Typage strict, `mypy --strict` | | |
| 30 | +| HTTP rapide | `curl_cffi` (impersonation TLS Chrome/Safari) | Passe la plupart des fingerprints JA3 | | |
| 31 | +| HTTP fallback | `httpx[http2]` async | Quand curl_cffi est indisponible | | |
| 32 | +| Navigateur | Playwright (Chromium par défaut, Firefox/WebKit optionnels) | Pool de contexts persistants, pas un browser par page | | |
| 33 | +| Stealth | patches maison (webdriver, plugins, canvas noise, WebGL, permissions) | Voir `core/antibot/stealth.py` | | |
| 34 | +| Parsing HTML | `selectolax` (Lexbor) | 10-20x plus rapide que BS4. `lxml` uniquement pour le fallback recover/XPath | | |
| 35 | +| Détection encodage | `charset-normalizer` | Fallback latin-1, jamais d'exception non gérée | | |
| 36 | +| PDF | `pymupdf` (texte, layout, tables) ; OCR : `rapidocr-onnxruntime` (extra `ocr`) | Détection scan = ratio texte/page < 50 | | |
| 37 | +| Markdown | Pipeline maison (`processors/html_to_md/`), pas de `markdownify` | Voir §4 | | |
| 38 | +| Tokenisation | `tiktoken` (cl100k) pour compter les tokens des chunks | | | |
| 39 | +| LLM | Adapters : Anthropic, OpenAI-compatible (couvre Ollama, vLLM, OpenRouter, llm.maclustr.io) | Interface unique `LLMClient` | | |
| 40 | +| Queue | **actuel** : worker asyncio embarqué (`worker/crawl.py`) · **cible** : Redis 7 + `arq` | Simple, retries natifs, cron | | |
| 41 | +| DB | **actuel** : SQLite (aiosqlite, WAL) · **cible** : PostgreSQL 16 (SQLModel/SQLAlchemy 2 async) | Jobs, pages, clés API, usage | | |
| 42 | +| Blobs | **actuel** : disque `data/blobs/` · **cible** : S3-compatible (MinIO en local) | HTML brut, screenshots, PDF sources | | |
| 43 | +| API | FastAPI + Pydantic v2 | OpenAPI générée = contrat du SDK | | |
| 44 | +| Realtime | SSE (progression jobs) ; WebSocket réservé à l'agent console | | | |
| 45 | +| UI | **actuel** : HTML/JS vanilla dense (`trawls/web/static/`) · **cible** : Next.js 15, TypeScript, Tailwind, shadcn/ui, TanStack Query (`web/`) | | | |
| 46 | +| SDK | Python (`trawls`) et TypeScript (`@trawls/sdk`) générés depuis OpenAPI puis polis à la main | phase 8 | | |
| 47 | +| Conteneurs | Docker Compose (api, worker, ollama) | `make up` doit tout lancer | | |
| 48 | + | |
| 49 | +--- | |
| 50 | + | |
| 51 | +## 2. Arborescence | |
| 52 | + | |
| 53 | +``` | |
| 54 | +trawls/ | |
| 55 | +├── trawls/ # package Python | |
| 56 | +│ ├── config.py # Settings pydantic-settings, un seul point d'entrée (préfixe TRAWLS_) | |
| 57 | +│ ├── models/ # Pydantic: ScrapeOptions, PageResult, Job, ErrorInfo… | |
| 58 | +│ ├── core/ | |
| 59 | +│ │ ├── fetcher/ # base (Protocol), http_fast (curl_cffi), browser (pool Playwright), strategy (escalade) | |
| 60 | +│ │ ├── antibot/ # detect, stealth, identity, blocklist.txt (proxy : pool à venir) | |
| 61 | +│ │ ├── scheduler/ # robots, politeness, dedup (normalisation URL + simhash) ; frontier dans worker/crawl.py | |
| 62 | +│ │ ├── resilience/ # retry, breaker, budget | |
| 63 | +│ │ ├── security.py # garde SSRF | |
| 64 | +│ │ └── scrape.py # pipeline d'une page — ne lève jamais | |
| 65 | +│ ├── processors/ | |
| 66 | +│ │ ├── html_to_md/ # clean, readability, convert, tables, citations | |
| 67 | +│ │ ├── pdf/ # extract + tables + OCR + to_md (un module) | |
| 68 | +│ │ ├── structured/ # metadata (head + OpenGraph + JSON-LD + microdata) | |
| 69 | +│ │ ├── chunker/ # by_heading, by_tokens | |
| 70 | +│ │ ├── encoding.py | |
| 71 | +│ │ └── links.py | |
| 72 | +│ ├── extract/ # css, llm, merge | |
| 73 | +│ ├── agent/ # phase 7 — à créer (loop, observe, tools, planner, memory, guards) | |
| 74 | +│ ├── map/ # sitemap + crawl_links + rank (BM25) dans un module | |
| 75 | +│ ├── llm/client.py # interface unique (anthropic + openai_compat) | |
| 76 | +│ ├── api/ # main (routes /v1, SSE, UI, docs), auth | |
| 77 | +│ ├── worker/ # store (SQLite), crawl (moteur de jobs : crawl/batch/extract, EventBus) | |
| 78 | +│ ├── web/static/ # UI : index.html, app.js, style.css | |
| 79 | +│ └── cli.py # trawls scrape|crawl|map|serve|worker|keys | |
| 80 | +├── tests/unit/ | |
| 81 | +├── deploy/trawls.manifest.json # manifeste mld (MacLustr) | |
| 82 | +├── Dockerfile · docker-compose.yml · Makefile · CLAUDE.md | |
| 83 | +``` | |
| 84 | + | |
| 85 | +--- | |
| 86 | + | |
| 87 | +## 3. Modèles de données (Pydantic v2) | |
| 88 | + | |
| 89 | +```python | |
| 90 | +class ScrapeOptions(BaseModel): | |
| 91 | + formats: list[Literal["markdown","html","raw_html","json","links","screenshot","chunks","metadata"]] = ["markdown"] | |
| 92 | + only_main_content: bool = True | |
| 93 | + include_tags: list[str] = [] # CSS, forcer l'inclusion | |
| 94 | + exclude_tags: list[str] = [] | |
| 95 | + wait_for: str | int | None = None # sélecteur CSS ou ms | |
| 96 | + timeout_ms: int = 30_000 | |
| 97 | + mode: Literal["auto","http","browser","stealth"] = "auto" | |
| 98 | + headers: dict[str, str] = {} | |
| 99 | + cookies: list[Cookie] = [] | |
| 100 | + proxy: str | None = None | |
| 101 | + actions: list[BrowserAction] = [] # click/scroll/type/wait/press/evaluate avant extraction | |
| 102 | + location: Location | None = None # pays, langues → headers + timezone | |
| 103 | + remove_base64_images: bool = True | |
| 104 | + chunk: ChunkOptions | None = None | |
| 105 | + extract: ExtractOptions | None = None # css: {champ: CssField} | llm: schema JSON + prompt | |
| 106 | + cache: Literal["use","bypass","refresh"] = "use" | |
| 107 | + max_age_s: int = 86_400 | |
| 108 | + citations: bool = False | |
| 109 | + verify_ssl: bool = True | |
| 110 | + respect_robots: bool = True | |
| 111 | + | |
| 112 | +class PageResult(BaseModel): | |
| 113 | + url: str; final_url: str | |
| 114 | + status: Literal["ok","failed","skipped"] | |
| 115 | + http_status: int | None | |
| 116 | + fetch_mode_used: Literal["http","browser","stealth"] | None | |
| 117 | + markdown: str | None; html: str | None; raw_html: str | None | |
| 118 | + json_data: dict | None # {"data", "errors"} pour extract ; {"jsonld"} pour format json | |
| 119 | + links: list[Link]; metadata: PageMetadata; chunks: list[Chunk] | None | |
| 120 | + screenshot_url: str | None; error: ErrorInfo | None | |
| 121 | + timings: Timings # ttfb, fetch, render, process, total (ms) | |
| 122 | + fetched_at: datetime; depth: int; from_cache: bool | |
| 123 | + | |
| 124 | +class ErrorInfo(BaseModel): | |
| 125 | + code: ErrorCode; message: str; retryable: bool; attempts: int; details: dict | None | |
| 126 | +``` | |
| 127 | + | |
| 128 | +Règle : aucun `dict` non typé ne sort de l'API. Tout passe par un modèle. | |
| 129 | + | |
| 130 | +--- | |
| 131 | + | |
| 132 | +## 4. Pipeline HTML → Markdown (le cœur du produit) | |
| 133 | + | |
| 134 | +Étapes, dans l'ordre, chacune testable isolément : | |
| 135 | + | |
| 136 | +1. **Parse** avec selectolax. Si le HTML est irrécupérable, fallback `lxml.html.fromstring` avec `recover=True`. | |
| 137 | +2. **Clean brut** : supprimer `script`, `style`, `noscript`, `iframe` (sauf embed vidéo → lien), `svg` décoratifs, éléments `hidden`/`display:none`/`aria-hidden`, commentaires, contrôles de formulaire. | |
| 138 | +3. **Suppression boilerplate** par heuristiques cumulées : tags sémantiques (`nav`, `footer`, `aside`, `header` sans `h1`, rôles ARIA), regex classes/ids (`cookie|consent|gdpr|banner|popup|modal|newsletter|share|social|sidebar|related|recommend|advert|promo|breadcrumb|comment…`) sauf si le bloc a l'air d'être le contenu, densité de liens (`liens/mots > 0.6` avec ≥ 4 liens). | |
| 139 | +4. **Scoring contenu principal** (`readability.py`) : score = log(1+n)·n·(1 − densité liens)²·bonus(paragraphes, ponctuation, `article|main|role=main`, classes contenu, titres). Remontée vers le parent tant qu'il n'ajoute pas trop de bruit. Sauté si `only_main_content=false`. | |
| 140 | +5. **Conversion DOM → MD** (`convert.py`) : titres normalisés (un seul `h1`), listes imbriquées, `pre/code` avec `language-*`, liens absolutisés, images (`data:` ignorées si `remove_base64_images`), tables (`tables.py` : thead absent, rowspan/colspan dupliqués, tables de mise en page aplaties), `dl`, `details/summary`, `figure/figcaption`, vidéo/audio → lien. | |
| 141 | +6. **Post-traitement** : lignes vides (>2 → 2), trim, NFC, zero-width, espaces multiples hors code. | |
| 142 | +7. **Citations** (option) : `[texte](url)` → `[texte][n]` + références en fin. | |
| 143 | + | |
| 144 | +Critères de qualité mesurés (tests golden, à constituer) : sur un corpus de 200 pages annotées, le MD doit contenir ≥ 95 % du texte principal attendu et ≤ 5 % de boilerplate. Toute PR touchant `html_to_md/` lance ce benchmark. | |
| 145 | + | |
| 146 | +--- | |
| 147 | + | |
| 148 | +## 5. Stratégie de fetch et escalade (`core/fetcher/strategy.py`) | |
| 149 | + | |
| 150 | +``` | |
| 151 | +auto: | |
| 152 | + 1. GET http_fast (curl_cffi, impersonate=chrome, redirections manuelles + re-check SSRF) | |
| 153 | + → si PDF → processors/pdf | |
| 154 | + → si HTML et heuristique "page vide" (texte visible < 200 chars ET scripts / racine SPA vide) → 2 | |
| 155 | + → si antibot.detect() positif (Cloudflare, DataDome, Akamai, PerimeterX, Kasada, Imperva, captchas) → 3 | |
| 156 | + → si 403/429/503 ou erreur TLS → 2 | |
| 157 | + → sinon OK | |
| 158 | + 2. browser (Playwright, images/fonts/media/pubs bloqués, domcontentloaded puis networkidle ≤ 15 s ou `wait_for`, actions) | |
| 159 | + → si antibot.detect() positif → 3 | |
| 160 | + 3. stealth (browser + patches JS + identité cohérente + proxy résidentiel si configuré) | |
| 161 | + → si toujours bloqué → ErrorInfo(code=BLOCKED, retryable=False, details.trace=[…]) | |
| 162 | +``` | |
| 163 | + | |
| 164 | +- Le mode résolu est mémorisé **par host** (cache 1 h, ne descend jamais). `actions`, `wait_for`, `screenshot` démarrent directement en navigateur. | |
| 165 | +- Pool navigateur : N contexts (défaut = CPU, 2..8), recyclés toutes les 50 pages ou 10 min. Un context = une identité cohérente (UA, viewport, timezone, locale, Accept-Language). | |
| 166 | +- Interception réseau : `image`, `font`, `media`, `stylesheet` + domaines de `antibot/blocklist.txt`, sauf si `screenshot` demandé. | |
| 167 | +- Timeouts : connect 10 s, lecture `timeout_ms`, render `timeout_ms`, téléchargement coupé à `max_size_mb` (slowloris → TIMEOUT_TTFB). | |
| 168 | + | |
| 169 | +--- | |
| 170 | + | |
| 171 | +## 6. Modules fonctionnels | |
| 172 | + | |
| 173 | +### 6.1 Crawl (`worker/crawl.py`) | |
| 174 | +- Frontier priorisée : BFS par défaut ; `strategy: bfs|dfs|best_first` (score BM25 URL/`search`). Amorçage par sitemaps sauf `ignore_sitemap`. | |
| 175 | +- Options : `max_depth`, `max_pages`, `include_paths[]`, `exclude_paths[]` (globs sur path/URL), `allow_subdomains`, `allow_external_links`, `ignore_sitemap`, `respect_robots` (défaut true), `delay_ms`, `concurrency`, `max_duration_s`. | |
| 176 | +- Dédup : normalisation (host minuscule, fragment, tri des params, retrait `utm_*|fbclid|gclid|ref|mc_*…`), puis simhash du MD (distance ≤ 3 → `skipped`). | |
| 177 | +- Chaque page est persistée dès qu'elle est prête ; SSE `page`/`progress`/`done`. | |
| 178 | +- Reprise : frontier sauvegardée en base toutes les 5 s ; jobs `running` au démarrage → `queued` et repris. | |
| 179 | + | |
| 180 | +### 6.2 Map (`map/`) | |
| 181 | +- Sources en parallèle : `robots.txt` → sitemaps (récursif, gzip, index) + crawl shallow http depth 2 (concurrence 16). Common Crawl : à venir. | |
| 182 | +- Sortie : URLs dédupliquées avec `sources[]`, `lastmod`, `depth`, `title` si `include_titles`, `score` si `search` (BM25 URL+title, plafond de collecte ×10 avant tri). | |
| 183 | + | |
| 184 | +### 6.3 PDF (`processors/pdf/`) | |
| 185 | +- Détection : content-type, magic `%PDF`. Extraction `pymupdf` `sort=True`, titres inférés (taille > médiane×1.2 → `##`, ×1.5 → `#`, gras court → `**`), `find_tables()` → MD, scan (< 50 car./page) → OCR rapidocr si installé, lignes coupées rejointes. Limites `max_pages_pdf` (500), `max_size_mb` (50). | |
| 186 | + | |
| 187 | +### 6.4 Extraction structurée (`extract/`) | |
| 188 | +- **CSS** : `{ champ: { selector, attr: text|href|src|html|…, type: str|int|float|date|url|list|bool, multiple } }`. Coercition typée, erreurs par champ. | |
| 189 | +- **LLM** : JSON Schema + `prompt` → MD tronqué par chunks pertinents (BM25-light sur le prompt) → sortie structurée native (response_format / tool use) → validation JSON Schema légère → ≤ 2 corrections. | |
| 190 | +- **Multi-pages** : `urls[]` ou `pattern` (map + glob) → `merge.py` (dédup entités par `merge_key`). Sync ≤ 5 URLs, sinon job. | |
| 191 | + | |
| 192 | +### 6.5 Chunker (`processors/chunker/`) | |
| 193 | +- `by_heading` : sections h1-h6, fusion des petites (< `min_tokens`), split des grandes (> `max_tokens`) sur frontières de phrases avec overlap. `by_tokens` : `size_tokens` (512) + `overlap_tokens` (64). Chaque chunk : `index`, `heading_path`, `token_count`, `char_range`, `url`. | |
| 194 | + | |
| 195 | +### 6.6 Agent — phase 7, non commencé | |
| 196 | +Entrée `{ url, objective, max_steps=30, schema?, allowed_domains?, budget_tokens? }`. Boucle observe (accessibility tree + Set-of-Marks) → décide (LLM → action JSON) → agit (Playwright, vérification post-action) → vérifie (planner, `done` validé contre `schema`). Tools `goto, click, type, scroll, wait, extract, back, done, ask_user`. Garde-fous : pas de soumission de formulaires `password|card|cvv|iban` sans whitelist, domaines autorisés, `max_steps`/budget, journal rejouable. | |
| 197 | + | |
| 198 | +--- | |
| 199 | + | |
| 200 | +## 7. API (`/v1`, OpenAPI sur `/docs` via Scalar) | |
| 201 | + | |
| 202 | +Auth `Authorization: Bearer <clé>` (ou `X-API-Key`, ou `?api_key=` pour les SSE). Sans `TRAWLS_REQUIRE_AUTH`, l'auth est désactivée ; `TRAWLS_ADMIN_KEY` donne accès à `/keys` et `/requests`. | |
| 203 | + | |
| 204 | +| Méthode | Route | Corps | Réponse | | |
| 205 | +|---|---|---|---| | |
| 206 | +| POST | `/scrape` | `{ url, ...ScrapeOptions }` | `PageResult` (sync) | | |
| 207 | +| POST | `/crawl` | `{ url, crawl: CrawlOptions, scrape: ScrapeOptions, webhook? }` | 202 `{ job_id, status, url }` | | |
| 208 | +| GET | `/crawl/{job_id}` | `?cursor&limit&status` | `JobSummary + pages[] + next_cursor` | | |
| 209 | +| GET | `/crawl/{job_id}/stream` | | SSE : `status`, `page`, `progress`, `done`, `error`, `ping` | | |
| 210 | +| DELETE | `/crawl/{job_id}` | | annulation | | |
| 211 | +| POST | `/map` | `{ url, search?, include_subdomains?, limit?, include_titles?, ignore_sitemap? }` | `{ url, count, urls[], took_ms }` | | |
| 212 | +| POST | `/extract` | `{ urls[] \| pattern, mode: css\|llm, css? \| schema?, prompt?, merge_key?, limit? }` | sync si ≤ 5 URLs `{ job_id, data, per_url[] }`, sinon 202 | | |
| 213 | +| POST | `/batch/scrape` | `{ urls[], concurrency?, ...ScrapeOptions }` | 202 `{ job_id }` | | |
| 214 | +| GET | `/jobs`, `/jobs/{id}`, `/jobs/{id}/stream`, DELETE `/jobs/{id}` | | statut générique | | |
| 215 | +| GET | `/jobs/{id}/export?format=jsonl\|zip\|md` | | export | | |
| 216 | +| GET | `/usage` | | requêtes/crédits par jour | | |
| 217 | +| GET/POST/DELETE | `/keys` | admin | clés argon2, valeur affichée une fois | | |
| 218 | +| GET | `/system`, `/blobs/{name}` | | diagnostic, screenshots | | |
| 219 | + | |
| 220 | +Conventions : erreurs `{ error: { code, message, retryable, details? } }` ; pagination par cursor ; idempotence via `Idempotency-Key` ; webhooks signés HMAC-SHA256 `X-Trawls-Signature` (3 essais, pas de redirection). Santé : `/healthz`, `/readyz` (db, browser, worker), `/metrics` Prometheus. | |
| 221 | + | |
| 222 | +Cycle de vie d'un job : `queued → running → completed | failed | cancelled` (`paused` réservé à l'agent). Résultats conservés `RESULT_TTL_DAYS` (7). | |
| 223 | + | |
| 224 | +--- | |
| 225 | + | |
| 226 | +## 8. Taxonomie des erreurs (`models.ErrorCode`) | |
| 227 | + | |
| 228 | +| Code | Récupérable | Déclencheur | | |
| 229 | +|---|---|---| | |
| 230 | +| `TIMEOUT_DNS` / `TIMEOUT_CONNECT` / `TIMEOUT_TTFB` / `TIMEOUT_RENDER` | oui | par étape | | |
| 231 | +| `HTTP_4XX` | non (408 → HTTP_5XX, 429 → RATE_LIMITED) | | | |
| 232 | +| `HTTP_5XX` | oui | | | |
| 233 | +| `RATE_LIMITED` | oui (respecter `Retry-After`) | 429 | | |
| 234 | +| `BLOCKED` | non | antibot après escalade complète (`details.trace`) | | |
| 235 | +| `ROBOTS_DISALLOWED` | non | | | |
| 236 | +| `TOO_LARGE` | non | > `max_size_mb` | | |
| 237 | +| `UNSUPPORTED_CONTENT` | non | binaire inconnu | | |
| 238 | +| `PARSE_FAILED` | non | HTML/PDF irrécupérable | | |
| 239 | +| `SSL_ERROR` | non (option `verify_ssl=false`) | | | |
| 240 | +| `SSRF_REFUSED` / `INVALID_URL` | non | garde sécurité | | |
| 241 | +| `CIRCUIT_OPEN` | oui (plus tard) | breaker host ouvert | | |
| 242 | +| `BUDGET_EXCEEDED` | non | job | | |
| 243 | +| `LLM_INVALID_OUTPUT` | oui (≤ 2) | extraction | | |
| 244 | +| `NETWORK` / `INTERNAL` / `CANCELLED` | NETWORK oui | filets | | |
| 245 | + | |
| 246 | +Retry : max 3, backoff `1s × 2^n + jitter(0-500ms)`, uniquement si `retryable`. Breaker : 5 échecs consécutifs sur un host → ouvert 60 s, half-open ensuite. | |
| 247 | + | |
| 248 | +--- | |
| 249 | + | |
| 250 | +## 9. Tests et qualité | |
| 251 | + | |
| 252 | +- `tests/unit/` : html_to_md (contenu principal, boilerplate, tables rowspan, listes, code, citations, HTML cassé), dedup/simhash, SSRF, détection anti-bot, chunker, extraction CSS, métadonnées. `pytest -q`. | |
| 253 | +- À constituer : `tests/fixtures_server/` (aiohttp : boucles de redirection, HTML tronqué, encodage faux, SPA vide, slowloris, faux challenge Cloudflare, PDF scanné/corrompu, doublons), golden MD `tests/golden/<site>/{input.html, expected.md}` (`make golden-update`), benchmarks vs crawl4ai/Firecrawl → `BENCHMARKS.md`. | |
| 254 | +- `ruff check`, `ruff format --check` bloquants ; `mypy --strict` objectif. | |
| 255 | + | |
| 256 | +--- | |
| 257 | + | |
| 258 | +## 10. Interface web (`trawls/web/static/`) | |
| 259 | + | |
| 260 | +Dark par défaut, fond neutre profond, accent ambre unique, `Inter` + `JetBrains Mono`, densité élevée. Routes servies par FastAPI : `/playground` (URL, options, Run ⌘↵, onglets Markdown/brut/JSON/HTML/Links/Screenshot/Chunks/Metadata, snippets cURL/Python/TS, historique local 20), `/crawls` (formulaire ou JSON brut, table des jobs, détail live SSE, arborescence des URLs, filtre, export ZIP/JSONL/MD), `/map` (liste par tranches, filtre instantané, groupement par path, CSV/JSON, « Crawler la sélection » → batch), `/extract` (builder de schéma ↔ JSON, CSS/LLM, résultat par URL + fusion), `/keys` (système, usage, clés, journal des requêtes), `/docs` (Scalar). Clé API obfusquée en localStorage uniquement. | |
| 261 | + | |
| 262 | +--- | |
| 263 | + | |
| 264 | +## 11. Déploiement et config | |
| 265 | + | |
| 266 | +- **MacLustr** : `deploy/trawls.manifest.json` → `M1M32:~/dispatch/apps/trawls.json` ; `mld stage ~/Desktop/Projets/apps-web/trawls trawls && mld deploy trawls`. PM2 `trawls-api` (uvicorn, port 8180) + `trawls-ngrok` (`www.trawls.dev`). Hook post_sync : venv uv 3.12 + `pip install -e .` + `playwright install chromium`. | |
| 267 | +- Docker : `docker-compose.yml` (api, worker optionnel, ollama sous profil `llm`). `make up`. | |
| 268 | +- Config env `TRAWLS_*` : `PORT`, `PUBLIC_URL`, `DATA_DIR`, `REQUIRE_AUTH`, `ADMIN_KEY`, `BROWSER_POOL_SIZE`, `MAX_CONCURRENCY_PER_HOST`, `MAX_CONCURRENCY_GLOBAL`, `MAX_SIZE_MB`, `PROXY_URLS`, `LLM_PROVIDER|LLM_BASE_URL|LLM_API_KEY|LLM_MODEL`, `RESULT_TTL_DAYS`, `RATE_LIMIT_PER_MINUTE`, `WORKER_CONCURRENCY`, `ALLOW_PRIVATE_TARGETS` (jamais en prod), `EMBEDDED_WORKER`. | |
| 269 | +- Healthchecks `/healthz`, `/readyz` ; métriques `/metrics`. | |
| 270 | + | |
| 271 | +--- | |
| 272 | + | |
| 273 | +## 12. Sécurité | |
| 274 | + | |
| 275 | +- SSRF (`core/security.py`) : refus IP privées/loopback/link-local/multicast, schémas non http(s), `localhost`/`.internal`/metadata ; DNS résolu **avant** la requête et re-vérifié à chaque redirection (http et navigateur). | |
| 276 | +- Limites strictes de taille et de temps ; pas d'`eval` serveur — `actions.evaluate` s'exécute uniquement dans le sandbox Playwright. | |
| 277 | +- Clés API hashées argon2, préfixe indexé, affichées une seule fois. Rate limit token-bucket par clé/IP. Quota journalier par clé. | |
| 278 | +- Webhooks : signature HMAC, 3 essais, pas de redirection suivie. | |
| 279 | + | |
| 280 | +--- | |
| 281 | + | |
| 282 | +## 13. Conventions de développement | |
| 283 | + | |
| 284 | +- Commits conventionnels (`feat`, `fix`, `perf`, `refactor`, `test`, `docs`). Remote `origin` = spbgit (`gitsrv:~/srv/git/trawls.git`). | |
| 285 | +- Une PR = un module ou une fonctionnalité ; description avec « Comment tester ». | |
| 286 | +- Chaque module a une docstring de tête expliquant son rôle et ses invariants. | |
| 287 | +- Pas de `print` hors CLI : `structlog` JSON, `job_id`/`url` dans le contexte. | |
| 288 | +- Pas de `except Exception: pass` dans le chemin chaud. Toute exception attrapée est convertie en `ErrorInfo` typé. | |
| 289 | +- Les options ont des valeurs par défaut sûres ; `verify_ssl=false`, `respect_robots=false`, `allow_private_targets` sont loguées explicitement. | |
| 290 | + | |
| 291 | +--- | |
| 292 | + | |
| 293 | +## 14. Feuille de route et définition de « terminé » | |
| 294 | + | |
| 295 | +| Phase | Contenu | Terminé quand | État | | |
| 296 | +|---|---|---|---| | |
| 297 | +| 1 | `fetcher` (http + browser + auto), `resilience` | 100 % des fixtures produisent un `PageResult` sans exception | livré (fixtures server à écrire) | | |
| 298 | +| 2 | `html_to_md` complet + golden tests | ≥ 95 % de rappel texte, ≤ 5 % boilerplate sur le corpus golden | livré (corpus golden à constituer) | | |
| 299 | +| 3 | API `/scrape`, `/crawl`, jobs, SSE, CLI | Crawl de 1 000 pages sans fuite mémoire, reprise après kill du worker | livré (reprise via frontier SQLite) | | |
| 300 | +| 4 | `map`, `pdf` (texte + tables + OCR) | Map 5 000 URLs < 10 s ; 20 PDF de référence convertis | livré | | |
| 301 | +| 5 | `extract` CSS + LLM + merge, `chunker` | 10 schémas de référence, 0 JSON invalide en sortie | livré | | |
| 302 | +| 6 | UI web : playground, crawls, map, extract, keys | Parité fonctionnelle Firecrawl | livré (vanilla, Next.js cible) | | |
| 303 | +| 7 | `agent` + console | 10 missions de référence réussies | à faire | | |
| 304 | +| 8 | SDK Python/TS, docs, benchmarks publiés | `pip install trawls`, `BENCHMARKS.md` | à faire | | |
| 305 | + | |
| 306 | +Ne pas commencer une phase tant que la précédente n'est pas « terminée » selon ce tableau. | |
added
Dockerfile
+10 −0
@@ -0,0 +1,10 @@ | ||
| 1 | +FROM mcr.microsoft.com/playwright/python:v1.47.0-jammy | |
| 2 | +WORKDIR /app | |
| 3 | +ENV PYTHONUNBUFFERED=1 TRAWLS_HOST=0.0.0.0 TRAWLS_PORT=8180 TRAWLS_DATA_DIR=/data | |
| 4 | +COPY pyproject.toml README.md ./ | |
| 5 | +COPY trawls ./trawls | |
| 6 | +RUN pip install --no-cache-dir -e . && playwright install chromium | |
| 7 | +VOLUME ["/data"] | |
| 8 | +EXPOSE 8180 | |
| 9 | +HEALTHCHECK --interval=30s --timeout=5s CMD curl -fsS http://localhost:8180/healthz || exit 1 | |
| 10 | +CMD ["trawls", "serve"] | |
added
Makefile
+29 −0
@@ -0,0 +1,29 @@ | ||
| 1 | +.PHONY: install dev serve test lint fmt up down golden-update | |
| 2 | + | |
| 3 | +install: | |
| 4 | + uv venv --python 3.12 || true | |
| 5 | + . .venv/bin/activate && uv pip install -e ".[dev]" && playwright install chromium | |
| 6 | + | |
| 7 | +serve: | |
| 8 | + . .venv/bin/activate && trawls serve | |
| 9 | + | |
| 10 | +dev: | |
| 11 | + . .venv/bin/activate && TRAWLS_LOG_LEVEL=DEBUG trawls serve --reload | |
| 12 | + | |
| 13 | +test: | |
| 14 | + . .venv/bin/activate && pytest -q | |
| 15 | + | |
| 16 | +lint: | |
| 17 | + . .venv/bin/activate && ruff check . && ruff format --check . | |
| 18 | + | |
| 19 | +fmt: | |
| 20 | + . .venv/bin/activate && ruff format . && ruff check --fix . | |
| 21 | + | |
| 22 | +up: | |
| 23 | + docker compose up -d --build | |
| 24 | + | |
| 25 | +down: | |
| 26 | + docker compose down | |
| 27 | + | |
| 28 | +golden-update: | |
| 29 | + . .venv/bin/activate && TRAWLS_GOLDEN_UPDATE=1 pytest -q tests/unit/test_html_to_md.py | |
added
README.md
+88 −0
@@ -0,0 +1,88 @@ | ||
| 1 | +# Trawls — www.trawls.dev | |
| 2 | + | |
| 3 | +Moteur de crawling, d'extraction et de navigation web, construit from scratch. Un chalut (*trawl*) ratisse le fond et remonte tout : Trawls cartographie et extrait n'importe quel site et renvoie du **Markdown propre, LLM-ready**. | |
| 4 | + | |
| 5 | +- **Ne crashe jamais** : une page qui échoue produit un `PageResult` `status=failed` avec une erreur typée. Le crawl continue. | |
| 6 | +- **Escalade automatique** : HTTP pur (curl_cffi, empreinte TLS Chrome) → navigateur (Playwright, pool de contexts) → stealth (patches JS, identité cohérente). Le mode résolu est mémorisé par host. | |
| 7 | +- **Sortie LLM-ready** : pipeline HTML → Markdown maison (nettoyage, suppression du boilerplate, scoring du contenu principal, tables avec rowspan/colspan, citations), métadonnées fusionnées (head, OpenGraph, JSON-LD), chunks par titres ou par tokens. | |
| 8 | +- **PDF** : texte avec ordre de lecture, titres inférés, tables, détection de scan (OCR optionnel). | |
| 9 | +- **Extraction structurée** : CSS typé (déterministe) ou LLM (JSON Schema, validation, corrections) avec fusion multi-pages. | |
| 10 | +- **Self-host first** : SQLite, worker embarqué, aucun service externe requis. LLM local via Ollama par défaut. | |
| 11 | +- **Observable** : structlog JSON, `/healthz`, `/readyz`, `/metrics` Prometheus, trace d'escalade par page. | |
| 12 | + | |
| 13 | +## Démarrage | |
| 14 | + | |
| 15 | +```bash | |
| 16 | +uv venv --python 3.12 && source .venv/bin/activate | |
| 17 | +uv pip install -e ".[dev]" | |
| 18 | +playwright install chromium | |
| 19 | +trawls serve # http://localhost:8180 (UI + API + worker) | |
| 20 | +``` | |
| 21 | + | |
| 22 | +```bash | |
| 23 | +trawls scrape https://fr.wikipedia.org/wiki/Chalut | |
| 24 | +trawls map https://docs.python.org --search "asyncio" | |
| 25 | +trawls crawl https://exemple.com --max-pages 50 -o pages.jsonl | |
| 26 | +``` | |
| 27 | + | |
| 28 | +Docker : `make up` (api + web sur http://localhost:8180). | |
| 29 | + | |
| 30 | +## API (`/v1`, OpenAPI sur `/docs`) | |
| 31 | + | |
| 32 | +| Méthode | Route | Rôle | | |
| 33 | +|---|---|---| | |
| 34 | +| POST | `/scrape` | une page, synchrone → `PageResult` | | |
| 35 | +| POST | `/crawl` → GET `/crawl/{id}` · `/crawl/{id}/stream` (SSE) · DELETE | crawl asynchrone, pages streamées | | |
| 36 | +| POST | `/map` | liste d'URLs (sitemaps + crawl shallow), tri BM25 si `search` | | |
| 37 | +| POST | `/extract` | CSS ou LLM sur `urls[]` / `pattern` ; sync ≤ 5 URLs | | |
| 38 | +| POST | `/batch/scrape` | N URLs en job | | |
| 39 | +| GET | `/jobs`, `/jobs/{id}`, `/jobs/{id}/stream`, `/jobs/{id}/export?format=zip\|jsonl\|md` | jobs génériques | | |
| 40 | +| GET | `/usage` · `/keys` (admin) | crédits, clés API (argon2, affichées une fois) | | |
| 41 | + | |
| 42 | +Erreurs : `{ "error": { "code", "message", "retryable", "details?" } }`. Idempotence : header `Idempotency-Key`. Webhooks signés `X-Trawls-Signature` (HMAC-SHA256). | |
| 43 | + | |
| 44 | +```bash | |
| 45 | +curl -s https://www.trawls.dev/v1/scrape -H 'Content-Type: application/json' \ | |
| 46 | + -d '{"url":"https://example.com","formats":["markdown","links"]}' | jq .markdown | |
| 47 | +``` | |
| 48 | + | |
| 49 | +## Configuration (préfixe `TRAWLS_`) | |
| 50 | + | |
| 51 | +`PORT` (8180), `PUBLIC_URL`, `DATA_DIR`, `REQUIRE_AUTH`, `ADMIN_KEY`, `BROWSER_POOL_SIZE`, `MAX_CONCURRENCY_PER_HOST`, `MAX_SIZE_MB`, `PROXY_URLS`, `LLM_PROVIDER` (`openai`|`anthropic`|`none`), `LLM_BASE_URL` (Ollama par défaut), `LLM_API_KEY`, `LLM_MODEL`, `RESULT_TTL_DAYS`, `RATE_LIMIT_PER_MINUTE`, `ALLOW_PRIVATE_TARGETS` (jamais en prod). Voir `.env.example`. | |
| 52 | + | |
| 53 | +## Architecture | |
| 54 | + | |
| 55 | +``` | |
| 56 | +trawls/ | |
| 57 | + config.py settings pydantic | |
| 58 | + models/ contrat Pydantic v2 (ScrapeOptions, PageResult, ErrorInfo…) | |
| 59 | + core/fetcher/ http_fast (curl_cffi), browser (pool Playwright), strategy (escalade) | |
| 60 | + core/antibot/ detect, identity, stealth, blocklist | |
| 61 | + core/scheduler/ dedup (normalisation + simhash), robots, politeness | |
| 62 | + core/resilience/ retry, breaker, budget | |
| 63 | + core/security.py garde SSRF (DNS résolu avant, re-vérifié après redirection) | |
| 64 | + core/scrape.py pipeline d'une page — ne lève jamais | |
| 65 | + processors/ html_to_md (clean, readability, convert, tables, citations), pdf, structured, chunker, links | |
| 66 | + extract/ css, llm, merge | |
| 67 | + map/ sitemaps + crawl shallow + BM25 | |
| 68 | + llm/ client unique (Anthropic, OpenAI-compatible) | |
| 69 | + worker/ store SQLite, moteur de jobs (crawl/batch/extract), SSE | |
| 70 | + api/ FastAPI /v1, auth, UI | |
| 71 | + web/static/ UI (playground, crawls, map, extract, keys) | |
| 72 | + cli.py | |
| 73 | +``` | |
| 74 | + | |
| 75 | +## Tests | |
| 76 | + | |
| 77 | +```bash | |
| 78 | +pytest -q # unitaires (html_to_md, dedup, ssrf, detect, chunker, css) | |
| 79 | +ruff check . && ruff format --check . | |
| 80 | +``` | |
| 81 | + | |
| 82 | +## Déploiement MacLustr | |
| 83 | + | |
| 84 | +Déployée via `mld` (manifeste `trawls.json`) : PM2 `trawls-api` + `trawls-ngrok` → https://www.trawls.dev. Voir `deploy/trawls.manifest.json`. | |
| 85 | + | |
| 86 | +## Feuille de route | |
| 87 | + | |
| 88 | +Phases 1–6 livrées (fetch + escalade, Markdown, API/jobs/SSE/CLI, map + PDF, extract + chunker, UI). Phase 7 (agent) et 8 (SDK, benchmarks) à venir — voir `CLAUDE.md`. | |
added
deploy/trawls.manifest.json
+82 −0
@@ -0,0 +1,82 @@ | ||
| 1 | +{ | |
| 2 | + "app": "trawls", | |
| 3 | + "label": "Trawls — crawl, extract, navigate", | |
| 4 | + "domain": "www.trawls.dev", | |
| 5 | + "port": 8180, | |
| 6 | + "health_path": "/healthz", | |
| 7 | + "dir": "~/apps/trawls", | |
| 8 | + "extra_paths": [], | |
| 9 | + "sync_excludes": [ | |
| 10 | + ".venv/", | |
| 11 | + "venv/", | |
| 12 | + "data/", | |
| 13 | + "__pycache__/", | |
| 14 | + ".pytest_cache/", | |
| 15 | + ".ruff_cache/", | |
| 16 | + ".mypy_cache/", | |
| 17 | + "*.egg-info/", | |
| 18 | + ".git/", | |
| 19 | + ".env" | |
| 20 | + ], | |
| 21 | + "requires": { | |
| 22 | + "runtimes": ["pm2", "ngrok", "uv-python@3.12", "uv"], | |
| 23 | + "ram_gb": 3, | |
| 24 | + "ports": [8180] | |
| 25 | + }, | |
| 26 | + "ram_mb_observed": 1500, | |
| 27 | + "size_mb": 5, | |
| 28 | + "placement": { | |
| 29 | + "pin": null, | |
| 30 | + "prefer": "M3U96a", | |
| 31 | + "avoid": ["M3U96b", "M1M32"], | |
| 32 | + "reason": "Playwright/Chromium : beaucoup de cœurs et RAM libre ; M3U96b réservé hfmarketdata" | |
| 33 | + }, | |
| 34 | + "processes": [ | |
| 35 | + { | |
| 36 | + "name": "trawls-api", | |
| 37 | + "manager": "pm2", | |
| 38 | + "script": "{{HOME}}/apps/trawls/.venv/bin/python", | |
| 39 | + "args": ["-m", "uvicorn", "trawls.api.main:app", "--host", "0.0.0.0", "--port", "8180", "--no-access-log"], | |
| 40 | + "interpreter": null, | |
| 41 | + "cwd": "{{HOME}}/apps/trawls", | |
| 42 | + "env": { | |
| 43 | + "TRAWLS_ENV": "production", | |
| 44 | + "TRAWLS_PORT": "8180", | |
| 45 | + "TRAWLS_PUBLIC_URL": "https://www.trawls.dev", | |
| 46 | + "TRAWLS_DATA_DIR": "{{HOME}}/apps/trawls/data", | |
| 47 | + "TRAWLS_BROWSER_POOL_SIZE": "6", | |
| 48 | + "TRAWLS_WORKER_CONCURRENCY": "4", | |
| 49 | + "TRAWLS_MAX_CONCURRENCY_GLOBAL": "24", | |
| 50 | + "TRAWLS_LLM_PROVIDER": "openai", | |
| 51 | + "TRAWLS_LLM_BASE_URL": "https://llm.maclustr.io/v1", | |
| 52 | + "TRAWLS_LLM_MODEL": "maclustr-general", | |
| 53 | + "TRAWLS_ADMIN_KEY": "trawls-admin-2026-Qx7mP3vL9kR2", | |
| 54 | + "TRAWLS_LOG_LEVEL": "INFO", | |
| 55 | + "PYTHONUNBUFFERED": "1", | |
| 56 | + "PLAYWRIGHT_BROWSERS_PATH": "{{HOME}}/Library/Caches/ms-playwright" | |
| 57 | + }, | |
| 58 | + "cron_restart": null, | |
| 59 | + "autorestart": true, | |
| 60 | + "max_memory_restart": "6G" | |
| 61 | + } | |
| 62 | + ], | |
| 63 | + "ngrok": { | |
| 64 | + "name": "trawls-ngrok", | |
| 65 | + "url": "www.trawls.dev", | |
| 66 | + "port": 8180 | |
| 67 | + }, | |
| 68 | + "launchd": [], | |
| 69 | + "env_overrides": {}, | |
| 70 | + "hooks": { | |
| 71 | + "post_sync": [ | |
| 72 | + "export PATH=\"$HOME/.local/bin:/opt/homebrew/bin:$PATH\"; test -x .venv/bin/python || uv venv --python 3.12 .venv && echo ' venv ok'", | |
| 73 | + "export PATH=\"$HOME/.local/bin:/opt/homebrew/bin:$PATH\"; uv pip install -q --python .venv/bin/python -e . && echo ' deps python ok'", | |
| 74 | + "PLAYWRIGHT_BROWSERS_PATH=$HOME/Library/Caches/ms-playwright .venv/bin/playwright install chromium 2>&1 | tail -1; echo ' chromium ok'", | |
| 75 | + "mkdir -p data/blobs && echo ' data ok'" | |
| 76 | + ], | |
| 77 | + "post_start": [] | |
| 78 | + }, | |
| 79 | + "ka_repo": false, | |
| 80 | + "source_node": null, | |
| 81 | + "rebuild": false | |
| 82 | +} | |
added
docker-compose.yml
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +services: | |
| 2 | + api: | |
| 3 | + build: . | |
| 4 | + ports: ["8180:8180"] | |
| 5 | + environment: | |
| 6 | + TRAWLS_PUBLIC_URL: http://localhost:8180 | |
| 7 | + TRAWLS_LLM_BASE_URL: http://ollama:11434/v1 | |
| 8 | + TRAWLS_LLM_MODEL: qwen2.5:7b | |
| 9 | + volumes: ["trawls-data:/data"] | |
| 10 | + restart: unless-stopped | |
| 11 | + worker: | |
| 12 | + build: . | |
| 13 | + command: ["trawls", "worker"] | |
| 14 | + environment: | |
| 15 | + TRAWLS_DATA_DIR: /data | |
| 16 | + volumes: ["trawls-data:/data"] | |
| 17 | + depends_on: [api] | |
| 18 | + deploy: { replicas: 0 } # `docker compose up --scale worker=2` pour des workers séparés (API avec TRAWLS_EMBEDDED_WORKER=0) | |
| 19 | + ollama: | |
| 20 | + image: ollama/ollama | |
| 21 | + volumes: ["ollama:/root/.ollama"] | |
| 22 | + profiles: ["llm"] | |
| 23 | +volumes: | |
| 24 | + trawls-data: {} | |
| 25 | + ollama: {} | |
added
pyproject.toml
+61 −0
@@ -0,0 +1,61 @@ | ||
| 1 | +[project] | |
| 2 | +name = "trawls" | |
| 3 | +version = "0.1.0" | |
| 4 | +description = "Trawls — moteur de crawling, d'extraction et de navigation web, LLM-ready." | |
| 5 | +readme = "README.md" | |
| 6 | +requires-python = ">=3.12" | |
| 7 | +license = { text = "MIT" } | |
| 8 | +authors = [{ name = "Simon-Pierre Boucher" }] | |
| 9 | +dependencies = [ | |
| 10 | + "fastapi>=0.115", | |
| 11 | + "uvicorn[standard]>=0.30", | |
| 12 | + "pydantic>=2.8", | |
| 13 | + "pydantic-settings>=2.4", | |
| 14 | + "curl_cffi>=0.7", | |
| 15 | + "httpx[http2]>=0.27", | |
| 16 | + "playwright>=1.47", | |
| 17 | + "selectolax>=0.3.21", | |
| 18 | + "lxml>=5.2", | |
| 19 | + "charset-normalizer>=3.3", | |
| 20 | + "pymupdf>=1.24", | |
| 21 | + "tiktoken>=0.7", | |
| 22 | + "structlog>=24.1", | |
| 23 | + "aiosqlite>=0.20", | |
| 24 | + "typer>=0.12", | |
| 25 | + "rich>=13.7", | |
| 26 | + "python-dateutil>=2.9", | |
| 27 | + "sse-starlette>=2.1", | |
| 28 | + "argon2-cffi>=23.1", | |
| 29 | + "orjson>=3.10", | |
| 30 | +] | |
| 31 | + | |
| 32 | +[project.optional-dependencies] | |
| 33 | +dev = ["pytest>=8", "pytest-asyncio>=0.23", "ruff>=0.5", "mypy>=1.10", "aiohttp>=3.9"] | |
| 34 | +ocr = ["rapidocr-onnxruntime>=1.3"] | |
| 35 | + | |
| 36 | +[project.scripts] | |
| 37 | +trawls = "trawls.cli:app" | |
| 38 | + | |
| 39 | +[build-system] | |
| 40 | +requires = ["hatchling"] | |
| 41 | +build-backend = "hatchling.build" | |
| 42 | + | |
| 43 | +[tool.hatch.build.targets.wheel] | |
| 44 | +packages = ["trawls"] | |
| 45 | + | |
| 46 | +[tool.ruff] | |
| 47 | +line-length = 110 | |
| 48 | +target-version = "py312" | |
| 49 | + | |
| 50 | +[tool.ruff.lint] | |
| 51 | +select = ["E", "F", "I", "UP", "B", "ASYNC"] | |
| 52 | +ignore = ["E501", "B008", "E402", "UP047", "ASYNC109", "ASYNC230"] | |
| 53 | + | |
| 54 | +[tool.mypy] | |
| 55 | +python_version = "3.12" | |
| 56 | +strict = true | |
| 57 | +ignore_missing_imports = true | |
| 58 | + | |
| 59 | +[tool.pytest.ini_options] | |
| 60 | +asyncio_mode = "auto" | |
| 61 | +testpaths = ["tests"] | |
added
tests/unit/test_core.py
+129 −0
@@ -0,0 +1,129 @@ | ||
| 1 | +import pytest | |
| 2 | + | |
| 3 | +from trawls.core.antibot.detect import detect | |
| 4 | +from trawls.core.scheduler.dedup import ContentDeduper, hamming, normalize_url, same_site, simhash | |
| 5 | +from trawls.core.security import SsrfError, check_url | |
| 6 | +from trawls.extract.css import extract_css | |
| 7 | +from trawls.models import ChunkOptions, CssField, ErrorCode | |
| 8 | +from trawls.processors.chunker import chunk | |
| 9 | +from trawls.processors.structured.metadata import extract_metadata | |
| 10 | + | |
| 11 | + | |
| 12 | +def test_normalize_url() -> None: | |
| 13 | + assert normalize_url("HTTP://Ex.COM:80/a/b/?utm_source=x&b=2&a=1#frag") == "http://ex.com/a/b?a=1&b=2" | |
| 14 | + assert normalize_url("https://ex.com") == "https://ex.com/" | |
| 15 | + assert normalize_url("/rel", "https://ex.com/dir/page") == "https://ex.com/rel" | |
| 16 | + assert normalize_url("javascript:void(0)") is None | |
| 17 | + assert normalize_url("mailto:a@b.c") is None | |
| 18 | + assert normalize_url("https://ex.com/a?fbclid=1&gclid=2") == "https://ex.com/a" | |
| 19 | + | |
| 20 | + | |
| 21 | +def test_same_site() -> None: | |
| 22 | + assert same_site("https://ex.com/a", "https://ex.com/b", False) | |
| 23 | + assert not same_site("https://ex.com/a", "https://blog.ex.com/b", False) | |
| 24 | + assert same_site("https://ex.com/a", "https://blog.ex.com/b", True) | |
| 25 | + assert not same_site("https://ex.com/a", "https://other.com/b", True) | |
| 26 | + | |
| 27 | + | |
| 28 | +def test_simhash_dedup() -> None: | |
| 29 | + a = "Le chalut est un filet remorqué par un navire. " * 20 | |
| 30 | + b = a.replace("navire", "bateau", 1) | |
| 31 | + c = "Texte complètement différent sur les oiseaux migrateurs et leurs routes. " * 20 | |
| 32 | + assert hamming(simhash(a), simhash(b)) <= 3 | |
| 33 | + assert hamming(simhash(a), simhash(c)) > 10 | |
| 34 | + d = ContentDeduper() | |
| 35 | + assert d.is_duplicate(a, "u1") is None | |
| 36 | + assert d.is_duplicate(b, "u2") == "u1" | |
| 37 | + assert d.is_duplicate(c, "u3") is None | |
| 38 | + | |
| 39 | + | |
| 40 | +@pytest.mark.asyncio | |
| 41 | +async def test_ssrf() -> None: | |
| 42 | + for bad in ( | |
| 43 | + "http://127.0.0.1/", | |
| 44 | + "http://localhost:8080/x", | |
| 45 | + "http://169.254.169.254/latest", | |
| 46 | + "http://10.0.0.1/", | |
| 47 | + "http://192.168.2.1/", | |
| 48 | + "ftp://ex.com/", | |
| 49 | + "http://[::1]/", | |
| 50 | + "http://0.0.0.0/", | |
| 51 | + ): | |
| 52 | + with pytest.raises(SsrfError): | |
| 53 | + await check_url(bad) | |
| 54 | + assert await check_url("https://1.1.1.1/") == "https://1.1.1.1/" | |
| 55 | + | |
| 56 | + | |
| 57 | +def test_detect() -> None: | |
| 58 | + assert ( | |
| 59 | + detect( | |
| 60 | + "<html><title>Just a moment...</title><div id='cf-browser-verification'></div></html>", 503 | |
| 61 | + ).kind | |
| 62 | + == "cloudflare" | |
| 63 | + ) | |
| 64 | + assert ( | |
| 65 | + detect('<html><body><div id="root"></div><script src="/app.js"></script></body></html>', 200).kind | |
| 66 | + == "spa" | |
| 67 | + ) | |
| 68 | + ok = detect("<html><body><p>" + "Du contenu normal et long. " * 50 + "</p></body></html>", 200) | |
| 69 | + assert ok.kind is None | |
| 70 | + assert detect("<html><body><h1>Access Denied</h1></body></html>", 403).blocked | |
| 71 | + | |
| 72 | + | |
| 73 | +def test_chunker() -> None: | |
| 74 | + md = ( | |
| 75 | + "# Titre\n\nIntro courte.\n\n## Section A\n\n" | |
| 76 | + + ("Phrase de la section A. " * 80) | |
| 77 | + + "\n\n## Section B\n\nPetite.\n" | |
| 78 | + ) | |
| 79 | + cs = chunk( | |
| 80 | + md, | |
| 81 | + "https://ex.com", | |
| 82 | + ChunkOptions(strategy="by_heading", max_tokens=200, min_tokens=20, overlap_tokens=20), | |
| 83 | + ) | |
| 84 | + assert len(cs) >= 3 | |
| 85 | + assert all(c.token_count <= 260 for c in cs) | |
| 86 | + assert any("Section A" in c.heading_path for c in cs) | |
| 87 | + assert cs[0].index == 0 and cs[-1].index == len(cs) - 1 | |
| 88 | + ct = chunk("Mot. " * 2000, "u", ChunkOptions(strategy="by_tokens", size_tokens=100, overlap_tokens=10)) | |
| 89 | + assert len(ct) > 10 and all(c.token_count <= 130 for c in ct) | |
| 90 | + | |
| 91 | + | |
| 92 | +def test_css_extract() -> None: | |
| 93 | + html = "<html><body><h1> Produit X </h1><span class='price'>1 234,50 $</span><a class='more' href='/p/1'>+</a><ul><li>a</li><li>b</li></ul><time datetime='2026-09-05'>5 sept</time></body></html>" | |
| 94 | + data, errs = extract_css( | |
| 95 | + html, | |
| 96 | + { | |
| 97 | + "title": CssField(selector="h1"), | |
| 98 | + "price": CssField(selector=".price", type="float"), | |
| 99 | + "url": CssField(selector="a.more", attr="href", type="url"), | |
| 100 | + "items": CssField(selector="li", multiple=True), | |
| 101 | + "date": CssField(selector="time", attr="datetime", type="date"), | |
| 102 | + "missing": CssField(selector=".nope", type="int"), | |
| 103 | + }, | |
| 104 | + "https://ex.com/x", | |
| 105 | + ) | |
| 106 | + assert data["title"] == "Produit X" | |
| 107 | + assert data["price"] == 1234.5 | |
| 108 | + assert data["url"] == "https://ex.com/p/1" | |
| 109 | + assert data["items"] == ["a", "b"] | |
| 110 | + assert data["date"].startswith("2026-09-05") | |
| 111 | + assert data["missing"] is None and "missing" in errs | |
| 112 | + | |
| 113 | + | |
| 114 | +def test_metadata() -> None: | |
| 115 | + html = """<html lang="fr-CA"><head><title>T</title><meta property="og:title" content="OG T"><meta name="description" content="D"> | |
| 116 | + <link rel="canonical" href="/canon"><script type="application/ld+json">{"@type":"Article","headline":"H","author":{"name":"Simon"},"datePublished":"2026-01-02"}</script></head><body></body></html>""" | |
| 117 | + m = extract_metadata(html, "https://ex.com/p") | |
| 118 | + assert m.title == "OG T" and m.description == "D" and m.language == "fr" | |
| 119 | + assert m.canonical_url == "https://ex.com/canon" | |
| 120 | + assert m.author == "Simon" and m.published_at == "2026-01-02" | |
| 121 | + assert m.jsonld and m.jsonld[0]["@type"] == "Article" | |
| 122 | + | |
| 123 | + | |
| 124 | +def test_error_codes_retryable() -> None: | |
| 125 | + from trawls.models import ErrorInfo | |
| 126 | + | |
| 127 | + assert ErrorInfo.make(ErrorCode.HTTP_5XX, "x").retryable | |
| 128 | + assert not ErrorInfo.make(ErrorCode.BLOCKED, "x").retryable | |
| 129 | + assert not ErrorInfo.make(ErrorCode.ROBOTS_DISALLOWED, "x").retryable | |
added
tests/unit/test_html_to_md.py
+92 −0
@@ -0,0 +1,92 @@ | ||
| 1 | +from trawls.processors.html_to_md import html_to_markdown | |
| 2 | + | |
| 3 | +ARTICLE = """ | |
| 4 | +<html lang="fr"><head><title>Titre</title><style>.x{}</style><script>var a=1;</script></head> | |
| 5 | +<body> | |
| 6 | +<nav><a href="/">Accueil</a><a href="/blog">Blog</a><a href="/contact">Contact</a><a href="/a">A</a></nav> | |
| 7 | +<div class="cookie-banner">Nous utilisons des cookies. <a href="#">Accepter</a></div> | |
| 8 | +<main> | |
| 9 | +<article> | |
| 10 | +<h1>Le chalut de fond</h1> | |
| 11 | +<p>Le <strong>chalut</strong> est un filet <em>remorqué</em> par un navire. Il ratisse le fond et remonte tout ce qui s'y trouve, ce qui en fait un engin très efficace mais controversé.</p> | |
| 12 | +<h2>Fonctionnement</h2> | |
| 13 | +<p>Le filet est maintenu ouvert par des panneaux. Voir <a href="/details">les détails</a>.</p> | |
| 14 | +<ul><li>Panneaux divergents</li><li>Bourrelet<ul><li>Diabolos</li></ul></li></ul> | |
| 15 | +<table><tr><th>Type</th><th>Profondeur</th></tr><tr><td>Fond</td><td>200 m</td></tr><tr><td rowspan="2">Pélagique</td><td>50 m</td></tr><tr><td>100 m</td></tr></table> | |
| 16 | +<pre><code class="language-python">print("ok")</code></pre> | |
| 17 | +<img src="/img/chalut.png" alt="Un chalut"> | |
| 18 | +<img src="data:image/png;base64,AAAA" alt="b64"> | |
| 19 | +</article> | |
| 20 | +</main> | |
| 21 | +<aside class="sidebar"><a href="/1">Lien 1</a><a href="/2">Lien 2</a><a href="/3">Lien 3</a><a href="/4">Lien 4</a></aside> | |
| 22 | +<footer>© 2026 Exemple — <a href="/mentions">Mentions légales</a></footer> | |
| 23 | +</body></html> | |
| 24 | +""" | |
| 25 | + | |
| 26 | + | |
| 27 | +def test_main_content_and_boilerplate() -> None: | |
| 28 | + r = html_to_markdown(ARTICLE, "https://ex.com/a/b") | |
| 29 | + md = r.markdown | |
| 30 | + assert md.startswith("# Le chalut de fond") | |
| 31 | + assert "**chalut**" in md and "*remorqué*" in md | |
| 32 | + assert "## Fonctionnement" in md | |
| 33 | + assert "[les détails](https://ex.com/details)" in md | |
| 34 | + assert "- Panneaux divergents" in md and " - Diabolos" in md | |
| 35 | + assert "| Type | Profondeur |" in md and "| Pélagique | 100 m |" in md | |
| 36 | + assert '```python\nprint("ok")\n```' in md | |
| 37 | + assert "" in md | |
| 38 | + assert "b64" not in md | |
| 39 | + for bad in ("Accueil", "cookies", "Mentions légales", "Lien 1"): | |
| 40 | + assert bad not in md, bad | |
| 41 | + | |
| 42 | + | |
| 43 | +def test_full_page_keeps_nav_text() -> None: | |
| 44 | + r = html_to_markdown(ARTICLE, "https://ex.com", only_main_content=False) | |
| 45 | + assert "Accueil" in r.markdown and "Le chalut de fond" in r.markdown | |
| 46 | + | |
| 47 | + | |
| 48 | +def test_broken_html_never_raises() -> None: | |
| 49 | + for html in ( | |
| 50 | + "", | |
| 51 | + "<div><p>ouvert", | |
| 52 | + "<<<>>>", | |
| 53 | + "<html><body><table><tr><td>x</table>", | |
| 54 | + "\x00\x01garbage", | |
| 55 | + "<p>" * 5000, | |
| 56 | + ): | |
| 57 | + r = html_to_markdown(html, "https://ex.com") | |
| 58 | + assert isinstance(r.markdown, str) | |
| 59 | + | |
| 60 | + | |
| 61 | +def test_citations() -> None: | |
| 62 | + r = html_to_markdown( | |
| 63 | + "<article><p>Voir <a href='https://a.com'>A</a> et <a href='https://b.com'>B</a> puis <a href='https://a.com'>A encore</a>. Un paragraphe assez long pour que le scoring le garde comme contenu principal, avec plusieurs phrases, des virgules, et du texte.</p></article>", | |
| 64 | + "https://ex.com", | |
| 65 | + citations=True, | |
| 66 | + ) | |
| 67 | + assert "[A][1]" in r.markdown and "[B][2]" in r.markdown and "[A encore][1]" in r.markdown | |
| 68 | + assert "[1]: https://a.com" in r.markdown | |
| 69 | + | |
| 70 | + | |
| 71 | +def test_headings_normalized() -> None: | |
| 72 | + r = html_to_markdown( | |
| 73 | + "<body><h1>Un</h1><p>Texte assez long pour compter comme du contenu, avec des phrases et de la ponctuation. Encore une phrase ici.</p><h1>Deux</h1><p>Suite du texte, encore quelques mots pour la densité, et une virgule.</p></body>", | |
| 74 | + "https://ex.com", | |
| 75 | + only_main_content=False, | |
| 76 | + ) | |
| 77 | + assert r.markdown.count("\n# ") + (1 if r.markdown.startswith("# ") else 0) == 1 | |
| 78 | + assert "## Deux" in r.markdown | |
| 79 | + | |
| 80 | + | |
| 81 | +def test_definition_list_and_details() -> None: | |
| 82 | + r = html_to_markdown( | |
| 83 | + "<body><dl><dt>Terme</dt><dd>Définition</dd></dl><details><summary>Plus</summary><p>Caché</p></details></body>", | |
| 84 | + "https://ex.com", | |
| 85 | + only_main_content=False, | |
| 86 | + ) | |
| 87 | + assert ( | |
| 88 | + "**Terme**" in r.markdown | |
| 89 | + and ": Définition" in r.markdown | |
| 90 | + and "**Plus**" in r.markdown | |
| 91 | + and "Caché" in r.markdown | |
| 92 | + ) | |
added
trawls/__init__.py
+3 −0
@@ -0,0 +1,3 @@ | ||
| 1 | +"""Trawls — moteur de crawling, d extraction et de navigation web, LLM-ready.""" | |
| 2 | + | |
| 3 | +__version__ = "0.1.0" | |
added
trawls/api/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/api/auth.py
+103 −0
@@ -0,0 +1,103 @@ | ||
| 1 | +"""Clés API (argon2, affichées une fois), quotas journaliers, rate limit token-bucket en mémoire. | |
| 2 | +En self-host sans TRAWLS_REQUIRE_AUTH, l'auth est désactivée (identité anonyme).""" | |
| 3 | + | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import time | |
| 7 | +from dataclasses import dataclass | |
| 8 | + | |
| 9 | +from fastapi import Depends, HTTPException, Request | |
| 10 | + | |
| 11 | +from trawls.config import get_settings | |
| 12 | +from trawls.worker.store import get_store | |
| 13 | + | |
| 14 | + | |
| 15 | +@dataclass | |
| 16 | +class Identity: | |
| 17 | + key_id: str | None | |
| 18 | + name: str | |
| 19 | + is_admin: bool = False | |
| 20 | + quota_per_day: int | None = None | |
| 21 | + | |
| 22 | + | |
| 23 | +class TokenBucket: | |
| 24 | + def __init__(self, rate_per_min: int) -> None: | |
| 25 | + self.rate = rate_per_min / 60.0 | |
| 26 | + self.cap = float(rate_per_min) | |
| 27 | + self._b: dict[str, tuple[float, float]] = {} | |
| 28 | + | |
| 29 | + def take(self, key: str) -> bool: | |
| 30 | + now = time.monotonic() | |
| 31 | + tokens, last = self._b.get(key, (self.cap, now)) | |
| 32 | + tokens = min(self.cap, tokens + (now - last) * self.rate) | |
| 33 | + if tokens < 1: | |
| 34 | + self._b[key] = (tokens, now) | |
| 35 | + return False | |
| 36 | + self._b[key] = (tokens - 1, now) | |
| 37 | + return True | |
| 38 | + | |
| 39 | + | |
| 40 | +_bucket: TokenBucket | None = None | |
| 41 | + | |
| 42 | + | |
| 43 | +def bucket() -> TokenBucket: | |
| 44 | + global _bucket | |
| 45 | + if _bucket is None: | |
| 46 | + _bucket = TokenBucket(get_settings().rate_limit_per_minute) | |
| 47 | + return _bucket | |
| 48 | + | |
| 49 | + | |
| 50 | +def _bearer(request: Request) -> str | None: | |
| 51 | + h = request.headers.get("authorization") or "" | |
| 52 | + if h.lower().startswith("bearer "): | |
| 53 | + return h[7:].strip() | |
| 54 | + return request.headers.get("x-api-key") or request.query_params.get("api_key") | |
| 55 | + | |
| 56 | + | |
| 57 | +async def current_identity(request: Request) -> Identity: | |
| 58 | + s = get_settings() | |
| 59 | + raw = _bearer(request) | |
| 60 | + if raw and s.admin_key and raw == s.admin_key: | |
| 61 | + ident = Identity(key_id="admin", name="admin", is_admin=True) | |
| 62 | + elif raw and raw.startswith("trw_"): | |
| 63 | + row = await get_store().verify_key(raw) | |
| 64 | + if not row: | |
| 65 | + raise HTTPException(401, detail={"code": "UNAUTHORIZED", "message": "clé API invalide"}) | |
| 66 | + ident = Identity(key_id=row["id"], name=row["name"] or row["id"], quota_per_day=row["quota_per_day"]) | |
| 67 | + elif s.require_auth: | |
| 68 | + raise HTTPException( | |
| 69 | + 401, detail={"code": "UNAUTHORIZED", "message": "Authorization: Bearer <clé> requis"} | |
| 70 | + ) | |
| 71 | + else: | |
| 72 | + ident = Identity(key_id=None, name="anonymous", is_admin=not s.admin_key) | |
| 73 | + rl_key = ident.key_id or (request.client.host if request.client else "anon") | |
| 74 | + if not bucket().take(rl_key): | |
| 75 | + raise HTTPException( | |
| 76 | + 429, detail={"code": "RATE_LIMITED", "message": "trop de requêtes", "retryable": True} | |
| 77 | + ) | |
| 78 | + request.state.identity = ident | |
| 79 | + return ident | |
| 80 | + | |
| 81 | + | |
| 82 | +async def require_admin(ident: Identity = Depends(current_identity)) -> Identity: | |
| 83 | + if not ident.is_admin: | |
| 84 | + raise HTTPException( | |
| 85 | + 403, detail={"code": "FORBIDDEN", "message": "clé admin requise (TRAWLS_ADMIN_KEY)"} | |
| 86 | + ) | |
| 87 | + return ident | |
| 88 | + | |
| 89 | + | |
| 90 | +async def charge(ident: Identity, endpoint: str, credits: int = 1) -> None: | |
| 91 | + store = get_store() | |
| 92 | + if ident.quota_per_day is not None and ident.key_id: | |
| 93 | + used = await store.usage_today(ident.key_id) | |
| 94 | + if used + credits > ident.quota_per_day: | |
| 95 | + raise HTTPException( | |
| 96 | + 429, | |
| 97 | + detail={ | |
| 98 | + "code": "QUOTA_EXCEEDED", | |
| 99 | + "message": f"quota journalier {ident.quota_per_day} atteint", | |
| 100 | + "retryable": True, | |
| 101 | + }, | |
| 102 | + ) | |
| 103 | + await store.record_usage(ident.key_id, endpoint, credits) | |
added
trawls/api/main.py
+702 −0
@@ -0,0 +1,702 @@ | ||
| 1 | +"""API Trawls (FastAPI) — préfixe /v1. Erreurs au format { error: { code, message, retryable, details? } }. | |
| 2 | +Sert aussi l'UI web (/) et les blobs (/v1/blobs). Le worker de jobs tourne dans le même processus | |
| 3 | +(self-host first) ; `TRAWLS_EMBEDDED_WORKER=0` pour le désactiver et lancer `trawls worker` à part. | |
| 4 | +""" | |
| 5 | + | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import asyncio | |
| 9 | +import logging | |
| 10 | +import os | |
| 11 | +import time | |
| 12 | +from collections.abc import AsyncIterator | |
| 13 | +from contextlib import asynccontextmanager | |
| 14 | +from pathlib import Path | |
| 15 | +from typing import Any | |
| 16 | + | |
| 17 | +import orjson | |
| 18 | +import structlog | |
| 19 | +from fastapi import Depends, FastAPI, HTTPException, Query, Request | |
| 20 | +from fastapi.exceptions import RequestValidationError | |
| 21 | +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response | |
| 22 | +from fastapi.staticfiles import StaticFiles | |
| 23 | +from pydantic import BaseModel, Field | |
| 24 | +from sse_starlette.sse import EventSourceResponse | |
| 25 | + | |
| 26 | +from trawls import __version__ | |
| 27 | +from trawls.api.auth import Identity, charge, current_identity, require_admin | |
| 28 | +from trawls.config import get_settings | |
| 29 | +from trawls.core.fetcher.browser import shared_browser | |
| 30 | +from trawls.core.fetcher.http_fast import shared_http | |
| 31 | +from trawls.core.fetcher.strategy import host_modes | |
| 32 | +from trawls.core.scrape import breaker, scrape | |
| 33 | +from trawls.map import map_site | |
| 34 | +from trawls.models import ( | |
| 35 | + ApiErrorEnvelope, | |
| 36 | + CrawlOptions, | |
| 37 | + CssField, | |
| 38 | + JobSummary, | |
| 39 | + MapOptions, | |
| 40 | + MappedUrl, | |
| 41 | + PageResult, | |
| 42 | + ScrapeOptions, | |
| 43 | +) | |
| 44 | +from trawls.worker.crawl import Worker, bus, resolve_extract_urls, run_extract | |
| 45 | +from trawls.worker.store import get_store | |
| 46 | + | |
| 47 | +log = structlog.get_logger("trawls.api") | |
| 48 | +_STATIC = Path(__file__).resolve().parent.parent / "web" / "static" | |
| 49 | + | |
| 50 | + | |
| 51 | +def _setup_logging() -> None: | |
| 52 | + s = get_settings() | |
| 53 | + structlog.configure( | |
| 54 | + processors=[ | |
| 55 | + structlog.contextvars.merge_contextvars, | |
| 56 | + structlog.processors.add_log_level, | |
| 57 | + structlog.processors.TimeStamper(fmt="iso"), | |
| 58 | + structlog.processors.JSONRenderer( | |
| 59 | + serializer=lambda o, **kw: orjson.dumps(o, default=str).decode() | |
| 60 | + ), | |
| 61 | + ], | |
| 62 | + wrapper_class=structlog.make_filtering_bound_logger( | |
| 63 | + getattr(logging, s.log_level.upper(), logging.INFO) | |
| 64 | + ), | |
| 65 | + cache_logger_on_first_use=True, | |
| 66 | + ) | |
| 67 | + | |
| 68 | + | |
| 69 | +_worker: Worker | None = None | |
| 70 | +_worker_task: asyncio.Task[None] | None = None | |
| 71 | +_started_at = time.time() | |
| 72 | + | |
| 73 | + | |
| 74 | +@asynccontextmanager | |
| 75 | +async def lifespan(app: FastAPI) -> AsyncIterator[None]: | |
| 76 | + global _worker, _worker_task | |
| 77 | + _setup_logging() | |
| 78 | + s = get_settings() | |
| 79 | + store = get_store() | |
| 80 | + await store.open() | |
| 81 | + if s.allow_private_targets: | |
| 82 | + log.warning("security.allow_private_targets", value=True) | |
| 83 | + if os.environ.get("TRAWLS_EMBEDDED_WORKER", "1") != "0": | |
| 84 | + _worker = Worker(store) | |
| 85 | + _worker_task = asyncio.create_task(_worker.run()) | |
| 86 | + # préchauffage navigateur en arrière-plan (non bloquant) | |
| 87 | + asyncio.create_task(_warm_browser()) | |
| 88 | + log.info("api.started", version=__version__, port=s.port, public_url=s.public_url) | |
| 89 | + try: | |
| 90 | + yield | |
| 91 | + finally: | |
| 92 | + if _worker: | |
| 93 | + _worker.stop() | |
| 94 | + if _worker_task: | |
| 95 | + _worker_task.cancel() | |
| 96 | + await shared_browser().close() | |
| 97 | + await shared_http().close() | |
| 98 | + await store.close() | |
| 99 | + | |
| 100 | + | |
| 101 | +async def _warm_browser() -> None: | |
| 102 | + try: | |
| 103 | + await shared_browser().start() | |
| 104 | + except Exception as e: | |
| 105 | + log.warning("browser.warmup_failed", error=str(e)) | |
| 106 | + | |
| 107 | + | |
| 108 | +app = FastAPI( | |
| 109 | + title="Trawls API", | |
| 110 | + version=__version__, | |
| 111 | + description="Moteur de crawling, d'extraction et de navigation web, LLM-ready. Sortie Markdown propre par défaut.", | |
| 112 | + lifespan=lifespan, | |
| 113 | + docs_url=None, | |
| 114 | + redoc_url=None, | |
| 115 | + openapi_url="/v1/openapi.json", | |
| 116 | +) | |
| 117 | + | |
| 118 | + | |
| 119 | +# ---- erreurs ------------------------------------------------------------------- | |
| 120 | + | |
| 121 | + | |
| 122 | +def _err(status: int, code: str, message: str, retryable: bool = False, details: Any = None) -> JSONResponse: | |
| 123 | + return JSONResponse( | |
| 124 | + status_code=status, | |
| 125 | + content={ | |
| 126 | + "error": { | |
| 127 | + "code": code, | |
| 128 | + "message": message, | |
| 129 | + "retryable": retryable, | |
| 130 | + **({"details": details} if details else {}), | |
| 131 | + } | |
| 132 | + }, | |
| 133 | + ) | |
| 134 | + | |
| 135 | + | |
| 136 | +@app.exception_handler(HTTPException) | |
| 137 | +async def _http_exc(request: Request, exc: HTTPException) -> JSONResponse: | |
| 138 | + d = exc.detail if isinstance(exc.detail, dict) else {"code": "HTTP_ERROR", "message": str(exc.detail)} | |
| 139 | + return _err( | |
| 140 | + exc.status_code, | |
| 141 | + d.get("code", "HTTP_ERROR"), | |
| 142 | + d.get("message", ""), | |
| 143 | + bool(d.get("retryable")), | |
| 144 | + d.get("details"), | |
| 145 | + ) | |
| 146 | + | |
| 147 | + | |
| 148 | +@app.exception_handler(RequestValidationError) | |
| 149 | +async def _val_exc(request: Request, exc: RequestValidationError) -> JSONResponse: | |
| 150 | + return _err(422, "VALIDATION", "corps de requête invalide", details={"errors": exc.errors()[:10]}) | |
| 151 | + | |
| 152 | + | |
| 153 | +@app.exception_handler(Exception) | |
| 154 | +async def _any_exc(request: Request, exc: Exception) -> JSONResponse: | |
| 155 | + log.error("api.unhandled", path=request.url.path, error=f"{type(exc).__name__}: {exc}") | |
| 156 | + return _err(500, "INTERNAL", f"{type(exc).__name__}: {exc}", retryable=True) | |
| 157 | + | |
| 158 | + | |
| 159 | +@app.middleware("http") | |
| 160 | +async def _log_requests(request: Request, call_next: Any) -> Response: | |
| 161 | + t0 = time.perf_counter() | |
| 162 | + resp: Response = await call_next(request) | |
| 163 | + ms = (time.perf_counter() - t0) * 1000 | |
| 164 | + resp.headers["X-Trawls-Version"] = __version__ | |
| 165 | + resp.headers["Server-Timing"] = f"app;dur={ms:.0f}" | |
| 166 | + if request.url.path.startswith("/v1/") and not request.url.path.startswith(("/v1/blobs", "/v1/openapi")): | |
| 167 | + ident = getattr(request.state, "identity", None) | |
| 168 | + try: | |
| 169 | + await get_store().log_request( | |
| 170 | + ident.key_id if ident else None, | |
| 171 | + request.method, | |
| 172 | + request.url.path, | |
| 173 | + resp.status_code, | |
| 174 | + ms, | |
| 175 | + request.client.host if request.client else None, | |
| 176 | + ) | |
| 177 | + except Exception: | |
| 178 | + pass | |
| 179 | + return resp | |
| 180 | + | |
| 181 | + | |
| 182 | +# ---- santé ----------------------------------------------------------------------- | |
| 183 | + | |
| 184 | + | |
| 185 | +@app.get("/healthz", include_in_schema=False) | |
| 186 | +async def healthz() -> dict[str, Any]: | |
| 187 | + return {"ok": True, "version": __version__, "uptime_s": round(time.time() - _started_at)} | |
| 188 | + | |
| 189 | + | |
| 190 | +@app.get("/readyz", include_in_schema=False) | |
| 191 | +async def readyz() -> JSONResponse: | |
| 192 | + checks = { | |
| 193 | + "db": False, | |
| 194 | + "browser": shared_browser().ready, | |
| 195 | + "worker": bool(_worker_task and not _worker_task.done()), | |
| 196 | + } | |
| 197 | + try: | |
| 198 | + await get_store().stats() | |
| 199 | + checks["db"] = True | |
| 200 | + except Exception: | |
| 201 | + pass | |
| 202 | + ok = checks["db"] | |
| 203 | + return JSONResponse(status_code=200 if ok else 503, content={"ok": ok, "checks": checks}) | |
| 204 | + | |
| 205 | + | |
| 206 | +@app.get("/metrics", include_in_schema=False) | |
| 207 | +async def metrics() -> Response: | |
| 208 | + st = await get_store().stats() | |
| 209 | + lines = ["trawls_up 1", f'trawls_info{{version="{__version__}"}} 1'] | |
| 210 | + for k, v in st.get("jobs", {}).items(): | |
| 211 | + lines.append(f'trawls_jobs_total{{status="{k}"}} {v}') | |
| 212 | + for k, v in st.get("pages", {}).items(): | |
| 213 | + lines.append(f'trawls_pages_total{{status="{k}"}} {v}') | |
| 214 | + lines.append(f"trawls_worker_active_jobs {_worker.active if _worker else 0}") | |
| 215 | + lines.append(f"trawls_browser_ready {1 if shared_browser().ready else 0}") | |
| 216 | + for host, snap in breaker.snapshot().items(): | |
| 217 | + lines.append(f'trawls_breaker_open{{host="{host}"}} {1 if snap["open"] else 0}') | |
| 218 | + return Response("\n".join(lines) + "\n", media_type="text/plain; version=0.0.4") | |
| 219 | + | |
| 220 | + | |
| 221 | +# ---- modèles de requêtes ------------------------------------------------------------ | |
| 222 | + | |
| 223 | + | |
| 224 | +class ScrapeRequest(ScrapeOptions): | |
| 225 | + url: str | |
| 226 | + | |
| 227 | + | |
| 228 | +class CrawlRequest(BaseModel): | |
| 229 | + url: str | |
| 230 | + crawl: CrawlOptions = Field(default_factory=CrawlOptions) | |
| 231 | + scrape: ScrapeOptions = Field(default_factory=ScrapeOptions) | |
| 232 | + webhook: str | None = None | |
| 233 | + | |
| 234 | + | |
| 235 | +class MapRequest(MapOptions): | |
| 236 | + url: str | |
| 237 | + | |
| 238 | + | |
| 239 | +class ExtractRequest(BaseModel): | |
| 240 | + urls: list[str] = [] | |
| 241 | + pattern: str | None = None | |
| 242 | + mode: str = "css" | |
| 243 | + schema_: dict[str, Any] | None = Field(default=None, alias="schema") | |
| 244 | + css: dict[str, CssField] | None = None | |
| 245 | + prompt: str | None = None | |
| 246 | + merge_key: str | None = None | |
| 247 | + limit: int = 50 | |
| 248 | + scrape: ScrapeOptions = Field(default_factory=ScrapeOptions) | |
| 249 | + webhook: str | None = None | |
| 250 | + | |
| 251 | + model_config = {"populate_by_name": True} | |
| 252 | + | |
| 253 | + | |
| 254 | +class BatchRequest(ScrapeOptions): | |
| 255 | + urls: list[str] | |
| 256 | + concurrency: int = 8 | |
| 257 | + webhook: str | None = None | |
| 258 | + | |
| 259 | + | |
| 260 | +class JobCreated(BaseModel): | |
| 261 | + job_id: str | |
| 262 | + status: str | |
| 263 | + url: str | |
| 264 | + | |
| 265 | + | |
| 266 | +class CrawlStatus(JobSummary): | |
| 267 | + pages: list[PageResult] = [] | |
| 268 | + next_cursor: int | None = None | |
| 269 | + | |
| 270 | + | |
| 271 | +class MapResponse(BaseModel): | |
| 272 | + url: str | |
| 273 | + count: int | |
| 274 | + urls: list[MappedUrl] | |
| 275 | + took_ms: float | |
| 276 | + | |
| 277 | + | |
| 278 | +class KeyCreate(BaseModel): | |
| 279 | + name: str | |
| 280 | + quota_per_day: int | None = None | |
| 281 | + | |
| 282 | + | |
| 283 | +def _strip(page: PageResult, opts: ScrapeOptions) -> PageResult: | |
| 284 | + if "links" not in opts.formats: | |
| 285 | + page.links = [] | |
| 286 | + return page | |
| 287 | + | |
| 288 | + | |
| 289 | +# ---- routes v1 -------------------------------------------------------------------------- | |
| 290 | + | |
| 291 | +_R = {401: {"model": ApiErrorEnvelope}, 422: {"model": ApiErrorEnvelope}, 429: {"model": ApiErrorEnvelope}} | |
| 292 | + | |
| 293 | + | |
| 294 | +@app.post("/v1/scrape", response_model=PageResult, responses=_R, tags=["scrape"]) | |
| 295 | +async def scrape_route(req: ScrapeRequest, ident: Identity = Depends(current_identity)) -> PageResult: | |
| 296 | + """Scrape synchrone d'une page (≤ ~60 s). Ne lève jamais : `status=failed` + `error` typée.""" | |
| 297 | + opts = ScrapeOptions.model_validate(req.model_dump(exclude={"url"})) | |
| 298 | + if opts.extract and opts.extract.mode == "llm" and not opts.extract.schema_: | |
| 299 | + raise HTTPException( | |
| 300 | + 422, detail={"code": "VALIDATION", "message": "extract.schema requis en mode llm"} | |
| 301 | + ) | |
| 302 | + await charge(ident, "scrape") | |
| 303 | + try: | |
| 304 | + page = await asyncio.wait_for( | |
| 305 | + scrape(req.url, opts), timeout=max(5, min(opts.timeout_ms / 1000 * 3 + 20, 170)) | |
| 306 | + ) | |
| 307 | + except TimeoutError: | |
| 308 | + from trawls.models import ErrorCode, ErrorInfo | |
| 309 | + | |
| 310 | + page = PageResult.failed(req.url, ErrorInfo.make(ErrorCode.TIMEOUT_RENDER, "délai global dépassé")) | |
| 311 | + return _strip(page, opts) | |
| 312 | + | |
| 313 | + | |
| 314 | +@app.post("/v1/crawl", response_model=JobCreated, status_code=202, responses=_R, tags=["crawl"]) | |
| 315 | +async def crawl_create( | |
| 316 | + req: CrawlRequest, request: Request, ident: Identity = Depends(current_identity) | |
| 317 | +) -> JobCreated: | |
| 318 | + await charge(ident, "crawl", 0) | |
| 319 | + job = await get_store().create_job( | |
| 320 | + "crawl", | |
| 321 | + req.model_dump(mode="json", exclude={"webhook"}), | |
| 322 | + req.url, | |
| 323 | + ident.key_id, | |
| 324 | + request.headers.get("idempotency-key"), | |
| 325 | + req.webhook, | |
| 326 | + ) | |
| 327 | + return JobCreated( | |
| 328 | + job_id=job.id, status=job.status, url=f"{get_settings().public_url.rstrip('/')}/v1/crawl/{job.id}" | |
| 329 | + ) | |
| 330 | + | |
| 331 | + | |
| 332 | +@app.get("/v1/crawl/{job_id}", response_model=CrawlStatus, responses=_R, tags=["crawl"]) | |
| 333 | +async def crawl_status( | |
| 334 | + job_id: str, | |
| 335 | + cursor: int = 0, | |
| 336 | + limit: int = Query(50, le=500), | |
| 337 | + status: str | None = None, | |
| 338 | + ident: Identity = Depends(current_identity), | |
| 339 | +) -> CrawlStatus: | |
| 340 | + job = await get_store().get_job(job_id) | |
| 341 | + if not job: | |
| 342 | + raise HTTPException(404, detail={"code": "NOT_FOUND", "message": "job inconnu"}) | |
| 343 | + pages, nxt = await get_store().pages(job_id, after=cursor, limit=limit, status=status) | |
| 344 | + return CrawlStatus(**job.model_dump(), pages=pages, next_cursor=nxt) | |
| 345 | + | |
| 346 | + | |
| 347 | +@app.get("/v1/crawl/{job_id}/stream", tags=["crawl"]) | |
| 348 | +async def crawl_stream( | |
| 349 | + job_id: str, request: Request, ident: Identity = Depends(current_identity) | |
| 350 | +) -> EventSourceResponse: | |
| 351 | + """SSE : `status`, `page`, `progress`, `done`, `error`. Rejoue d'abord l'état courant.""" | |
| 352 | + return await _stream(job_id, request) | |
| 353 | + | |
| 354 | + | |
| 355 | +async def _stream(job_id: str, request: Request) -> EventSourceResponse: | |
| 356 | + store = get_store() | |
| 357 | + job = await store.get_job(job_id) | |
| 358 | + if not job: | |
| 359 | + raise HTTPException(404, detail={"code": "NOT_FOUND", "message": "job inconnu"}) | |
| 360 | + | |
| 361 | + async def gen() -> AsyncIterator[dict[str, Any]]: | |
| 362 | + q = bus.subscribe(job_id) | |
| 363 | + try: | |
| 364 | + j = await store.get_job(job_id) | |
| 365 | + assert j is not None | |
| 366 | + yield {"event": "status", "data": orjson.dumps(j.model_dump(mode="json"), default=str).decode()} | |
| 367 | + if j.status in ("completed", "failed", "cancelled"): | |
| 368 | + yield { | |
| 369 | + "event": "done", | |
| 370 | + "data": orjson.dumps({"status": j.status, **j.meta}, default=str).decode(), | |
| 371 | + } | |
| 372 | + return | |
| 373 | + while True: | |
| 374 | + if await request.is_disconnected(): | |
| 375 | + return | |
| 376 | + try: | |
| 377 | + ev = await asyncio.wait_for(q.get(), timeout=15) | |
| 378 | + except TimeoutError: | |
| 379 | + yield {"event": "ping", "data": "{}"} | |
| 380 | + continue | |
| 381 | + if ev is None: | |
| 382 | + return | |
| 383 | + yield {"event": ev["event"], "data": orjson.dumps(ev["data"], default=str).decode()} | |
| 384 | + if ev["event"] in ("done", "error"): | |
| 385 | + return | |
| 386 | + finally: | |
| 387 | + bus.unsubscribe(job_id, q) | |
| 388 | + | |
| 389 | + return EventSourceResponse(gen()) | |
| 390 | + | |
| 391 | + | |
| 392 | +@app.delete("/v1/crawl/{job_id}", tags=["crawl"]) | |
| 393 | +async def crawl_cancel(job_id: str, ident: Identity = Depends(current_identity)) -> dict[str, Any]: | |
| 394 | + return await _cancel(job_id) | |
| 395 | + | |
| 396 | + | |
| 397 | +async def _cancel(job_id: str) -> dict[str, Any]: | |
| 398 | + store = get_store() | |
| 399 | + job = await store.get_job(job_id) | |
| 400 | + if not job: | |
| 401 | + raise HTTPException(404, detail={"code": "NOT_FOUND", "message": "job inconnu"}) | |
| 402 | + if job.status in ("queued", "running", "paused"): | |
| 403 | + await store.set_status(job_id, "cancelled") | |
| 404 | + bus.publish(job_id, "done", {"status": "cancelled"}) | |
| 405 | + return {"job_id": job_id, "status": "cancelled"} | |
| 406 | + | |
| 407 | + | |
| 408 | +@app.post("/v1/map", response_model=MapResponse, responses=_R, tags=["map"]) | |
| 409 | +async def map_route(req: MapRequest, ident: Identity = Depends(current_identity)) -> MapResponse: | |
| 410 | + """Cartographie synchrone : sitemaps + crawl shallow, dédupliqué, trié (BM25 si `search`).""" | |
| 411 | + from trawls.core.security import SsrfError, check_url | |
| 412 | + | |
| 413 | + try: | |
| 414 | + await check_url(req.url) | |
| 415 | + except SsrfError as e: | |
| 416 | + raise HTTPException(400, detail={"code": e.code, "message": e.message}) from None | |
| 417 | + await charge(ident, "map") | |
| 418 | + t0 = time.perf_counter() | |
| 419 | + opts = MapOptions.model_validate(req.model_dump(exclude={"url"})) | |
| 420 | + urls = await map_site(req.url, opts) | |
| 421 | + return MapResponse( | |
| 422 | + url=req.url, count=len(urls), urls=urls, took_ms=round((time.perf_counter() - t0) * 1000, 1) | |
| 423 | + ) | |
| 424 | + | |
| 425 | + | |
| 426 | +@app.post("/v1/extract", responses=_R, tags=["extract"]) | |
| 427 | +async def extract_route( | |
| 428 | + req: ExtractRequest, request: Request, ident: Identity = Depends(current_identity) | |
| 429 | +) -> Any: | |
| 430 | + """Extraction structurée CSS ou LLM sur `urls[]` ou un `pattern`. Sync si ≤ 5 URLs, sinon job.""" | |
| 431 | + if req.mode == "llm" and not req.schema_: | |
| 432 | + raise HTTPException(422, detail={"code": "VALIDATION", "message": "schema requis en mode llm"}) | |
| 433 | + if req.mode == "css" and not req.css: | |
| 434 | + raise HTTPException(422, detail={"code": "VALIDATION", "message": "css requis en mode css"}) | |
| 435 | + body = req.model_dump(mode="json", by_alias=True, exclude={"webhook"}) | |
| 436 | + urls = await resolve_extract_urls(body) | |
| 437 | + if not urls: | |
| 438 | + raise HTTPException(422, detail={"code": "VALIDATION", "message": "aucune URL (urls[] ou pattern)"}) | |
| 439 | + store = get_store() | |
| 440 | + await charge(ident, "extract", len(urls)) | |
| 441 | + job = await store.create_job( | |
| 442 | + "extract", | |
| 443 | + body, | |
| 444 | + urls[0], | |
| 445 | + ident.key_id, | |
| 446 | + request.headers.get("idempotency-key"), | |
| 447 | + req.webhook, | |
| 448 | + total=len(urls), | |
| 449 | + ) | |
| 450 | + if len(urls) <= 5: | |
| 451 | + await store.set_status(job.id, "running") | |
| 452 | + # on retire le job de la file du worker (déjà pris en charge ici) | |
| 453 | + meta = await run_extract(job.id, body, store, urls) | |
| 454 | + j = await store.get_job(job.id) | |
| 455 | + return { | |
| 456 | + "job_id": job.id, | |
| 457 | + "status": j.status if j else "completed", | |
| 458 | + "data": meta.get("merged"), | |
| 459 | + "per_url": meta.get("per_url"), | |
| 460 | + } | |
| 461 | + return JSONResponse( | |
| 462 | + status_code=202, | |
| 463 | + content={ | |
| 464 | + "job_id": job.id, | |
| 465 | + "status": "queued", | |
| 466 | + "url": f"{get_settings().public_url.rstrip('/')}/v1/jobs/{job.id}", | |
| 467 | + }, | |
| 468 | + ) | |
| 469 | + | |
| 470 | + | |
| 471 | +@app.post("/v1/batch/scrape", response_model=JobCreated, status_code=202, responses=_R, tags=["batch"]) | |
| 472 | +async def batch_route( | |
| 473 | + req: BatchRequest, request: Request, ident: Identity = Depends(current_identity) | |
| 474 | +) -> JobCreated: | |
| 475 | + if not req.urls: | |
| 476 | + raise HTTPException(422, detail={"code": "VALIDATION", "message": "urls[] vide"}) | |
| 477 | + if len(req.urls) > 10_000: | |
| 478 | + raise HTTPException(422, detail={"code": "VALIDATION", "message": "max 10 000 URLs par batch"}) | |
| 479 | + await charge(ident, "batch", 0) | |
| 480 | + job = await get_store().create_job( | |
| 481 | + "batch", | |
| 482 | + req.model_dump(mode="json", exclude={"webhook"}), | |
| 483 | + req.urls[0], | |
| 484 | + ident.key_id, | |
| 485 | + request.headers.get("idempotency-key"), | |
| 486 | + req.webhook, | |
| 487 | + total=len(req.urls), | |
| 488 | + ) | |
| 489 | + return JobCreated( | |
| 490 | + job_id=job.id, status=job.status, url=f"{get_settings().public_url.rstrip('/')}/v1/jobs/{job.id}" | |
| 491 | + ) | |
| 492 | + | |
| 493 | + | |
| 494 | +@app.get("/v1/jobs", response_model=list[JobSummary], tags=["jobs"]) | |
| 495 | +async def jobs_list( | |
| 496 | + kind: str | None = None, | |
| 497 | + status: str | None = None, | |
| 498 | + limit: int = Query(50, le=500), | |
| 499 | + ident: Identity = Depends(current_identity), | |
| 500 | +) -> list[JobSummary]: | |
| 501 | + return await get_store().list_jobs( | |
| 502 | + limit=limit, | |
| 503 | + kind=kind, | |
| 504 | + status=status, | |
| 505 | + api_key_id=None if ident.is_admin or not ident.key_id else ident.key_id, | |
| 506 | + ) | |
| 507 | + | |
| 508 | + | |
| 509 | +@app.get("/v1/jobs/{job_id}", response_model=CrawlStatus, tags=["jobs"]) | |
| 510 | +async def job_get( | |
| 511 | + job_id: str, | |
| 512 | + cursor: int = 0, | |
| 513 | + limit: int = Query(50, le=500), | |
| 514 | + status: str | None = None, | |
| 515 | + ident: Identity = Depends(current_identity), | |
| 516 | +) -> CrawlStatus: | |
| 517 | + return await crawl_status(job_id, cursor, limit, status, ident) | |
| 518 | + | |
| 519 | + | |
| 520 | +@app.get("/v1/jobs/{job_id}/stream", tags=["jobs"]) | |
| 521 | +async def job_stream( | |
| 522 | + job_id: str, request: Request, ident: Identity = Depends(current_identity) | |
| 523 | +) -> EventSourceResponse: | |
| 524 | + return await _stream(job_id, request) | |
| 525 | + | |
| 526 | + | |
| 527 | +@app.delete("/v1/jobs/{job_id}", tags=["jobs"]) | |
| 528 | +async def job_cancel(job_id: str, ident: Identity = Depends(current_identity)) -> dict[str, Any]: | |
| 529 | + return await _cancel(job_id) | |
| 530 | + | |
| 531 | + | |
| 532 | +@app.get("/v1/jobs/{job_id}/export", tags=["jobs"]) | |
| 533 | +async def job_export( | |
| 534 | + job_id: str, | |
| 535 | + format: str = Query("jsonl", pattern="^(jsonl|zip|md)$"), | |
| 536 | + ident: Identity = Depends(current_identity), | |
| 537 | +) -> Response: | |
| 538 | + """Export JSONL (une page par ligne), ZIP (un .md par page) ou MD concaténé.""" | |
| 539 | + import io | |
| 540 | + import re | |
| 541 | + import zipfile | |
| 542 | + | |
| 543 | + store = get_store() | |
| 544 | + if not await store.get_job(job_id): | |
| 545 | + raise HTTPException(404, detail={"code": "NOT_FOUND", "message": "job inconnu"}) | |
| 546 | + pages: list[PageResult] = [] | |
| 547 | + cursor = 0 | |
| 548 | + while True: | |
| 549 | + batch, nxt = await store.pages(job_id, after=cursor, limit=500) | |
| 550 | + pages.extend(batch) | |
| 551 | + if nxt is None: | |
| 552 | + break | |
| 553 | + cursor = nxt | |
| 554 | + if format == "jsonl": | |
| 555 | + body = b"\n".join(orjson.dumps(p.model_dump(mode="json"), default=str) for p in pages) + b"\n" | |
| 556 | + return Response( | |
| 557 | + body, | |
| 558 | + media_type="application/x-ndjson", | |
| 559 | + headers={"Content-Disposition": f'attachment; filename="{job_id}.jsonl"'}, | |
| 560 | + ) | |
| 561 | + if format == "md": | |
| 562 | + body_s = "\n\n---\n\n".join( | |
| 563 | + f"<!-- {p.final_url} -->\n\n{p.markdown or ''}" for p in pages if p.status == "ok" | |
| 564 | + ) | |
| 565 | + return Response( | |
| 566 | + body_s, | |
| 567 | + media_type="text/markdown; charset=utf-8", | |
| 568 | + headers={"Content-Disposition": f'attachment; filename="{job_id}.md"'}, | |
| 569 | + ) | |
| 570 | + buf = io.BytesIO() | |
| 571 | + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: | |
| 572 | + for i, p in enumerate(pages): | |
| 573 | + if p.status != "ok": | |
| 574 | + continue | |
| 575 | + name = re.sub(r"[^a-zA-Z0-9._-]+", "_", p.final_url.split("://", 1)[-1])[:120] or f"page_{i}" | |
| 576 | + z.writestr(f"{i:05d}_{name}.md", f"<!-- {p.final_url} -->\n\n{p.markdown or ''}") | |
| 577 | + z.writestr( | |
| 578 | + "_index.jsonl", | |
| 579 | + "\n".join( | |
| 580 | + orjson.dumps({"url": p.final_url, "status": p.status, "title": p.metadata.title}).decode() | |
| 581 | + for p in pages | |
| 582 | + ), | |
| 583 | + ) | |
| 584 | + return Response( | |
| 585 | + buf.getvalue(), | |
| 586 | + media_type="application/zip", | |
| 587 | + headers={"Content-Disposition": f'attachment; filename="{job_id}.zip"'}, | |
| 588 | + ) | |
| 589 | + | |
| 590 | + | |
| 591 | +@app.get("/v1/usage", tags=["usage"]) | |
| 592 | +async def usage_route( | |
| 593 | + days: int = Query(30, le=365), ident: Identity = Depends(current_identity) | |
| 594 | +) -> dict[str, Any]: | |
| 595 | + rows = await get_store().usage(None if ident.is_admin else ident.key_id, days) | |
| 596 | + by_day: dict[str, dict[str, int]] = {} | |
| 597 | + for r in rows: | |
| 598 | + d = by_day.setdefault(r["day"], {"requests": 0, "credits": 0}) | |
| 599 | + d["requests"] += int(r["count"] or 0) | |
| 600 | + d["credits"] += int(r["credits"] or 0) | |
| 601 | + return { | |
| 602 | + "identity": ident.name, | |
| 603 | + "days": [{"day": k, **v} for k, v in sorted(by_day.items())], | |
| 604 | + "by_endpoint": rows, | |
| 605 | + } | |
| 606 | + | |
| 607 | + | |
| 608 | +@app.get("/v1/keys", tags=["keys"]) | |
| 609 | +async def keys_list(ident: Identity = Depends(require_admin)) -> list[dict[str, Any]]: | |
| 610 | + return await get_store().list_keys() | |
| 611 | + | |
| 612 | + | |
| 613 | +@app.post("/v1/keys", status_code=201, tags=["keys"]) | |
| 614 | +async def keys_create(req: KeyCreate, ident: Identity = Depends(require_admin)) -> dict[str, Any]: | |
| 615 | + """Crée une clé ; la valeur brute n'est renvoyée qu'une seule fois.""" | |
| 616 | + kid, raw = await get_store().create_key(req.name, req.quota_per_day) | |
| 617 | + return {"id": kid, "name": req.name, "key": raw, "quota_per_day": req.quota_per_day} | |
| 618 | + | |
| 619 | + | |
| 620 | +@app.delete("/v1/keys/{key_id}", tags=["keys"]) | |
| 621 | +async def keys_revoke(key_id: str, ident: Identity = Depends(require_admin)) -> dict[str, Any]: | |
| 622 | + await get_store().revoke_key(key_id) | |
| 623 | + return {"id": key_id, "disabled": True} | |
| 624 | + | |
| 625 | + | |
| 626 | +@app.get("/v1/requests", include_in_schema=False) | |
| 627 | +async def requests_log( | |
| 628 | + limit: int = Query(100, le=1000), ident: Identity = Depends(require_admin) | |
| 629 | +) -> list[dict[str, Any]]: | |
| 630 | + return await get_store().recent_requests(limit) | |
| 631 | + | |
| 632 | + | |
| 633 | +@app.get("/v1/system", include_in_schema=False) | |
| 634 | +async def system_info(ident: Identity = Depends(current_identity)) -> dict[str, Any]: | |
| 635 | + s = get_settings() | |
| 636 | + from trawls.llm.client import shared_llm | |
| 637 | + | |
| 638 | + return { | |
| 639 | + "version": __version__, | |
| 640 | + "uptime_s": round(time.time() - _started_at), | |
| 641 | + "browser_ready": shared_browser().ready, | |
| 642 | + "worker_active": _worker.active if _worker else None, | |
| 643 | + "host_modes": host_modes.snapshot(), | |
| 644 | + "breakers": breaker.snapshot(), | |
| 645 | + "llm": {"provider": s.llm_provider, "model": s.llm_model, "configured": shared_llm().configured}, | |
| 646 | + "require_auth": s.require_auth, | |
| 647 | + "stats": await get_store().stats(), | |
| 648 | + } | |
| 649 | + | |
| 650 | + | |
| 651 | +@app.get("/v1/blobs/{name}", include_in_schema=False) | |
| 652 | +async def blob(name: str) -> FileResponse: | |
| 653 | + import re | |
| 654 | + | |
| 655 | + if not re.fullmatch(r"[a-f0-9]{24}\.(jpg|png|pdf|html)", name): | |
| 656 | + raise HTTPException(404, detail={"code": "NOT_FOUND", "message": "blob inconnu"}) | |
| 657 | + p = get_settings().blob_dir / name | |
| 658 | + if not p.exists(): | |
| 659 | + raise HTTPException(404, detail={"code": "NOT_FOUND", "message": "blob inconnu"}) | |
| 660 | + return FileResponse(p) | |
| 661 | + | |
| 662 | + | |
| 663 | +# ---- docs & UI ------------------------------------------------------------------------- | |
| 664 | + | |
| 665 | + | |
| 666 | +@app.get("/docs", include_in_schema=False) | |
| 667 | +async def docs() -> HTMLResponse: | |
| 668 | + return HTMLResponse( | |
| 669 | + """<!doctype html><html><head><meta charset="utf-8"><title>Trawls API — docs</title> | |
| 670 | +<link rel="icon" href="/static/favicon.svg"></head><body style="margin:0;background:#0b0c0e"> | |
| 671 | +<script id="api-reference" data-url="/v1/openapi.json" data-configuration='{"theme":"deepSpace","darkMode":true,"hideModels":false}'></script> | |
| 672 | +<script src="https://cdn.jsdelivr.net/npm/@scalar/api-reference"></script></body></html>""" | |
| 673 | + ) | |
| 674 | + | |
| 675 | + | |
| 676 | +@app.get("/v1/docs", include_in_schema=False) | |
| 677 | +async def docs_v1() -> RedirectResponse: | |
| 678 | + return RedirectResponse("/docs") | |
| 679 | + | |
| 680 | + | |
| 681 | +if _STATIC.exists(): | |
| 682 | + app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static") | |
| 683 | + | |
| 684 | + @app.get("/", include_in_schema=False) | |
| 685 | + @app.get("/playground", include_in_schema=False) | |
| 686 | + @app.get("/crawls", include_in_schema=False) | |
| 687 | + @app.get("/crawls/{job_id}", include_in_schema=False) | |
| 688 | + @app.get("/map", include_in_schema=False) | |
| 689 | + @app.get("/extract", include_in_schema=False) | |
| 690 | + @app.get("/keys", include_in_schema=False) | |
| 691 | + @app.get("/agent", include_in_schema=False) | |
| 692 | + async def spa(job_id: str | None = None) -> FileResponse: | |
| 693 | + return FileResponse(_STATIC / "index.html") | |
| 694 | + | |
| 695 | + | |
| 696 | +def run() -> None: # pragma: no cover | |
| 697 | + import uvicorn | |
| 698 | + | |
| 699 | + s = get_settings() | |
| 700 | + uvicorn.run( | |
| 701 | + "trawls.api.main:app", host=s.host, port=s.port, log_level=s.log_level.lower(), access_log=False | |
| 702 | + ) | |
added
trawls/cli.py
+205 −0
@@ -0,0 +1,205 @@ | ||
| 1 | +"""CLI : `trawls scrape URL`, `trawls crawl URL`, `trawls map URL`, `trawls serve`, `trawls worker`, `trawls keys`.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import asyncio | |
| 6 | +import json | |
| 7 | +import sys | |
| 8 | +from typing import Any | |
| 9 | + | |
| 10 | +import typer | |
| 11 | +from rich.console import Console | |
| 12 | + | |
| 13 | +app = typer.Typer(help="Trawls — crawling, extraction et navigation web LLM-ready.", no_args_is_help=True) | |
| 14 | +console = Console(stderr=True) | |
| 15 | + | |
| 16 | + | |
| 17 | +def _run(coro: Any) -> Any: | |
| 18 | + return asyncio.run(coro) | |
| 19 | + | |
| 20 | + | |
| 21 | +@app.command() | |
| 22 | +def scrape( | |
| 23 | + url: str, | |
| 24 | + fmt: str = typer.Option( | |
| 25 | + "markdown", | |
| 26 | + "--format", | |
| 27 | + "-f", | |
| 28 | + help="markdown|html|raw_html|json|links|chunks|metadata (séparés par des virgules)", | |
| 29 | + ), | |
| 30 | + mode: str = typer.Option("auto", help="auto|http|browser|stealth"), | |
| 31 | + full: bool = typer.Option(False, "--full", help="Désactive only_main_content"), | |
| 32 | + wait_for: str | None = typer.Option(None, help="Sélecteur CSS ou ms"), | |
| 33 | + out_json: bool = typer.Option(False, "--json", help="Sortie PageResult JSON complet"), | |
| 34 | +) -> None: | |
| 35 | + """Scrape une page et affiche le Markdown (ou le JSON complet).""" | |
| 36 | + from trawls.core.scrape import scrape as _scrape | |
| 37 | + from trawls.models import ScrapeOptions | |
| 38 | + | |
| 39 | + wf: str | int | None = int(wait_for) if wait_for and wait_for.isdigit() else wait_for | |
| 40 | + opts = ScrapeOptions( | |
| 41 | + formats=[f.strip() for f in fmt.split(",")], mode=mode, only_main_content=not full, wait_for=wf | |
| 42 | + ) # type: ignore[arg-type] | |
| 43 | + | |
| 44 | + async def go() -> None: | |
| 45 | + page = await _scrape(url, opts) | |
| 46 | + await _shutdown() | |
| 47 | + if out_json: | |
| 48 | + print(page.model_dump_json(indent=2)) | |
| 49 | + elif page.status != "ok": | |
| 50 | + console.print( | |
| 51 | + f"[red]{page.error.code if page.error else 'failed'}[/red] {page.error.message if page.error else ''}" | |
| 52 | + ) | |
| 53 | + sys.exit(1) | |
| 54 | + else: | |
| 55 | + console.print( | |
| 56 | + f"[dim]{page.final_url} · {page.fetch_mode_used} · {page.metadata.word_count} mots · {page.timings.total_ms} ms[/dim]" | |
| 57 | + ) | |
| 58 | + if "markdown" in opts.formats: | |
| 59 | + print(page.markdown or "") | |
| 60 | + else: | |
| 61 | + print( | |
| 62 | + page.model_dump_json( | |
| 63 | + indent=2, exclude={"markdown"} if "markdown" not in opts.formats else set() | |
| 64 | + ) | |
| 65 | + ) | |
| 66 | + | |
| 67 | + _run(go()) | |
| 68 | + | |
| 69 | + | |
| 70 | +@app.command() | |
| 71 | +def crawl( | |
| 72 | + url: str, | |
| 73 | + max_pages: int = typer.Option(50, help="Nombre max de pages"), | |
| 74 | + max_depth: int = typer.Option(3), | |
| 75 | + concurrency: int = typer.Option(5), | |
| 76 | + out: str | None = typer.Option(None, "--out", "-o", help="Fichier JSONL de sortie (défaut stdout)"), | |
| 77 | +) -> None: | |
| 78 | + """Crawl un site et écrit une ligne JSON par page.""" | |
| 79 | + from trawls.worker.crawl import bus, run_crawl | |
| 80 | + from trawls.worker.store import Store | |
| 81 | + | |
| 82 | + async def go() -> None: | |
| 83 | + store = Store(":memory:") | |
| 84 | + await store.open() | |
| 85 | + req = { | |
| 86 | + "url": url, | |
| 87 | + "crawl": {"max_pages": max_pages, "max_depth": max_depth, "concurrency": concurrency}, | |
| 88 | + "scrape": {"formats": ["markdown"]}, | |
| 89 | + } | |
| 90 | + job = await store.create_job("crawl", req, url) | |
| 91 | + q = bus.subscribe(job.id) | |
| 92 | + task = asyncio.create_task(run_crawl(job.id, req, store)) | |
| 93 | + f = open(out, "w") if out else None | |
| 94 | + while True: | |
| 95 | + ev = await q.get() | |
| 96 | + if ev is None: | |
| 97 | + break | |
| 98 | + if ev["event"] == "page": | |
| 99 | + d = ev["data"] | |
| 100 | + console.print( | |
| 101 | + f"[{'green' if d['status'] == 'ok' else 'red'}]{d['status']:7}[/] d{d['depth']} {d['mode'] or '-':8} {d['url']}" | |
| 102 | + ) | |
| 103 | + elif ev["event"] == "done": | |
| 104 | + console.print(f"[bold]terminé[/bold] {ev['data']}") | |
| 105 | + await task | |
| 106 | + pages, _ = await store.pages(job.id, limit=100_000) | |
| 107 | + for p in pages: | |
| 108 | + line = p.model_dump_json() | |
| 109 | + (f.write(line + "\n") if f else print(line)) | |
| 110 | + if f: | |
| 111 | + f.close() | |
| 112 | + await _shutdown() | |
| 113 | + | |
| 114 | + _run(go()) | |
| 115 | + | |
| 116 | + | |
| 117 | +@app.command("map") | |
| 118 | +def map_cmd(url: str, search: str | None = None, limit: int = 1000, titles: bool = False) -> None: | |
| 119 | + """Liste les URLs d'un site (sitemaps + crawl shallow).""" | |
| 120 | + from trawls.map import map_site | |
| 121 | + from trawls.models import MapOptions | |
| 122 | + | |
| 123 | + async def go() -> None: | |
| 124 | + urls = await map_site(url, MapOptions(search=search, limit=limit, include_titles=titles)) | |
| 125 | + await _shutdown() | |
| 126 | + for u in urls: | |
| 127 | + print(u.url if not search else f"{u.score:7.3f} {u.url}") | |
| 128 | + console.print(f"[dim]{len(urls)} URLs[/dim]") | |
| 129 | + | |
| 130 | + _run(go()) | |
| 131 | + | |
| 132 | + | |
| 133 | +@app.command() | |
| 134 | +def serve(host: str | None = None, port: int | None = None, reload: bool = False) -> None: | |
| 135 | + """Lance l'API + UI (+ worker embarqué).""" | |
| 136 | + import uvicorn | |
| 137 | + | |
| 138 | + from trawls.config import get_settings | |
| 139 | + | |
| 140 | + s = get_settings() | |
| 141 | + uvicorn.run( | |
| 142 | + "trawls.api.main:app", | |
| 143 | + host=host or s.host, | |
| 144 | + port=port or s.port, | |
| 145 | + reload=reload, | |
| 146 | + log_level=s.log_level.lower(), | |
| 147 | + access_log=False, | |
| 148 | + ) | |
| 149 | + | |
| 150 | + | |
| 151 | +@app.command() | |
| 152 | +def worker() -> None: | |
| 153 | + """Lance un worker de jobs seul (API démarrée avec TRAWLS_EMBEDDED_WORKER=0).""" | |
| 154 | + from trawls.worker.crawl import Worker | |
| 155 | + from trawls.worker.store import get_store | |
| 156 | + | |
| 157 | + async def go() -> None: | |
| 158 | + st = get_store() | |
| 159 | + await st.open() | |
| 160 | + await Worker(st).run() | |
| 161 | + | |
| 162 | + _run(go()) | |
| 163 | + | |
| 164 | + | |
| 165 | +keys = typer.Typer(help="Gestion des clés API") | |
| 166 | +app.add_typer(keys, name="keys") | |
| 167 | + | |
| 168 | + | |
| 169 | +@keys.command("create") | |
| 170 | +def keys_create(name: str, quota_per_day: int | None = None) -> None: | |
| 171 | + from trawls.worker.store import get_store | |
| 172 | + | |
| 173 | + async def go() -> None: | |
| 174 | + st = get_store() | |
| 175 | + await st.open() | |
| 176 | + kid, raw = await st.create_key(name, quota_per_day) | |
| 177 | + print(json.dumps({"id": kid, "name": name, "key": raw, "quota_per_day": quota_per_day}, indent=2)) | |
| 178 | + await st.close() | |
| 179 | + | |
| 180 | + _run(go()) | |
| 181 | + | |
| 182 | + | |
| 183 | +@keys.command("list") | |
| 184 | +def keys_list() -> None: | |
| 185 | + from trawls.worker.store import get_store | |
| 186 | + | |
| 187 | + async def go() -> None: | |
| 188 | + st = get_store() | |
| 189 | + await st.open() | |
| 190 | + print(json.dumps(await st.list_keys(), indent=2, default=str)) | |
| 191 | + await st.close() | |
| 192 | + | |
| 193 | + _run(go()) | |
| 194 | + | |
| 195 | + | |
| 196 | +async def _shutdown() -> None: | |
| 197 | + from trawls.core.fetcher.browser import shared_browser | |
| 198 | + from trawls.core.fetcher.http_fast import shared_http | |
| 199 | + | |
| 200 | + await shared_browser().close() | |
| 201 | + await shared_http().close() | |
| 202 | + | |
| 203 | + | |
| 204 | +if __name__ == "__main__": # pragma: no cover | |
| 205 | + app() | |
added
trawls/config.py
+79 −0
@@ -0,0 +1,79 @@ | ||
| 1 | +"""Configuration Trawls — un seul point d'entrée (pydantic-settings, préfixe TRAWLS_). | |
| 2 | + | |
| 3 | +Invariants : toutes les valeurs ont un défaut sûr permettant un fonctionnement 100 % self-host | |
| 4 | +sans clé externe. Les valeurs dangereuses sont loguées explicitement au démarrage. | |
| 5 | +""" | |
| 6 | + | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import os | |
| 10 | +from functools import lru_cache | |
| 11 | +from pathlib import Path | |
| 12 | + | |
| 13 | +from pydantic import Field | |
| 14 | +from pydantic_settings import BaseSettings, SettingsConfigDict | |
| 15 | + | |
| 16 | + | |
| 17 | +class Settings(BaseSettings): | |
| 18 | + model_config = SettingsConfigDict(env_prefix="TRAWLS_", env_file=".env", extra="ignore") | |
| 19 | + | |
| 20 | + env: str = "development" | |
| 21 | + host: str = "0.0.0.0" | |
| 22 | + port: int = 8180 | |
| 23 | + public_url: str = "http://localhost:8180" | |
| 24 | + data_dir: Path = Field(default_factory=lambda: Path(os.environ.get("TRAWLS_DATA_DIR", "./data"))) | |
| 25 | + | |
| 26 | + # Persistance (SQLite par défaut : self-host sans dépendance) | |
| 27 | + database_url: str | None = None # réservé : PostgreSQL ultérieur | |
| 28 | + redis_url: str | None = None # réservé : queue arq ultérieure | |
| 29 | + result_ttl_days: int = 7 | |
| 30 | + | |
| 31 | + # Fetch | |
| 32 | + browser_pool_size: int = Field(default_factory=lambda: max(2, min(8, (os.cpu_count() or 4)))) | |
| 33 | + browser_context_max_pages: int = 50 | |
| 34 | + browser_context_max_age_s: int = 600 | |
| 35 | + max_concurrency_per_host: int = 4 | |
| 36 | + max_concurrency_global: int = 16 | |
| 37 | + max_size_mb: int = 50 | |
| 38 | + max_pages_pdf: int = 500 | |
| 39 | + proxy_urls: str = "" # séparés par des virgules | |
| 40 | + user_agent_extra: str = "" | |
| 41 | + host_mode_cache_ttl_s: int = 3600 | |
| 42 | + | |
| 43 | + # Sécurité | |
| 44 | + require_auth: bool = False | |
| 45 | + admin_key: str | None = None # clé maître pour créer des clés API | |
| 46 | + allow_private_targets: bool = False # SSRF : jamais true en prod | |
| 47 | + rate_limit_per_minute: int = 300 | |
| 48 | + | |
| 49 | + # LLM | |
| 50 | + llm_provider: str = "openai" # openai | anthropic | none | |
| 51 | + llm_api_key: str | None = None | |
| 52 | + llm_base_url: str | None = "http://localhost:11434/v1" # Ollama par défaut | |
| 53 | + llm_model: str = "qwen2.5:7b" | |
| 54 | + | |
| 55 | + # Jobs | |
| 56 | + worker_concurrency: int = 4 | |
| 57 | + job_default_max_pages: int = 100 | |
| 58 | + | |
| 59 | + log_level: str = "INFO" | |
| 60 | + | |
| 61 | + @property | |
| 62 | + def proxies(self) -> list[str]: | |
| 63 | + return [p.strip() for p in self.proxy_urls.split(",") if p.strip()] | |
| 64 | + | |
| 65 | + @property | |
| 66 | + def db_path(self) -> Path: | |
| 67 | + self.data_dir.mkdir(parents=True, exist_ok=True) | |
| 68 | + return self.data_dir / "trawls.db" | |
| 69 | + | |
| 70 | + @property | |
| 71 | + def blob_dir(self) -> Path: | |
| 72 | + d = self.data_dir / "blobs" | |
| 73 | + d.mkdir(parents=True, exist_ok=True) | |
| 74 | + return d | |
| 75 | + | |
| 76 | + | |
| 77 | +@lru_cache | |
| 78 | +def get_settings() -> Settings: | |
| 79 | + return Settings() | |
added
trawls/core/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/core/antibot/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/core/antibot/blocklist.txt
+59 −0
@@ -0,0 +1,59 @@ | ||
| 1 | +# Domaines bloqués en mode navigateur (pub, tracking, widgets lourds). Un domaine par ligne. | |
| 2 | +doubleclick.net | |
| 3 | +googlesyndication.com | |
| 4 | +googleadservices.com | |
| 5 | +google-analytics.com | |
| 6 | +googletagmanager.com | |
| 7 | +googletagservices.com | |
| 8 | +adservice.google.com | |
| 9 | +adnxs.com | |
| 10 | +adsrvr.org | |
| 11 | +amazon-adsystem.com | |
| 12 | +criteo.com | |
| 13 | +criteo.net | |
| 14 | +outbrain.com | |
| 15 | +taboola.com | |
| 16 | +rubiconproject.com | |
| 17 | +pubmatic.com | |
| 18 | +openx.net | |
| 19 | +casalemedia.com | |
| 20 | +scorecardresearch.com | |
| 21 | +quantserve.com | |
| 22 | +moatads.com | |
| 23 | +hotjar.com | |
| 24 | +mouseflow.com | |
| 25 | +fullstory.com | |
| 26 | +clarity.ms | |
| 27 | +facebook.net | |
| 28 | +connect.facebook.net | |
| 29 | +ads-twitter.com | |
| 30 | +static.ads-twitter.com | |
| 31 | +analytics.twitter.com | |
| 32 | +bat.bing.com | |
| 33 | +segment.io | |
| 34 | +segment.com | |
| 35 | +mixpanel.com | |
| 36 | +amplitude.com | |
| 37 | +optimizely.com | |
| 38 | +newrelic.com | |
| 39 | +nr-data.net | |
| 40 | +sentry.io | |
| 41 | +intercom.io | |
| 42 | +intercomcdn.com | |
| 43 | +drift.com | |
| 44 | +zdassets.com | |
| 45 | +zopim.com | |
| 46 | +livechatinc.com | |
| 47 | +tawk.to | |
| 48 | +crisp.chat | |
| 49 | +onesignal.com | |
| 50 | +pushwoosh.com | |
| 51 | +cookielaw.org | |
| 52 | +onetrust.com | |
| 53 | +cookiebot.com | |
| 54 | +trustarc.com | |
| 55 | +consensu.org | |
| 56 | +didomi.io | |
| 57 | +sharethis.com | |
| 58 | +addthis.com | |
| 59 | +disqus.com | |
added
trawls/core/antibot/detect.py
+108 −0
@@ -0,0 +1,108 @@ | ||
| 1 | +"""Détection des challenges anti-bot (Cloudflare, DataDome, Akamai, PerimeterX, captchas, pages vides). | |
| 2 | + | |
| 3 | +Sortie : `Detection(kind, confidence, reason)` ; `kind=None` = page normale. | |
| 4 | +""" | |
| 5 | + | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import re | |
| 9 | +from dataclasses import dataclass | |
| 10 | + | |
| 11 | +_PATTERNS: list[tuple[str, re.Pattern[str], float]] = [ | |
| 12 | + ( | |
| 13 | + "cloudflare", | |
| 14 | + re.compile( | |
| 15 | + r"cf-browser-verification|cf_chl_opt|challenge-platform|Checking your browser|Just a moment\.\.\.|cf-turnstile|__cf_chl", | |
| 16 | + re.I, | |
| 17 | + ), | |
| 18 | + 0.95, | |
| 19 | + ), | |
| 20 | + ( | |
| 21 | + "cloudflare", | |
| 22 | + re.compile(r"Attention Required!\s*\|\s*Cloudflare|Sorry, you have been blocked", re.I), | |
| 23 | + 0.9, | |
| 24 | + ), | |
| 25 | + ("datadome", re.compile(r"datadome|dd\.js|geo\.captcha-delivery\.com", re.I), 0.9), | |
| 26 | + ("akamai", re.compile(r"_abck|akam/|Access Denied.{0,200}Reference #", re.I | re.S), 0.85), | |
| 27 | + ("perimeterx", re.compile(r"_pxhd|px-captcha|perimeterx|Human Challenge|px-cdn", re.I), 0.9), | |
| 28 | + ("kasada", re.compile(r"kasada|ips\.js\?", re.I), 0.7), | |
| 29 | + ("imperva", re.compile(r"Incapsula|_Incapsula_Resource|imperva", re.I), 0.85), | |
| 30 | + ( | |
| 31 | + "captcha", | |
| 32 | + re.compile( | |
| 33 | + r"g-recaptcha|hcaptcha\.com|recaptcha/api|verify you are human|Are you a robot|Pardon Our Interruption", | |
| 34 | + re.I, | |
| 35 | + ), | |
| 36 | + 0.8, | |
| 37 | + ), | |
| 38 | + ( | |
| 39 | + "generic_block", | |
| 40 | + re.compile(r"<title>\s*(Access Denied|403 Forbidden|Blocked|Request blocked)\s*</title>", re.I), | |
| 41 | + 0.7, | |
| 42 | + ), | |
| 43 | +] | |
| 44 | + | |
| 45 | +_SPA_ROOTS = re.compile( | |
| 46 | + r'<(div|main|section)[^>]+id=["\'](root|app|__next|__nuxt|main|application)["\'][^>]*>\s*</\1>', re.I | |
| 47 | +) | |
| 48 | +_TAG = re.compile(r"<[^>]+>") | |
| 49 | +_SCRIPTS = re.compile(r"<(script|style|noscript)[^>]*>.*?</\1>", re.I | re.S) | |
| 50 | + | |
| 51 | + | |
| 52 | +@dataclass | |
| 53 | +class Detection: | |
| 54 | + kind: str | None | |
| 55 | + confidence: float | |
| 56 | + reason: str | |
| 57 | + | |
| 58 | + @property | |
| 59 | + def blocked(self) -> bool: | |
| 60 | + return self.kind is not None and self.kind not in ("empty", "spa") | |
| 61 | + | |
| 62 | + @property | |
| 63 | + def needs_browser(self) -> bool: | |
| 64 | + return self.kind in ("empty", "spa") | |
| 65 | + | |
| 66 | + | |
| 67 | +def visible_text_len(html: str) -> int: | |
| 68 | + body = _SCRIPTS.sub(" ", html) | |
| 69 | + m = re.search(r"<body[^>]*>(.*)</body>", body, re.I | re.S) | |
| 70 | + if m: | |
| 71 | + body = m.group(1) | |
| 72 | + txt = _TAG.sub(" ", body) | |
| 73 | + return len(re.sub(r"\s+", " ", txt).strip()) | |
| 74 | + | |
| 75 | + | |
| 76 | +def detect(html: str | None, status: int, headers: dict[str, str] | None = None) -> Detection: | |
| 77 | + headers = {k.lower(): v for k, v in (headers or {}).items()} | |
| 78 | + html = html or "" | |
| 79 | + head = html[:200_000] | |
| 80 | + server = headers.get("server", "").lower() | |
| 81 | + | |
| 82 | + if status in (403, 429, 503) or status == 200: | |
| 83 | + for kind, pat, conf in _PATTERNS: | |
| 84 | + if pat.search(head): | |
| 85 | + if ( | |
| 86 | + status == 200 | |
| 87 | + and kind in ("datadome", "akamai", "kasada", "perimeterx") | |
| 88 | + and visible_text_len(head) > 1500 | |
| 89 | + ): | |
| 90 | + # script présent mais page servie : pas un blocage | |
| 91 | + continue | |
| 92 | + return Detection(kind, conf, f"marqueur {kind} (HTTP {status})") | |
| 93 | + if status == 403 and "cloudflare" in server: | |
| 94 | + return Detection("cloudflare", 0.8, "403 servi par Cloudflare") | |
| 95 | + if status in (403, 429, 503) and visible_text_len(head) < 600: | |
| 96 | + return Detection("generic_block", 0.6, f"HTTP {status} avec page quasi vide") | |
| 97 | + | |
| 98 | + if status < 400: | |
| 99 | + tl = visible_text_len(head) | |
| 100 | + if tl < 200: | |
| 101 | + if _SPA_ROOTS.search(head) or re.search(r"<script[^>]+src=[^>]+\.(js|mjs)", head, re.I): | |
| 102 | + return Detection("spa", 0.8, f"racine SPA vide ({tl} caractères visibles)") | |
| 103 | + if "<script" in head.lower(): | |
| 104 | + return Detection("empty", 0.6, f"body quasi vide ({tl} caractères visibles) avec scripts") | |
| 105 | + return Detection(None, 0.0, f"page courte mais statique ({tl} caractères)") | |
| 106 | + if re.search(r"<noscript>[^<]{0,200}(enable|activer)\s+javascript", head, re.I) and tl < 800: | |
| 107 | + return Detection("spa", 0.7, "noscript 'enable JavaScript' + peu de texte") | |
| 108 | + return Detection(None, 0.0, "ok") | |
added
trawls/core/antibot/identity.py
+74 −0
@@ -0,0 +1,74 @@ | ||
| 1 | +"""Identités cohérentes : UA + headers + viewport + locale + timezone. Un context navigateur = une identité.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import random | |
| 6 | +from dataclasses import dataclass | |
| 7 | + | |
| 8 | +from trawls.models import Location | |
| 9 | + | |
| 10 | +_CHROME_VERSIONS = ["128.0.0.0", "129.0.0.0", "130.0.0.0", "131.0.0.0"] | |
| 11 | +_MAC_VERSIONS = ["10_15_7", "14_6_1", "15_0"] | |
| 12 | +_VIEWPORTS = [(1440, 900), (1536, 864), (1920, 1080), (1680, 1050), (1366, 768)] | |
| 13 | +_TZ_BY_COUNTRY = { | |
| 14 | + "CA": "America/Toronto", | |
| 15 | + "US": "America/New_York", | |
| 16 | + "FR": "Europe/Paris", | |
| 17 | + "GB": "Europe/London", | |
| 18 | + "DE": "Europe/Berlin", | |
| 19 | +} | |
| 20 | + | |
| 21 | + | |
| 22 | +@dataclass(frozen=True) | |
| 23 | +class Identity: | |
| 24 | + user_agent: str | |
| 25 | + chrome_version: str | |
| 26 | + viewport: tuple[int, int] | |
| 27 | + locale: str | |
| 28 | + accept_language: str | |
| 29 | + timezone: str | |
| 30 | + platform: str = "MacIntel" | |
| 31 | + | |
| 32 | + @property | |
| 33 | + def impersonate(self) -> str: | |
| 34 | + major = self.chrome_version.split(".")[0] | |
| 35 | + return "chrome" + major if major in ("120", "123", "124", "131") else "chrome" | |
| 36 | + | |
| 37 | + def headers(self, extra: dict[str, str] | None = None) -> dict[str, str]: | |
| 38 | + major = self.chrome_version.split(".")[0] | |
| 39 | + h = { | |
| 40 | + "User-Agent": self.user_agent, | |
| 41 | + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", | |
| 42 | + "Accept-Language": self.accept_language, | |
| 43 | + "Accept-Encoding": "gzip, deflate, br", | |
| 44 | + "Sec-CH-UA": f'"Chromium";v="{major}", "Google Chrome";v="{major}", "Not_A Brand";v="24"', | |
| 45 | + "Sec-CH-UA-Mobile": "?0", | |
| 46 | + "Sec-CH-UA-Platform": '"macOS"', | |
| 47 | + "Sec-Fetch-Dest": "document", | |
| 48 | + "Sec-Fetch-Mode": "navigate", | |
| 49 | + "Sec-Fetch-Site": "none", | |
| 50 | + "Sec-Fetch-User": "?1", | |
| 51 | + "Upgrade-Insecure-Requests": "1", | |
| 52 | + "Cache-Control": "max-age=0", | |
| 53 | + } | |
| 54 | + if extra: | |
| 55 | + h.update(extra) | |
| 56 | + return h | |
| 57 | + | |
| 58 | + | |
| 59 | +def make_identity(location: Location | None = None, seed: str | None = None) -> Identity: | |
| 60 | + rnd = random.Random(seed) if seed else random.Random() | |
| 61 | + cv = rnd.choice(_CHROME_VERSIONS) | |
| 62 | + mv = rnd.choice(_MAC_VERSIONS) | |
| 63 | + ua = f"Mozilla/5.0 (Macintosh; Intel Mac OS X {mv}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{cv} Safari/537.36" | |
| 64 | + langs = (location.languages if location else ["fr-CA", "fr", "en-US", "en"]) or ["en-US", "en"] | |
| 65 | + al = ",".join(f"{lg};q={max(0.1, 1 - 0.1 * i):.1f}".replace(";q=1.0", "") for i, lg in enumerate(langs)) | |
| 66 | + country = (location.country if location else "CA").upper() | |
| 67 | + return Identity( | |
| 68 | + user_agent=ua, | |
| 69 | + chrome_version=cv, | |
| 70 | + viewport=rnd.choice(_VIEWPORTS), | |
| 71 | + locale=langs[0], | |
| 72 | + accept_language=al, | |
| 73 | + timezone=_TZ_BY_COUNTRY.get(country, "UTC"), | |
| 74 | + ) | |
added
trawls/core/antibot/stealth.py
+87 −0
@@ -0,0 +1,87 @@ | ||
| 1 | +"""Patches JS injectés avant tout script de page (mode stealth) : webdriver, plugins, languages, | |
| 2 | +chrome.runtime, permissions, WebGL vendor, bruit canvas léger, hardwareConcurrency cohérent. | |
| 3 | +""" | |
| 4 | + | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +STEALTH_JS = r""" | |
| 8 | +(() => { | |
| 9 | + const define = (obj, prop, value) => { | |
| 10 | + try { Object.defineProperty(obj, prop, { get: () => value, configurable: true }); } catch (e) {} | |
| 11 | + }; | |
| 12 | + // navigator.webdriver | |
| 13 | + define(Navigator.prototype, 'webdriver', undefined); | |
| 14 | + delete navigator.__proto__.webdriver; | |
| 15 | + // languages / platform / hardware | |
| 16 | + define(navigator, 'languages', ['%LANGS%']); | |
| 17 | + define(navigator, 'platform', 'MacIntel'); | |
| 18 | + define(navigator, 'hardwareConcurrency', 8); | |
| 19 | + define(navigator, 'deviceMemory', 8); | |
| 20 | + define(navigator, 'maxTouchPoints', 0); | |
| 21 | + // plugins réalistes | |
| 22 | + const fakePlugins = [ | |
| 23 | + { name: 'PDF Viewer', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, | |
| 24 | + { name: 'Chrome PDF Viewer', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, | |
| 25 | + { name: 'Chromium PDF Viewer', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, | |
| 26 | + { name: 'Microsoft Edge PDF Viewer', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, | |
| 27 | + { name: 'WebKit built-in PDF', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, | |
| 28 | + ]; | |
| 29 | + const plugins = Object.create(PluginArray.prototype); | |
| 30 | + fakePlugins.forEach((p, i) => { plugins[i] = p; plugins[p.name] = p; }); | |
| 31 | + define(plugins, 'length', fakePlugins.length); | |
| 32 | + plugins.item = i => plugins[i] || null; plugins.namedItem = n => plugins[n] || null; | |
| 33 | + plugins.refresh = () => {}; | |
| 34 | + define(navigator, 'plugins', plugins); | |
| 35 | + // chrome.runtime | |
| 36 | + if (!window.chrome) window.chrome = {}; | |
| 37 | + if (!window.chrome.runtime) window.chrome.runtime = { connect: () => {}, sendMessage: () => {}, id: undefined }; | |
| 38 | + if (!window.chrome.loadTimes) window.chrome.loadTimes = () => ({}); | |
| 39 | + if (!window.chrome.csi) window.chrome.csi = () => ({}); | |
| 40 | + // permissions | |
| 41 | + const origQuery = window.navigator.permissions && window.navigator.permissions.query; | |
| 42 | + if (origQuery) { | |
| 43 | + window.navigator.permissions.query = (p) => (p && p.name === 'notifications') | |
| 44 | + ? Promise.resolve({ state: Notification.permission, onchange: null }) | |
| 45 | + : origQuery.call(window.navigator.permissions, p); | |
| 46 | + } | |
| 47 | + // WebGL vendor/renderer | |
| 48 | + const patchGL = (proto) => { | |
| 49 | + if (!proto) return; | |
| 50 | + const gp = proto.getParameter; | |
| 51 | + proto.getParameter = function (p) { | |
| 52 | + if (p === 37445) return 'Google Inc. (Apple)'; | |
| 53 | + if (p === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)'; | |
| 54 | + return gp.call(this, p); | |
| 55 | + }; | |
| 56 | + }; | |
| 57 | + patchGL(window.WebGLRenderingContext && WebGLRenderingContext.prototype); | |
| 58 | + patchGL(window.WebGL2RenderingContext && WebGL2RenderingContext.prototype); | |
| 59 | + // Canvas : bruit imperceptible | |
| 60 | + const toDataURL = HTMLCanvasElement.prototype.toDataURL; | |
| 61 | + HTMLCanvasElement.prototype.toDataURL = function (...args) { | |
| 62 | + try { | |
| 63 | + const ctx = this.getContext('2d'); | |
| 64 | + if (ctx && this.width > 16 && this.height > 16) { | |
| 65 | + const d = ctx.getImageData(0, 0, 1, 1); | |
| 66 | + d.data[0] = (d.data[0] + 1) % 256; ctx.putImageData(d, 0, 0); | |
| 67 | + } | |
| 68 | + } catch (e) {} | |
| 69 | + return toDataURL.apply(this, args); | |
| 70 | + }; | |
| 71 | + // iframe contentWindow (Headless détection) | |
| 72 | + try { | |
| 73 | + const desc = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'contentWindow'); | |
| 74 | + Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', { | |
| 75 | + get: function () { const w = desc.get.call(this); if (w) { try { define(w.navigator, 'webdriver', undefined); } catch (e) {} } return w; } | |
| 76 | + }); | |
| 77 | + } catch (e) {} | |
| 78 | + // outerWidth/Height cohérents | |
| 79 | + define(window, 'outerWidth', window.innerWidth); | |
| 80 | + define(window, 'outerHeight', window.innerHeight + 85); | |
| 81 | +})(); | |
| 82 | +""" | |
| 83 | + | |
| 84 | + | |
| 85 | +def stealth_script(languages: list[str]) -> str: | |
| 86 | + langs = "', '".join(languages or ["en-US", "en"]) | |
| 87 | + return STEALTH_JS.replace("%LANGS%", langs) | |
added
trawls/core/fetcher/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/core/fetcher/base.py
+53 −0
@@ -0,0 +1,53 @@ | ||
| 1 | +"""Protocol Fetcher + FetchResult : contrat commun HTTP rapide / navigateur / stealth.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +from dataclasses import dataclass, field | |
| 6 | +from typing import Protocol | |
| 7 | + | |
| 8 | +from trawls.models import ResolvedMode, ScrapeOptions, Timings | |
| 9 | + | |
| 10 | + | |
| 11 | +@dataclass | |
| 12 | +class FetchResult: | |
| 13 | + url: str | |
| 14 | + final_url: str | |
| 15 | + status: int | |
| 16 | + headers: dict[str, str] | |
| 17 | + body: bytes | |
| 18 | + mode: ResolvedMode | |
| 19 | + timings: Timings = field(default_factory=Timings) | |
| 20 | + text: str | None = None # HTML rendu (navigateur) ou décodé (http) | |
| 21 | + screenshot: bytes | None = None | |
| 22 | + charset: str | None = None | |
| 23 | + | |
| 24 | + @property | |
| 25 | + def content_type(self) -> str: | |
| 26 | + return (self.headers.get("content-type") or "").split(";")[0].strip().lower() | |
| 27 | + | |
| 28 | + @property | |
| 29 | + def is_pdf(self) -> bool: | |
| 30 | + return self.content_type == "application/pdf" or self.body[:5] == b"%PDF-" | |
| 31 | + | |
| 32 | + @property | |
| 33 | + def is_html(self) -> bool: | |
| 34 | + ct = self.content_type | |
| 35 | + if ct in ("text/html", "application/xhtml+xml") or ct == "": | |
| 36 | + return not self.is_pdf | |
| 37 | + return False | |
| 38 | + | |
| 39 | + @property | |
| 40 | + def is_text(self) -> bool: | |
| 41 | + ct = self.content_type | |
| 42 | + return ct.startswith("text/") or ct in ( | |
| 43 | + "application/json", | |
| 44 | + "application/xml", | |
| 45 | + "application/rss+xml", | |
| 46 | + "application/atom+xml", | |
| 47 | + ) | |
| 48 | + | |
| 49 | + | |
| 50 | +class Fetcher(Protocol): | |
| 51 | + mode: ResolvedMode | |
| 52 | + | |
| 53 | + async def fetch(self, url: str, opts: ScrapeOptions) -> FetchResult: ... | |
added
trawls/core/fetcher/browser.py
+317 −0
@@ -0,0 +1,317 @@ | ||
| 1 | +"""Fetcher navigateur : pool de contexts Playwright persistants (pas un browser par page). | |
| 2 | + | |
| 3 | +Invariants : | |
| 4 | +- un context = une identité cohérente (UA, viewport, locale, timezone) ; | |
| 5 | +- recyclage après N pages ou T secondes ; | |
| 6 | +- interception réseau : images/fonts/media/pubs bloquées sauf si screenshot demandé ; | |
| 7 | +- mode stealth = même pool + script d'init + identité verrouillée par host. | |
| 8 | +""" | |
| 9 | + | |
| 10 | +from __future__ import annotations | |
| 11 | + | |
| 12 | +import asyncio | |
| 13 | +import time | |
| 14 | +from dataclasses import dataclass, field | |
| 15 | +from pathlib import Path | |
| 16 | +from typing import Any | |
| 17 | +from urllib.parse import urlsplit | |
| 18 | + | |
| 19 | +import structlog | |
| 20 | + | |
| 21 | +from trawls.config import get_settings | |
| 22 | +from trawls.core.antibot.identity import Identity, make_identity | |
| 23 | +from trawls.core.antibot.stealth import stealth_script | |
| 24 | +from trawls.core.fetcher.base import FetchResult | |
| 25 | +from trawls.core.resilience.retry import FetchError | |
| 26 | +from trawls.core.security import SsrfError, check_url | |
| 27 | +from trawls.models import BrowserAction, ErrorCode, ResolvedMode, ScrapeOptions, Timings | |
| 28 | + | |
| 29 | +log = structlog.get_logger(__name__) | |
| 30 | + | |
| 31 | +_BLOCKLIST_PATH = Path(__file__).resolve().parent.parent / "antibot" / "blocklist.txt" | |
| 32 | +_BLOCKED_DOMAINS: frozenset[str] = frozenset( | |
| 33 | + ln.strip() for ln in _BLOCKLIST_PATH.read_text().splitlines() if ln.strip() and not ln.startswith("#") | |
| 34 | +) | |
| 35 | +_BLOCKED_TYPES = {"image", "font", "media", "stylesheet"} | |
| 36 | + | |
| 37 | + | |
| 38 | +def _domain_blocked(url: str) -> bool: | |
| 39 | + host = (urlsplit(url).hostname or "").lower() | |
| 40 | + while host: | |
| 41 | + if host in _BLOCKED_DOMAINS: | |
| 42 | + return True | |
| 43 | + host = host.partition(".")[2] | |
| 44 | + return False | |
| 45 | + | |
| 46 | + | |
| 47 | +@dataclass | |
| 48 | +class _Ctx: | |
| 49 | + context: Any | |
| 50 | + identity: Identity | |
| 51 | + stealth: bool | |
| 52 | + pages_served: int = 0 | |
| 53 | + created: float = field(default_factory=time.monotonic) | |
| 54 | + busy: bool = False | |
| 55 | + | |
| 56 | + def expired(self, max_pages: int, max_age: int) -> bool: | |
| 57 | + return self.pages_served >= max_pages or time.monotonic() - self.created > max_age | |
| 58 | + | |
| 59 | + | |
| 60 | +class BrowserPool: | |
| 61 | + mode: ResolvedMode = "browser" | |
| 62 | + | |
| 63 | + def __init__(self) -> None: | |
| 64 | + self.settings = get_settings() | |
| 65 | + self._pw: Any = None | |
| 66 | + self._browser: Any = None | |
| 67 | + self._ctxs: list[_Ctx] = [] | |
| 68 | + self._lock = asyncio.Lock() | |
| 69 | + self._slots = asyncio.Semaphore(self.settings.browser_pool_size) | |
| 70 | + self._started = False | |
| 71 | + | |
| 72 | + @property | |
| 73 | + def ready(self) -> bool: | |
| 74 | + return self._browser is not None and self._browser.is_connected() | |
| 75 | + | |
| 76 | + async def start(self) -> None: | |
| 77 | + async with self._lock: | |
| 78 | + if self._started and self.ready: | |
| 79 | + return | |
| 80 | + try: | |
| 81 | + from playwright.async_api import async_playwright | |
| 82 | + except ImportError as e: # pragma: no cover | |
| 83 | + raise FetchError(ErrorCode.INTERNAL, f"playwright indisponible: {e}") from None | |
| 84 | + self._pw = await async_playwright().start() | |
| 85 | + args = [ | |
| 86 | + "--disable-blink-features=AutomationControlled", | |
| 87 | + "--disable-dev-shm-usage", | |
| 88 | + "--no-first-run", | |
| 89 | + "--no-default-browser-check", | |
| 90 | + "--disable-background-timer-throttling", | |
| 91 | + "--disable-renderer-backgrounding", | |
| 92 | + ] | |
| 93 | + self._browser = await self._pw.chromium.launch(headless=True, args=args) | |
| 94 | + self._started = True | |
| 95 | + log.info("browser.started", pool=self.settings.browser_pool_size) | |
| 96 | + | |
| 97 | + async def close(self) -> None: | |
| 98 | + for c in self._ctxs: | |
| 99 | + try: | |
| 100 | + await c.context.close() | |
| 101 | + except Exception: | |
| 102 | + pass | |
| 103 | + self._ctxs.clear() | |
| 104 | + if self._browser: | |
| 105 | + try: | |
| 106 | + await self._browser.close() | |
| 107 | + except Exception: | |
| 108 | + pass | |
| 109 | + if self._pw: | |
| 110 | + try: | |
| 111 | + await self._pw.stop() | |
| 112 | + except Exception: | |
| 113 | + pass | |
| 114 | + self._browser = self._pw = None | |
| 115 | + self._started = False | |
| 116 | + | |
| 117 | + async def _new_ctx(self, identity: Identity, stealth: bool, proxy: str | None) -> _Ctx: | |
| 118 | + kw: dict[str, Any] = dict( | |
| 119 | + user_agent=identity.user_agent, | |
| 120 | + viewport={"width": identity.viewport[0], "height": identity.viewport[1]}, | |
| 121 | + locale=identity.locale, | |
| 122 | + timezone_id=identity.timezone, | |
| 123 | + extra_http_headers={"Accept-Language": identity.accept_language}, | |
| 124 | + java_script_enabled=True, | |
| 125 | + ignore_https_errors=True, | |
| 126 | + device_scale_factor=2 if stealth else 1, | |
| 127 | + ) | |
| 128 | + if proxy: | |
| 129 | + kw["proxy"] = {"server": proxy} | |
| 130 | + ctx = await self._browser.new_context(**kw) | |
| 131 | + if stealth: | |
| 132 | + await ctx.add_init_script(stealth_script(identity.accept_language.split(",")[0:3])) | |
| 133 | + return _Ctx(context=ctx, identity=identity, stealth=stealth) | |
| 134 | + | |
| 135 | + async def _acquire(self, stealth: bool, location: Any, proxy: str | None) -> _Ctx: | |
| 136 | + if not self.ready: | |
| 137 | + await self.start() | |
| 138 | + async with self._lock: | |
| 139 | + # recyclage | |
| 140 | + keep: list[_Ctx] = [] | |
| 141 | + for c in self._ctxs: | |
| 142 | + if not c.busy and c.expired( | |
| 143 | + self.settings.browser_context_max_pages, self.settings.browser_context_max_age_s | |
| 144 | + ): | |
| 145 | + try: | |
| 146 | + await c.context.close() | |
| 147 | + except Exception: | |
| 148 | + pass | |
| 149 | + else: | |
| 150 | + keep.append(c) | |
| 151 | + self._ctxs = keep | |
| 152 | + for c in self._ctxs: | |
| 153 | + if not c.busy and c.stealth == stealth and not proxy: | |
| 154 | + c.busy = True | |
| 155 | + return c | |
| 156 | + c = await self._new_ctx(make_identity(location), stealth, proxy) | |
| 157 | + c.busy = True | |
| 158 | + self._ctxs.append(c) | |
| 159 | + return c | |
| 160 | + | |
| 161 | + def _release(self, c: _Ctx) -> None: | |
| 162 | + c.busy = False | |
| 163 | + c.pages_served += 1 | |
| 164 | + | |
| 165 | + async def fetch(self, url: str, opts: ScrapeOptions, stealth: bool = False) -> FetchResult: | |
| 166 | + try: | |
| 167 | + await check_url(url) | |
| 168 | + except SsrfError as e: | |
| 169 | + raise FetchError(e.code, e.message) from None | |
| 170 | + want_shot = "screenshot" in opts.formats | |
| 171 | + proxy = opts.proxy or (self.settings.proxies[0] if (stealth and self.settings.proxies) else None) | |
| 172 | + t0 = time.perf_counter() | |
| 173 | + await self._slots.acquire() | |
| 174 | + ctx: _Ctx | None = None | |
| 175 | + page: Any = None | |
| 176 | + try: | |
| 177 | + ctx = await self._acquire(stealth, opts.location, proxy) | |
| 178 | + page = await ctx.context.new_page() | |
| 179 | + if opts.cookies: | |
| 180 | + await ctx.context.add_cookies( | |
| 181 | + [ | |
| 182 | + { | |
| 183 | + "name": c.name, | |
| 184 | + "value": c.value, | |
| 185 | + "domain": c.domain or (urlsplit(url).hostname or ""), | |
| 186 | + "path": c.path, | |
| 187 | + } | |
| 188 | + for c in opts.cookies | |
| 189 | + ] | |
| 190 | + ) | |
| 191 | + if opts.headers: | |
| 192 | + await page.set_extra_http_headers(opts.headers) | |
| 193 | + | |
| 194 | + async def _route(route: Any) -> None: | |
| 195 | + req = route.request | |
| 196 | + if ( | |
| 197 | + req.resource_type in _BLOCKED_TYPES | |
| 198 | + and not (want_shot and req.resource_type in ("image", "stylesheet")) | |
| 199 | + ) or _domain_blocked(req.url): | |
| 200 | + await route.abort() | |
| 201 | + else: | |
| 202 | + await route.continue_() | |
| 203 | + | |
| 204 | + await page.route("**/*", _route) | |
| 205 | + timeout = opts.timeout_ms | |
| 206 | + resp = None | |
| 207 | + try: | |
| 208 | + resp = await page.goto(url, wait_until="domcontentloaded", timeout=timeout) | |
| 209 | + except Exception as e: | |
| 210 | + raise _pw_error(e) from None | |
| 211 | + ttfb = (time.perf_counter() - t0) * 1000 | |
| 212 | + # attente | |
| 213 | + try: | |
| 214 | + if isinstance(opts.wait_for, int): | |
| 215 | + await page.wait_for_timeout(min(opts.wait_for, timeout)) | |
| 216 | + elif isinstance(opts.wait_for, str) and opts.wait_for: | |
| 217 | + await page.wait_for_selector(opts.wait_for, timeout=timeout) | |
| 218 | + else: | |
| 219 | + try: | |
| 220 | + await page.wait_for_load_state("networkidle", timeout=min(timeout, 15_000)) | |
| 221 | + except Exception: | |
| 222 | + pass # networkidle jamais atteint (setTimeout infini) : on prend l'état courant | |
| 223 | + except Exception as e: | |
| 224 | + raise _pw_error(e) from None | |
| 225 | + await _run_actions(page, opts.actions, timeout) | |
| 226 | + final_url = page.url | |
| 227 | + try: | |
| 228 | + await check_url(final_url) | |
| 229 | + except SsrfError as e: | |
| 230 | + raise FetchError(e.code, e.message) from None | |
| 231 | + html = await page.content() | |
| 232 | + shot: bytes | None = None | |
| 233 | + if want_shot: | |
| 234 | + try: | |
| 235 | + shot = await page.screenshot(full_page=False, type="jpeg", quality=80) | |
| 236 | + except Exception as e: | |
| 237 | + log.info("browser.screenshot_failed", url=url, error=str(e)) | |
| 238 | + status = resp.status if resp else 200 | |
| 239 | + headers = {k.lower(): v for k, v in (await resp.all_headers()).items()} if resp else {} | |
| 240 | + headers.setdefault("content-type", "text/html") | |
| 241 | + total = (time.perf_counter() - t0) * 1000 | |
| 242 | + return FetchResult( | |
| 243 | + url=url, | |
| 244 | + final_url=final_url, | |
| 245 | + status=status, | |
| 246 | + headers=headers, | |
| 247 | + body=html.encode("utf-8"), | |
| 248 | + text=html, | |
| 249 | + mode="stealth" if stealth else "browser", | |
| 250 | + screenshot=shot, | |
| 251 | + charset="utf-8", | |
| 252 | + timings=Timings( | |
| 253 | + ttfb_ms=round(ttfb, 1), render_ms=round(total - ttfb, 1), fetch_ms=round(total, 1) | |
| 254 | + ), | |
| 255 | + ) | |
| 256 | + finally: | |
| 257 | + if page is not None: | |
| 258 | + try: | |
| 259 | + await page.close() | |
| 260 | + except Exception: | |
| 261 | + pass | |
| 262 | + if ctx is not None: | |
| 263 | + self._release(ctx) | |
| 264 | + self._slots.release() | |
| 265 | + | |
| 266 | + | |
| 267 | +def _pw_error(e: Exception) -> FetchError: | |
| 268 | + msg = f"{type(e).__name__}: {e}".split("\n")[0] | |
| 269 | + low = msg.lower() | |
| 270 | + if "timeout" in low: | |
| 271 | + return FetchError(ErrorCode.TIMEOUT_RENDER, msg) | |
| 272 | + if "err_name_not_resolved" in low: | |
| 273 | + return FetchError(ErrorCode.TIMEOUT_DNS, msg) | |
| 274 | + if "err_connection_refused" in low or "err_connection_timed_out" in low: | |
| 275 | + return FetchError(ErrorCode.TIMEOUT_CONNECT, msg) | |
| 276 | + if "err_cert" in low or "ssl" in low: | |
| 277 | + return FetchError(ErrorCode.SSL_ERROR, msg) | |
| 278 | + if "err_too_many_redirects" in low: | |
| 279 | + return FetchError(ErrorCode.NETWORK, msg) | |
| 280 | + if "err_aborted" in low or "frame was detached" in low: | |
| 281 | + return FetchError(ErrorCode.NETWORK, msg) | |
| 282 | + return FetchError(ErrorCode.NETWORK, msg) | |
| 283 | + | |
| 284 | + | |
| 285 | +async def _run_actions(page: Any, actions: list[BrowserAction], timeout: int) -> None: | |
| 286 | + for a in actions: | |
| 287 | + try: | |
| 288 | + if a.type == "click" and a.selector: | |
| 289 | + await page.click(a.selector, timeout=min(timeout, 10_000)) | |
| 290 | + elif a.type == "type" and a.selector: | |
| 291 | + await page.fill(a.selector, a.text or "", timeout=min(timeout, 10_000)) | |
| 292 | + elif a.type == "press": | |
| 293 | + await page.keyboard.press(a.key or "Enter") | |
| 294 | + elif a.type == "wait": | |
| 295 | + if a.selector: | |
| 296 | + await page.wait_for_selector(a.selector, timeout=min(timeout, 15_000)) | |
| 297 | + else: | |
| 298 | + await page.wait_for_timeout(min(a.ms or 1000, 15_000)) | |
| 299 | + elif a.type == "scroll": | |
| 300 | + dy = a.amount if a.direction == "down" else -a.amount | |
| 301 | + await page.mouse.wheel(0, dy) | |
| 302 | + await page.wait_for_timeout(300) | |
| 303 | + elif a.type == "evaluate" and a.script: | |
| 304 | + # sandbox Playwright uniquement ; jamais d'eval côté serveur | |
| 305 | + await page.evaluate(a.script) | |
| 306 | + except Exception as e: | |
| 307 | + log.info("browser.action_failed", action=a.type, selector=a.selector, error=str(e).split("\n")[0]) | |
| 308 | + | |
| 309 | + | |
| 310 | +_shared: BrowserPool | None = None | |
| 311 | + | |
| 312 | + | |
| 313 | +def shared_browser() -> BrowserPool: | |
| 314 | + global _shared | |
| 315 | + if _shared is None: | |
| 316 | + _shared = BrowserPool() | |
| 317 | + return _shared | |
added
trawls/core/fetcher/http_fast.py
+237 −0
@@ -0,0 +1,237 @@ | ||
| 1 | +"""Fetcher HTTP rapide : curl_cffi (impersonation TLS Chrome), fallback httpx. | |
| 2 | + | |
| 3 | +Invariants : | |
| 4 | +- redirections suivies manuellement (max 10) avec re-vérification SSRF de chaque saut ; | |
| 5 | +- téléchargement en streaming, coupé à `max_size_mb` → TOO_LARGE ; | |
| 6 | +- toute exception est convertie en FetchError typé. | |
| 7 | +""" | |
| 8 | + | |
| 9 | +from __future__ import annotations | |
| 10 | + | |
| 11 | +import asyncio | |
| 12 | +import time | |
| 13 | +from typing import Any | |
| 14 | +from urllib.parse import urljoin | |
| 15 | + | |
| 16 | +import structlog | |
| 17 | + | |
| 18 | +from trawls.config import get_settings | |
| 19 | +from trawls.core.antibot.identity import Identity, make_identity | |
| 20 | +from trawls.core.fetcher.base import FetchResult | |
| 21 | +from trawls.core.resilience.retry import FetchError | |
| 22 | +from trawls.core.security import SsrfError, check_url | |
| 23 | +from trawls.models import ErrorCode, ScrapeOptions, Timings | |
| 24 | + | |
| 25 | +log = structlog.get_logger(__name__) | |
| 26 | + | |
| 27 | +try: | |
| 28 | + from curl_cffi.requests import AsyncSession as _CurlSession | |
| 29 | + from curl_cffi.requests import errors as _curl_errors | |
| 30 | + | |
| 31 | + HAVE_CURL = True | |
| 32 | +except Exception: # pragma: no cover | |
| 33 | + HAVE_CURL = False | |
| 34 | + | |
| 35 | +import httpx | |
| 36 | + | |
| 37 | +_MAX_REDIRECTS = 10 | |
| 38 | + | |
| 39 | + | |
| 40 | +def _classify_exception(e: Exception) -> FetchError: | |
| 41 | + msg = f"{type(e).__name__}: {e}" | |
| 42 | + low = msg.lower() | |
| 43 | + if "ssl" in low or "certificate" in low or "tls" in low: | |
| 44 | + return FetchError(ErrorCode.SSL_ERROR, msg) | |
| 45 | + if "resolve" in low or "getaddrinfo" in low or "name or service" in low or "could not resolve" in low: | |
| 46 | + return FetchError(ErrorCode.TIMEOUT_DNS, msg) | |
| 47 | + if "timed out" in low or "timeout" in low: | |
| 48 | + if "connect" in low: | |
| 49 | + return FetchError(ErrorCode.TIMEOUT_CONNECT, msg) | |
| 50 | + return FetchError(ErrorCode.TIMEOUT_TTFB, msg) | |
| 51 | + if "connection refused" in low or "failed to connect" in low or "couldn't connect" in low: | |
| 52 | + return FetchError(ErrorCode.TIMEOUT_CONNECT, msg) | |
| 53 | + return FetchError(ErrorCode.NETWORK, msg) | |
| 54 | + | |
| 55 | + | |
| 56 | +def _status_error(status: int, headers: dict[str, str], url: str) -> FetchError | None: | |
| 57 | + if status == 429 or status == 503 and "retry-after" in headers: | |
| 58 | + ra = headers.get("retry-after") | |
| 59 | + retry_after = None | |
| 60 | + if ra and ra.isdigit(): | |
| 61 | + retry_after = min(float(ra), 60.0) | |
| 62 | + return FetchError( | |
| 63 | + ErrorCode.RATE_LIMITED, f"HTTP {status} sur {url}", retry_after_s=retry_after, http_status=status | |
| 64 | + ) | |
| 65 | + if status >= 500: | |
| 66 | + return FetchError(ErrorCode.HTTP_5XX, f"HTTP {status} sur {url}", http_status=status) | |
| 67 | + if status >= 400: | |
| 68 | + code = ErrorCode.HTTP_5XX if status == 408 else ErrorCode.HTTP_4XX | |
| 69 | + return FetchError(code, f"HTTP {status} sur {url}", http_status=status) | |
| 70 | + return None | |
| 71 | + | |
| 72 | + | |
| 73 | +class HttpFastFetcher: | |
| 74 | + mode = "http" | |
| 75 | + | |
| 76 | + def __init__(self) -> None: | |
| 77 | + self.settings = get_settings() | |
| 78 | + self._curl: Any = None | |
| 79 | + self._httpx: httpx.AsyncClient | None = None | |
| 80 | + self._lock = asyncio.Lock() | |
| 81 | + | |
| 82 | + async def _sessions(self) -> tuple[Any, httpx.AsyncClient]: | |
| 83 | + async with self._lock: | |
| 84 | + if HAVE_CURL and self._curl is None: | |
| 85 | + self._curl = _CurlSession(max_clients=self.settings.max_concurrency_global) | |
| 86 | + if self._httpx is None: | |
| 87 | + self._httpx = httpx.AsyncClient(follow_redirects=False, http2=True, timeout=30.0) | |
| 88 | + return self._curl, self._httpx | |
| 89 | + | |
| 90 | + async def close(self) -> None: | |
| 91 | + if self._curl is not None: | |
| 92 | + await self._curl.close() | |
| 93 | + self._curl = None | |
| 94 | + if self._httpx is not None: | |
| 95 | + await self._httpx.aclose() | |
| 96 | + self._httpx = None | |
| 97 | + | |
| 98 | + async def fetch(self, url: str, opts: ScrapeOptions, identity: Identity | None = None) -> FetchResult: | |
| 99 | + identity = identity or make_identity(opts.location) | |
| 100 | + headers = identity.headers(opts.headers) | |
| 101 | + cookies = {c.name: c.value for c in opts.cookies} or None | |
| 102 | + proxy = opts.proxy or (self.settings.proxies[0] if self.settings.proxies else None) | |
| 103 | + max_bytes = self.settings.max_size_mb * 1024 * 1024 | |
| 104 | + timeout_s = opts.timeout_ms / 1000 | |
| 105 | + t0 = time.perf_counter() | |
| 106 | + current = url | |
| 107 | + history: list[str] = [] | |
| 108 | + for _ in range(_MAX_REDIRECTS + 1): | |
| 109 | + try: | |
| 110 | + await check_url(current) | |
| 111 | + except SsrfError as e: | |
| 112 | + raise FetchError(e.code, e.message) from None | |
| 113 | + status, resp_headers, body, ttfb = await self._one( | |
| 114 | + current, headers, cookies, proxy, timeout_s, max_bytes, opts.verify_ssl, identity | |
| 115 | + ) | |
| 116 | + if status in (301, 302, 303, 307, 308) and resp_headers.get("location"): | |
| 117 | + nxt = urljoin(current, resp_headers["location"]) | |
| 118 | + if nxt in history or nxt == current: | |
| 119 | + raise FetchError(ErrorCode.NETWORK, f"boucle de redirection sur {nxt}") | |
| 120 | + history.append(current) | |
| 121 | + current = nxt | |
| 122 | + if len(history) > _MAX_REDIRECTS: | |
| 123 | + raise FetchError(ErrorCode.NETWORK, "trop de redirections") | |
| 124 | + continue | |
| 125 | + total = (time.perf_counter() - t0) * 1000 | |
| 126 | + return FetchResult( | |
| 127 | + url=url, | |
| 128 | + final_url=current, | |
| 129 | + status=status, | |
| 130 | + headers=resp_headers, | |
| 131 | + body=body, | |
| 132 | + mode="http", | |
| 133 | + timings=Timings(ttfb_ms=round(ttfb, 1), fetch_ms=round(total, 1)), | |
| 134 | + ) | |
| 135 | + raise FetchError(ErrorCode.NETWORK, "trop de redirections") | |
| 136 | + | |
| 137 | + async def _one( | |
| 138 | + self, | |
| 139 | + url: str, | |
| 140 | + headers: dict[str, str], | |
| 141 | + cookies: dict[str, str] | None, | |
| 142 | + proxy: str | None, | |
| 143 | + timeout_s: float, | |
| 144 | + max_bytes: int, | |
| 145 | + verify: bool, | |
| 146 | + identity: Identity, | |
| 147 | + ) -> tuple[int, dict[str, str], bytes, float]: | |
| 148 | + curl, hx = await self._sessions() | |
| 149 | + t0 = time.perf_counter() | |
| 150 | + if curl is not None: | |
| 151 | + try: | |
| 152 | + resp = await curl.request( | |
| 153 | + "GET", | |
| 154 | + url, | |
| 155 | + headers=headers, | |
| 156 | + cookies=cookies, | |
| 157 | + proxy=proxy, | |
| 158 | + timeout=(10, timeout_s), | |
| 159 | + allow_redirects=False, | |
| 160 | + impersonate="chrome", | |
| 161 | + verify=verify, | |
| 162 | + stream=True, | |
| 163 | + max_recv_speed=0, | |
| 164 | + ) | |
| 165 | + except _curl_errors.RequestsError as e: | |
| 166 | + raise _classify_exception(e) from None | |
| 167 | + except Exception as e: | |
| 168 | + raise _classify_exception(e) from None | |
| 169 | + ttfb = (time.perf_counter() - t0) * 1000 | |
| 170 | + try: | |
| 171 | + chunks: list[bytes] = [] | |
| 172 | + size = 0 | |
| 173 | + async for chunk in resp.aiter_content(): | |
| 174 | + size += len(chunk) | |
| 175 | + if size > max_bytes: | |
| 176 | + raise FetchError(ErrorCode.TOO_LARGE, f"réponse > {max_bytes // (1024 * 1024)} MB") | |
| 177 | + chunks.append(chunk) | |
| 178 | + if time.perf_counter() - t0 > timeout_s: | |
| 179 | + raise FetchError(ErrorCode.TIMEOUT_TTFB, "téléchargement trop lent (slowloris ?)") | |
| 180 | + except FetchError: | |
| 181 | + raise | |
| 182 | + except Exception as e: | |
| 183 | + raise _classify_exception(e) from None | |
| 184 | + finally: | |
| 185 | + try: | |
| 186 | + await resp.aclose() | |
| 187 | + except Exception: | |
| 188 | + pass | |
| 189 | + rh = {k.lower(): v for k, v in resp.headers.items()} | |
| 190 | + return int(resp.status_code), rh, b"".join(chunks), ttfb | |
| 191 | + # fallback httpx | |
| 192 | + try: | |
| 193 | + async with hx.stream("GET", url, headers=headers, cookies=cookies, timeout=timeout_s) as r: | |
| 194 | + ttfb = (time.perf_counter() - t0) * 1000 | |
| 195 | + chunks = [] | |
| 196 | + size = 0 | |
| 197 | + async for chunk in r.aiter_bytes(): | |
| 198 | + size += len(chunk) | |
| 199 | + if size > max_bytes: | |
| 200 | + raise FetchError(ErrorCode.TOO_LARGE, f"réponse > {max_bytes // (1024 * 1024)} MB") | |
| 201 | + chunks.append(chunk) | |
| 202 | + rh = {k.lower(): v for k, v in r.headers.items()} | |
| 203 | + return r.status_code, rh, b"".join(chunks), ttfb | |
| 204 | + except FetchError: | |
| 205 | + raise | |
| 206 | + except Exception as e: | |
| 207 | + raise _classify_exception(e) from None | |
| 208 | + | |
| 209 | + | |
| 210 | +_shared: HttpFastFetcher | None = None | |
| 211 | + | |
| 212 | + | |
| 213 | +def shared_http() -> HttpFastFetcher: | |
| 214 | + global _shared | |
| 215 | + if _shared is None: | |
| 216 | + _shared = HttpFastFetcher() | |
| 217 | + return _shared | |
| 218 | + | |
| 219 | + | |
| 220 | +async def fetch_text(url: str, timeout_s: float = 10.0, max_mb: int = 10) -> tuple[int, str]: | |
| 221 | + """Utilitaire léger (robots.txt, sitemaps) : renvoie (status, texte décodé).""" | |
| 222 | + from trawls.processors.encoding import decode_body | |
| 223 | + | |
| 224 | + opts = ScrapeOptions(timeout_ms=int(timeout_s * 1000)) | |
| 225 | + res = await shared_http().fetch(url, opts) | |
| 226 | + if len(res.body) > max_mb * 1024 * 1024: | |
| 227 | + raise FetchError(ErrorCode.TOO_LARGE, "trop volumineux") | |
| 228 | + body = res.body | |
| 229 | + if res.headers.get("content-type", "").endswith("gzip") or url.endswith(".gz") or body[:2] == b"\x1f\x8b": | |
| 230 | + import gzip | |
| 231 | + | |
| 232 | + try: | |
| 233 | + body = gzip.decompress(body) | |
| 234 | + except Exception: | |
| 235 | + pass | |
| 236 | + text, _ = decode_body(body, res.headers.get("content-type")) | |
| 237 | + return res.status, text | |
added
trawls/core/fetcher/strategy.py
+147 −0
@@ -0,0 +1,147 @@ | ||
| 1 | +"""Auto-détection + escalade : http → browser → stealth. Mode résolu mémorisé par host (1 h). | |
| 2 | + | |
| 3 | +Invariant : `fetch_auto` ne lève que FetchError ; toute autre exception est convertie. | |
| 4 | +""" | |
| 5 | + | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import time | |
| 9 | + | |
| 10 | +import structlog | |
| 11 | + | |
| 12 | +from trawls.config import get_settings | |
| 13 | +from trawls.core.antibot.detect import Detection, detect | |
| 14 | +from trawls.core.fetcher.base import FetchResult | |
| 15 | +from trawls.core.fetcher.browser import shared_browser | |
| 16 | +from trawls.core.fetcher.http_fast import shared_http | |
| 17 | +from trawls.core.resilience.retry import FetchError | |
| 18 | +from trawls.core.scheduler.dedup import host_of | |
| 19 | +from trawls.models import ErrorCode, ResolvedMode, ScrapeOptions | |
| 20 | +from trawls.processors.encoding import decode_body | |
| 21 | + | |
| 22 | +log = structlog.get_logger(__name__) | |
| 23 | + | |
| 24 | +_ORDER: list[ResolvedMode] = ["http", "browser", "stealth"] | |
| 25 | + | |
| 26 | + | |
| 27 | +class HostModeCache: | |
| 28 | + def __init__(self, ttl_s: int) -> None: | |
| 29 | + self.ttl_s = ttl_s | |
| 30 | + self._m: dict[str, tuple[float, ResolvedMode]] = {} | |
| 31 | + | |
| 32 | + def get(self, host: str) -> ResolvedMode | None: | |
| 33 | + hit = self._m.get(host) | |
| 34 | + if hit and time.monotonic() - hit[0] < self.ttl_s: | |
| 35 | + return hit[1] | |
| 36 | + return None | |
| 37 | + | |
| 38 | + def set(self, host: str, mode: ResolvedMode) -> None: | |
| 39 | + cur = self.get(host) | |
| 40 | + if cur is None or _ORDER.index(mode) >= _ORDER.index(cur): | |
| 41 | + self._m[host] = (time.monotonic(), mode) | |
| 42 | + | |
| 43 | + def snapshot(self) -> dict[str, str]: | |
| 44 | + return {h: m for h, (_, m) in self._m.items()} | |
| 45 | + | |
| 46 | + | |
| 47 | +host_modes = HostModeCache(get_settings().host_mode_cache_ttl_s) | |
| 48 | + | |
| 49 | + | |
| 50 | +def _decode(res: FetchResult) -> None: | |
| 51 | + if res.text is None and (res.is_html or res.is_text): | |
| 52 | + res.text, res.charset = decode_body(res.body, res.headers.get("content-type")) | |
| 53 | + | |
| 54 | + | |
| 55 | +async def _fetch_mode(url: str, opts: ScrapeOptions, mode: ResolvedMode) -> FetchResult: | |
| 56 | + try: | |
| 57 | + if mode == "http": | |
| 58 | + res = await shared_http().fetch(url, opts) | |
| 59 | + else: | |
| 60 | + res = await shared_browser().fetch(url, opts, stealth=(mode == "stealth")) | |
| 61 | + except FetchError: | |
| 62 | + raise | |
| 63 | + except Exception as e: # dernier filet : jamais d'exception brute | |
| 64 | + raise FetchError(ErrorCode.INTERNAL, f"{type(e).__name__}: {e}") from None | |
| 65 | + _decode(res) | |
| 66 | + return res | |
| 67 | + | |
| 68 | + | |
| 69 | +def _inspect(res: FetchResult) -> Detection: | |
| 70 | + if res.is_pdf or not res.is_html: | |
| 71 | + return Detection(None, 0.0, "non-HTML") | |
| 72 | + return detect(res.text or "", res.status, res.headers) | |
| 73 | + | |
| 74 | + | |
| 75 | +async def fetch_auto(url: str, opts: ScrapeOptions) -> tuple[FetchResult, list[str]]: | |
| 76 | + """Renvoie (résultat, trace d'escalade). Lève FetchError(BLOCKED) si tout échoue.""" | |
| 77 | + host = host_of(url) | |
| 78 | + trace: list[str] = [] | |
| 79 | + if opts.mode != "auto": | |
| 80 | + res = await _fetch_mode(url, opts, opts.mode) | |
| 81 | + d = _inspect(res) | |
| 82 | + trace.append(f"{opts.mode}: {d.reason}") | |
| 83 | + if d.blocked: | |
| 84 | + raise FetchError( | |
| 85 | + ErrorCode.BLOCKED, | |
| 86 | + f"anti-bot {d.kind} en mode forcé {opts.mode}", | |
| 87 | + http_status=res.status, | |
| 88 | + detection=d.kind, | |
| 89 | + ) | |
| 90 | + if res.status >= 400 and not d.kind: | |
| 91 | + _raise_status(res) | |
| 92 | + return res, trace | |
| 93 | + | |
| 94 | + start: ResolvedMode = host_modes.get(host) or "http" | |
| 95 | + if opts.actions or opts.wait_for is not None or "screenshot" in opts.formats: | |
| 96 | + start = "browser" if _ORDER.index(start) < 1 else start | |
| 97 | + idx = _ORDER.index(start) | |
| 98 | + last: FetchResult | None = None | |
| 99 | + last_det: Detection | None = None | |
| 100 | + while idx < len(_ORDER): | |
| 101 | + mode = _ORDER[idx] | |
| 102 | + try: | |
| 103 | + res = await _fetch_mode(url, opts, mode) | |
| 104 | + except FetchError as e: | |
| 105 | + # en http, un 403 « sec » ou une erreur TLS/empreinte peut cacher un anti-bot : on escalade | |
| 106 | + if ( | |
| 107 | + mode == "http" | |
| 108 | + and e.code in (ErrorCode.HTTP_4XX, ErrorCode.SSL_ERROR, ErrorCode.NETWORK) | |
| 109 | + and (e.info.details or {}).get("http_status") in (403, 429, 503, None) | |
| 110 | + ): | |
| 111 | + trace.append(f"{mode}: {e.code} → escalade") | |
| 112 | + idx += 1 | |
| 113 | + continue | |
| 114 | + raise | |
| 115 | + d = _inspect(res) | |
| 116 | + trace.append(f"{mode}: {d.reason}") | |
| 117 | + last, last_det = res, d | |
| 118 | + if d.kind is None: | |
| 119 | + if res.status >= 400: | |
| 120 | + _raise_status(res) | |
| 121 | + host_modes.set(host, mode) | |
| 122 | + return res, trace | |
| 123 | + if d.needs_browser and mode == "http": | |
| 124 | + idx += 1 | |
| 125 | + continue | |
| 126 | + if d.blocked: | |
| 127 | + idx = max(idx + 1, 2) if mode == "http" else idx + 1 | |
| 128 | + continue | |
| 129 | + # empty/spa déjà en navigateur : on accepte tel quel | |
| 130 | + host_modes.set(host, mode) | |
| 131 | + return res, trace | |
| 132 | + assert last is not None and last_det is not None | |
| 133 | + raise FetchError( | |
| 134 | + ErrorCode.BLOCKED, | |
| 135 | + f"bloqué après escalade complète ({last_det.kind}: {last_det.reason})", | |
| 136 | + http_status=last.status, | |
| 137 | + detection=last_det.kind, | |
| 138 | + trace=trace, | |
| 139 | + ) | |
| 140 | + | |
| 141 | + | |
| 142 | +def _raise_status(res: FetchResult) -> None: | |
| 143 | + from trawls.core.fetcher.http_fast import _status_error | |
| 144 | + | |
| 145 | + err = _status_error(res.status, res.headers, res.final_url) | |
| 146 | + if err: | |
| 147 | + raise err | |
added
trawls/core/resilience/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/core/resilience/breaker.py
+63 −0
@@ -0,0 +1,63 @@ | ||
| 1 | +"""Circuit breaker par host : 5 échecs consécutifs → ouvert 60 s → half-open (1 essai).""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import time | |
| 6 | +from dataclasses import dataclass, field | |
| 7 | + | |
| 8 | + | |
| 9 | +@dataclass | |
| 10 | +class _State: | |
| 11 | + failures: int = 0 | |
| 12 | + opened_at: float | None = None | |
| 13 | + half_open: bool = False | |
| 14 | + history: list[str] = field(default_factory=list) | |
| 15 | + | |
| 16 | + | |
| 17 | +class HostBreaker: | |
| 18 | + def __init__(self, threshold: int = 5, open_s: float = 60.0) -> None: | |
| 19 | + self.threshold = threshold | |
| 20 | + self.open_s = open_s | |
| 21 | + self._hosts: dict[str, _State] = {} | |
| 22 | + | |
| 23 | + def _st(self, host: str) -> _State: | |
| 24 | + s = self._hosts.get(host) | |
| 25 | + if s is None: | |
| 26 | + s = self._hosts[host] = _State() | |
| 27 | + return s | |
| 28 | + | |
| 29 | + def allow(self, host: str) -> bool: | |
| 30 | + s = self._st(host) | |
| 31 | + if s.opened_at is None: | |
| 32 | + return True | |
| 33 | + if time.monotonic() - s.opened_at >= self.open_s: | |
| 34 | + if not s.half_open: | |
| 35 | + s.half_open = True | |
| 36 | + return True | |
| 37 | + return False | |
| 38 | + return False | |
| 39 | + | |
| 40 | + def record_success(self, host: str) -> None: | |
| 41 | + s = self._st(host) | |
| 42 | + s.failures = 0 | |
| 43 | + s.opened_at = None | |
| 44 | + s.half_open = False | |
| 45 | + | |
| 46 | + def record_failure(self, host: str, code: str = "") -> None: | |
| 47 | + s = self._st(host) | |
| 48 | + s.failures += 1 | |
| 49 | + s.history = (s.history + [code])[-10:] | |
| 50 | + if s.half_open or s.failures >= self.threshold: | |
| 51 | + s.opened_at = time.monotonic() | |
| 52 | + s.half_open = False | |
| 53 | + | |
| 54 | + def snapshot(self) -> dict[str, dict[str, object]]: | |
| 55 | + return { | |
| 56 | + h: { | |
| 57 | + "failures": s.failures, | |
| 58 | + "open": s.opened_at is not None, | |
| 59 | + "half_open": s.half_open, | |
| 60 | + "last": s.history[-3:], | |
| 61 | + } | |
| 62 | + for h, s in self._hosts.items() | |
| 63 | + } | |
added
trawls/core/resilience/budget.py
+46 −0
@@ -0,0 +1,46 @@ | ||
| 1 | +"""Budget par job : temps, pages, tokens LLM. Dépassement → BUDGET_EXCEEDED (non récupérable).""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import time | |
| 6 | +from dataclasses import dataclass, field | |
| 7 | + | |
| 8 | + | |
| 9 | +@dataclass | |
| 10 | +class Budget: | |
| 11 | + max_pages: int | None = None | |
| 12 | + max_duration_s: float | None = None | |
| 13 | + max_tokens: int | None = None | |
| 14 | + pages: int = 0 | |
| 15 | + tokens: int = 0 | |
| 16 | + started: float = field(default_factory=time.monotonic) | |
| 17 | + | |
| 18 | + def elapsed(self) -> float: | |
| 19 | + return time.monotonic() - self.started | |
| 20 | + | |
| 21 | + def remaining_time(self) -> float | None: | |
| 22 | + if self.max_duration_s is None: | |
| 23 | + return None | |
| 24 | + return max(0.0, self.max_duration_s - self.elapsed()) | |
| 25 | + | |
| 26 | + def can_fetch_page(self) -> bool: | |
| 27 | + if self.max_pages is not None and self.pages >= self.max_pages: | |
| 28 | + return False | |
| 29 | + rt = self.remaining_time() | |
| 30 | + return rt is None or rt > 0 | |
| 31 | + | |
| 32 | + def consume_page(self) -> None: | |
| 33 | + self.pages += 1 | |
| 34 | + | |
| 35 | + def consume_tokens(self, n: int) -> bool: | |
| 36 | + self.tokens += n | |
| 37 | + return self.max_tokens is None or self.tokens <= self.max_tokens | |
| 38 | + | |
| 39 | + def why_exhausted(self) -> str: | |
| 40 | + if self.max_pages is not None and self.pages >= self.max_pages: | |
| 41 | + return f"max_pages={self.max_pages} atteint" | |
| 42 | + if self.max_duration_s is not None and self.elapsed() >= self.max_duration_s: | |
| 43 | + return f"max_duration_s={self.max_duration_s} atteint" | |
| 44 | + if self.max_tokens is not None and self.tokens > self.max_tokens: | |
| 45 | + return f"max_tokens={self.max_tokens} dépassé" | |
| 46 | + return "budget épuisé" | |
added
trawls/core/resilience/retry.py
+58 −0
@@ -0,0 +1,58 @@ | ||
| 1 | +"""Retry avec backoff exponentiel + jitter, uniquement sur erreurs récupérables. | |
| 2 | + | |
| 3 | +Invariant : max 3 tentatives, `base=1s × 2^n + jitter(0-500ms)`, respect de `Retry-After`. | |
| 4 | +""" | |
| 5 | + | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import asyncio | |
| 9 | +import random | |
| 10 | +from collections.abc import Awaitable, Callable | |
| 11 | +from typing import TypeVar | |
| 12 | + | |
| 13 | +import structlog | |
| 14 | + | |
| 15 | +from trawls.models import ErrorCode, ErrorInfo | |
| 16 | + | |
| 17 | +log = structlog.get_logger(__name__) | |
| 18 | +T = TypeVar("T") | |
| 19 | + | |
| 20 | + | |
| 21 | +class FetchError(Exception): | |
| 22 | + """Erreur typée transportant un ErrorInfo. Toute exception du fetch est convertie en FetchError.""" | |
| 23 | + | |
| 24 | + def __init__( | |
| 25 | + self, code: ErrorCode, message: str, retry_after_s: float | None = None, **details: object | |
| 26 | + ) -> None: | |
| 27 | + super().__init__(message) | |
| 28 | + self.info = ErrorInfo.make(code, message, **{k: v for k, v in details.items() if v is not None}) | |
| 29 | + self.retry_after_s = retry_after_s | |
| 30 | + | |
| 31 | + @property | |
| 32 | + def code(self) -> ErrorCode: | |
| 33 | + return self.info.code | |
| 34 | + | |
| 35 | + | |
| 36 | +async def with_retry( | |
| 37 | + fn: Callable[[], Awaitable[T]], | |
| 38 | + *, | |
| 39 | + max_attempts: int = 3, | |
| 40 | + base_s: float = 1.0, | |
| 41 | + url: str = "", | |
| 42 | +) -> tuple[T, int]: | |
| 43 | + """Exécute `fn` ; renvoie (résultat, tentatives). Relève FetchError avec attempts renseigné.""" | |
| 44 | + attempt = 0 | |
| 45 | + while True: | |
| 46 | + attempt += 1 | |
| 47 | + try: | |
| 48 | + return await fn(), attempt | |
| 49 | + except FetchError as e: | |
| 50 | + if not e.info.retryable or attempt >= max_attempts: | |
| 51 | + e.info.attempts = attempt | |
| 52 | + raise | |
| 53 | + delay = ( | |
| 54 | + e.retry_after_s if e.retry_after_s else base_s * (2 ** (attempt - 1)) + random.uniform(0, 0.5) | |
| 55 | + ) | |
| 56 | + delay = min(delay, 30.0) | |
| 57 | + log.info("retry", url=url, code=e.code, attempt=attempt, delay_s=round(delay, 2)) | |
| 58 | + await asyncio.sleep(delay) | |
added
trawls/core/scheduler/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/core/scheduler/dedup.py
+121 −0
@@ -0,0 +1,121 @@ | ||
| 1 | +"""Normalisation d'URL + simhash du contenu pour la déduplication. | |
| 2 | + | |
| 3 | +Invariants : `normalize_url` est idempotente ; deux URLs équivalentes (casse de l'hôte, fragment, | |
| 4 | +ordre des paramètres, paramètres de tracking) donnent la même chaîne. | |
| 5 | +""" | |
| 6 | + | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import hashlib | |
| 10 | +import re | |
| 11 | +from urllib.parse import parse_qsl, quote, unquote, urlencode, urljoin, urlsplit, urlunsplit | |
| 12 | + | |
| 13 | +_TRACKING = re.compile( | |
| 14 | + r"^(utm_.*|fbclid|gclid|dclid|msclkid|ref|ref_src|mc_.*|_ga|_gl|yclid|igshid|si)$", re.I | |
| 15 | +) | |
| 16 | +_DEFAULT_PORTS = {"http": "80", "https": "443"} | |
| 17 | + | |
| 18 | + | |
| 19 | +def normalize_url(url: str, base: str | None = None) -> str | None: | |
| 20 | + """Retourne l'URL normalisée ou None si inutilisable (schéma non http, vide…).""" | |
| 21 | + if not url: | |
| 22 | + return None | |
| 23 | + url = url.strip().replace("\n", "").replace("\r", "").replace("\t", "") | |
| 24 | + if base: | |
| 25 | + url = urljoin(base, url) | |
| 26 | + try: | |
| 27 | + p = urlsplit(url) | |
| 28 | + except ValueError: | |
| 29 | + return None | |
| 30 | + if p.scheme not in ("http", "https") or not p.hostname: | |
| 31 | + return None | |
| 32 | + host = p.hostname.lower().rstrip(".") | |
| 33 | + port = p.port | |
| 34 | + netloc = host if (port is None or str(port) == _DEFAULT_PORTS.get(p.scheme)) else f"{host}:{port}" | |
| 35 | + path = quote(unquote(p.path or "/"), safe="/:@!$&'()*+,;=-._~%") | |
| 36 | + path = re.sub(r"/{2,}", "/", path) | |
| 37 | + if path != "/" and path.endswith("/") and "." not in path.rsplit("/", 2)[-2]: | |
| 38 | + # on garde la barre finale telle quelle mais on la normalise plus bas : trailing slash canonique retiré | |
| 39 | + path = path.rstrip("/") or "/" | |
| 40 | + params = [(k, v) for k, v in parse_qsl(p.query, keep_blank_values=True) if not _TRACKING.match(k)] | |
| 41 | + params.sort() | |
| 42 | + query = urlencode(params, doseq=True) | |
| 43 | + return urlunsplit((p.scheme, netloc, path, query, "")) | |
| 44 | + | |
| 45 | + | |
| 46 | +def host_of(url: str) -> str: | |
| 47 | + try: | |
| 48 | + return (urlsplit(url).hostname or "").lower() | |
| 49 | + except ValueError: | |
| 50 | + return "" | |
| 51 | + | |
| 52 | + | |
| 53 | +def registrable_domain(host: str) -> str: | |
| 54 | + """Approximation sans PSL : deux derniers labels (trois si TLD à 2 lettres + second niveau court).""" | |
| 55 | + parts = host.split(".") | |
| 56 | + if len(parts) <= 2: | |
| 57 | + return host | |
| 58 | + if len(parts[-1]) == 2 and len(parts[-2]) <= 3: | |
| 59 | + return ".".join(parts[-3:]) | |
| 60 | + return ".".join(parts[-2:]) | |
| 61 | + | |
| 62 | + | |
| 63 | +def same_site(a: str, b: str, allow_subdomains: bool) -> bool: | |
| 64 | + ha, hb = host_of(a), host_of(b) | |
| 65 | + if ha == hb: | |
| 66 | + return True | |
| 67 | + if allow_subdomains: | |
| 68 | + return registrable_domain(ha) == registrable_domain(hb) | |
| 69 | + return False | |
| 70 | + | |
| 71 | + | |
| 72 | +# --- simhash ----------------------------------------------------------------- | |
| 73 | + | |
| 74 | +_TOKEN = re.compile(r"\w+", re.U) | |
| 75 | + | |
| 76 | + | |
| 77 | +def _shingles(text: str, n: int = 3) -> list[str]: | |
| 78 | + toks = _TOKEN.findall(text.lower()) | |
| 79 | + if len(toks) < n: | |
| 80 | + return [" ".join(toks)] if toks else [] | |
| 81 | + return [" ".join(toks[i : i + n]) for i in range(len(toks) - n + 1)] | |
| 82 | + | |
| 83 | + | |
| 84 | +def simhash(text: str, bits: int = 64) -> int: | |
| 85 | + v = [0] * bits | |
| 86 | + for sh in _shingles(text): | |
| 87 | + h = int.from_bytes(hashlib.blake2b(sh.encode(), digest_size=8).digest(), "big") | |
| 88 | + for i in range(bits): | |
| 89 | + v[i] += 1 if (h >> i) & 1 else -1 | |
| 90 | + out = 0 | |
| 91 | + for i in range(bits): | |
| 92 | + if v[i] > 0: | |
| 93 | + out |= 1 << i | |
| 94 | + return out | |
| 95 | + | |
| 96 | + | |
| 97 | +def hamming(a: int, b: int) -> int: | |
| 98 | + return bin(a ^ b).count("1") | |
| 99 | + | |
| 100 | + | |
| 101 | +class ContentDeduper: | |
| 102 | + """Mémorise les simhash vus ; `is_duplicate` renvoie True si distance ≤ seuil.""" | |
| 103 | + | |
| 104 | + def __init__(self, threshold: int = 3) -> None: | |
| 105 | + self.threshold = threshold | |
| 106 | + self._seen: list[tuple[int, str]] = [] | |
| 107 | + self._exact: set[str] = set() | |
| 108 | + | |
| 109 | + def is_duplicate(self, text: str, url: str) -> str | None: | |
| 110 | + if not text or len(text) < 200: | |
| 111 | + return None | |
| 112 | + digest = hashlib.sha1(text.encode()).hexdigest() | |
| 113 | + if digest in self._exact: | |
| 114 | + return "exact" | |
| 115 | + h = simhash(text) | |
| 116 | + for other, other_url in self._seen: | |
| 117 | + if other_url != url and hamming(h, other) <= self.threshold: | |
| 118 | + return other_url | |
| 119 | + self._exact.add(digest) | |
| 120 | + self._seen.append((h, url)) | |
| 121 | + return None | |
added
trawls/core/scheduler/politeness.py
+56 −0
@@ -0,0 +1,56 @@ | ||
| 1 | +"""Politesse : limite de concurrence par host + délai minimal entre requêtes (crawl-delay).""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import asyncio | |
| 6 | +import time | |
| 7 | +from collections import defaultdict | |
| 8 | + | |
| 9 | + | |
| 10 | +class HostLimiter: | |
| 11 | + def __init__(self, per_host: int = 4, global_limit: int = 16, default_delay_s: float = 0.0) -> None: | |
| 12 | + self.per_host = per_host | |
| 13 | + self.default_delay_s = default_delay_s | |
| 14 | + self._sem: dict[str, asyncio.Semaphore] = {} | |
| 15 | + self._global = asyncio.Semaphore(global_limit) | |
| 16 | + self._next_ok: dict[str, float] = defaultdict(float) | |
| 17 | + self._delay: dict[str, float] = {} | |
| 18 | + self._lock = asyncio.Lock() | |
| 19 | + | |
| 20 | + def set_delay(self, host: str, delay_s: float) -> None: | |
| 21 | + self._delay[host] = max(self._delay.get(host, 0.0), delay_s) | |
| 22 | + | |
| 23 | + def _sem_for(self, host: str) -> asyncio.Semaphore: | |
| 24 | + s = self._sem.get(host) | |
| 25 | + if s is None: | |
| 26 | + s = self._sem[host] = asyncio.Semaphore(self.per_host) | |
| 27 | + return s | |
| 28 | + | |
| 29 | + async def acquire(self, host: str) -> None: | |
| 30 | + await self._global.acquire() | |
| 31 | + await self._sem_for(host).acquire() | |
| 32 | + delay = self._delay.get(host, self.default_delay_s) | |
| 33 | + if delay > 0: | |
| 34 | + async with self._lock: | |
| 35 | + now = time.monotonic() | |
| 36 | + wait = max(0.0, self._next_ok[host] - now) | |
| 37 | + self._next_ok[host] = max(now, self._next_ok[host]) + delay | |
| 38 | + if wait > 0: | |
| 39 | + await asyncio.sleep(wait) | |
| 40 | + | |
| 41 | + def release(self, host: str) -> None: | |
| 42 | + self._sem_for(host).release() | |
| 43 | + self._global.release() | |
| 44 | + | |
| 45 | + class _Ctx: | |
| 46 | + def __init__(self, lim: HostLimiter, host: str) -> None: | |
| 47 | + self.lim, self.host = lim, host | |
| 48 | + | |
| 49 | + async def __aenter__(self) -> None: | |
| 50 | + await self.lim.acquire(self.host) | |
| 51 | + | |
| 52 | + async def __aexit__(self, *exc: object) -> None: | |
| 53 | + self.lim.release(self.host) | |
| 54 | + | |
| 55 | + def slot(self, host: str) -> HostLimiter._Ctx: | |
| 56 | + return HostLimiter._Ctx(self, host) | |
added
trawls/core/scheduler/robots.py
+84 −0
@@ -0,0 +1,84 @@ | ||
| 1 | +"""Parser + cache robots.txt (par host, TTL 1 h). Jamais d'exception : en cas d'erreur réseau, | |
| 2 | +on considère le site autorisé (robots absent) mais on le trace. | |
| 3 | +""" | |
| 4 | + | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import asyncio | |
| 8 | +import time | |
| 9 | +from urllib.parse import urlsplit | |
| 10 | +from urllib.robotparser import RobotFileParser | |
| 11 | + | |
| 12 | +import structlog | |
| 13 | + | |
| 14 | +log = structlog.get_logger(__name__) | |
| 15 | + | |
| 16 | +_UA = "Trawls" | |
| 17 | + | |
| 18 | + | |
| 19 | +class RobotsCache: | |
| 20 | + def __init__(self, ttl_s: int = 3600) -> None: | |
| 21 | + self.ttl_s = ttl_s | |
| 22 | + self._cache: dict[str, tuple[float, RobotFileParser | None, list[str], float | None]] = {} | |
| 23 | + self._locks: dict[str, asyncio.Lock] = {} | |
| 24 | + | |
| 25 | + def _key(self, url: str) -> str: | |
| 26 | + p = urlsplit(url) | |
| 27 | + return f"{p.scheme}://{p.netloc}" | |
| 28 | + | |
| 29 | + async def _fetch(self, origin: str) -> tuple[RobotFileParser | None, list[str], float | None]: | |
| 30 | + from trawls.core.fetcher.http_fast import fetch_text | |
| 31 | + | |
| 32 | + try: | |
| 33 | + status, text = await fetch_text(origin + "/robots.txt", timeout_s=8) | |
| 34 | + except Exception as e: # réseau : on ne bloque pas le crawl | |
| 35 | + log.info("robots.fetch_failed", origin=origin, error=str(e)) | |
| 36 | + return None, [], None | |
| 37 | + if status != 200 or not text: | |
| 38 | + return None, [], None | |
| 39 | + rp = RobotFileParser() | |
| 40 | + rp.parse(text.splitlines()) | |
| 41 | + sitemaps = [ | |
| 42 | + ln.split(":", 1)[1].strip() for ln in text.splitlines() if ln.lower().startswith("sitemap:") | |
| 43 | + ] | |
| 44 | + delay = rp.crawl_delay(_UA) or rp.crawl_delay("*") | |
| 45 | + return rp, sitemaps, float(delay) if delay else None | |
| 46 | + | |
| 47 | + async def get(self, url: str) -> tuple[RobotFileParser | None, list[str], float | None]: | |
| 48 | + origin = self._key(url) | |
| 49 | + now = time.monotonic() | |
| 50 | + hit = self._cache.get(origin) | |
| 51 | + if hit and now - hit[0] < self.ttl_s: | |
| 52 | + return hit[1], hit[2], hit[3] | |
| 53 | + lock = self._locks.setdefault(origin, asyncio.Lock()) | |
| 54 | + async with lock: | |
| 55 | + hit = self._cache.get(origin) | |
| 56 | + if hit and now - hit[0] < self.ttl_s: | |
| 57 | + return hit[1], hit[2], hit[3] | |
| 58 | + rp, sm, delay = await self._fetch(origin) | |
| 59 | + self._cache[origin] = (time.monotonic(), rp, sm, delay) | |
| 60 | + return rp, sm, delay | |
| 61 | + | |
| 62 | + async def allowed(self, url: str) -> bool: | |
| 63 | + rp, _, _ = await self.get(url) | |
| 64 | + if rp is None: | |
| 65 | + return True | |
| 66 | + try: | |
| 67 | + return ( | |
| 68 | + rp.can_fetch(_UA, url) or rp.can_fetch("*", url) | |
| 69 | + if rp.default_entry is None | |
| 70 | + else rp.can_fetch(_UA, url) | |
| 71 | + ) | |
| 72 | + except Exception: | |
| 73 | + return True | |
| 74 | + | |
| 75 | + async def sitemaps(self, url: str) -> list[str]: | |
| 76 | + _, sm, _ = await self.get(url) | |
| 77 | + return sm | |
| 78 | + | |
| 79 | + async def crawl_delay(self, url: str) -> float | None: | |
| 80 | + _, _, d = await self.get(url) | |
| 81 | + return d | |
| 82 | + | |
| 83 | + | |
| 84 | +robots_cache = RobotsCache() | |
added
trawls/core/scrape.py
+287 −0
@@ -0,0 +1,287 @@ | ||
| 1 | +"""Pipeline d'une page : garde SSRF → robots → breaker → fetch auto (retry) → PDF|HTML → MD → | |
| 2 | +métadonnées → liens → chunks → extraction → screenshot. Ne lève JAMAIS : renvoie un PageResult. | |
| 3 | +""" | |
| 4 | + | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import hashlib | |
| 8 | +import time | |
| 9 | +from datetime import UTC, datetime | |
| 10 | +from typing import Any | |
| 11 | + | |
| 12 | +import structlog | |
| 13 | + | |
| 14 | +from trawls.config import get_settings | |
| 15 | +from trawls.core.fetcher.base import FetchResult | |
| 16 | +from trawls.core.fetcher.strategy import fetch_auto | |
| 17 | +from trawls.core.resilience.breaker import HostBreaker | |
| 18 | +from trawls.core.resilience.retry import FetchError, with_retry | |
| 19 | +from trawls.core.scheduler.dedup import host_of, normalize_url | |
| 20 | +from trawls.core.scheduler.robots import robots_cache | |
| 21 | +from trawls.core.security import SsrfError, check_url | |
| 22 | +from trawls.extract.css import extract_css | |
| 23 | +from trawls.extract.llm import extract_llm | |
| 24 | +from trawls.models import ChunkOptions, ErrorCode, ErrorInfo, PageMetadata, PageResult, ScrapeOptions, Timings | |
| 25 | +from trawls.processors.chunker import chunk as make_chunks | |
| 26 | +from trawls.processors.html_to_md import html_to_markdown | |
| 27 | +from trawls.processors.links import extract_links | |
| 28 | +from trawls.processors.pdf import pdf_to_markdown | |
| 29 | +from trawls.processors.structured.metadata import extract_metadata | |
| 30 | + | |
| 31 | +log = structlog.get_logger(__name__) | |
| 32 | +breaker = HostBreaker() | |
| 33 | + | |
| 34 | +# cache résultats en mémoire (clé = url normalisée + options structurantes) — simple, borné | |
| 35 | +_CACHE: dict[str, tuple[float, PageResult]] = {} | |
| 36 | +_CACHE_MAX = 2000 | |
| 37 | + | |
| 38 | + | |
| 39 | +def _cache_key(url: str, opts: ScrapeOptions) -> str: | |
| 40 | + sig = f"{url}|{opts.only_main_content}|{sorted(opts.formats)}|{opts.mode}|{opts.include_tags}|{opts.exclude_tags}|{opts.wait_for}|{[a.model_dump() for a in opts.actions]}|{opts.citations}" | |
| 41 | + return hashlib.sha1(sig.encode()).hexdigest() | |
| 42 | + | |
| 43 | + | |
| 44 | +def _cache_get(key: str, max_age_s: int) -> PageResult | None: | |
| 45 | + hit = _CACHE.get(key) | |
| 46 | + if hit and time.time() - hit[0] < max_age_s: | |
| 47 | + r = hit[1].model_copy(deep=True) | |
| 48 | + r.from_cache = True | |
| 49 | + return r | |
| 50 | + return None | |
| 51 | + | |
| 52 | + | |
| 53 | +def _cache_put(key: str, res: PageResult) -> None: | |
| 54 | + if len(_CACHE) >= _CACHE_MAX: | |
| 55 | + oldest = sorted(_CACHE.items(), key=lambda kv: kv[1][0])[: _CACHE_MAX // 10] | |
| 56 | + for k, _ in oldest: | |
| 57 | + _CACHE.pop(k, None) | |
| 58 | + # copie profonde : l'appelant (crawl, API) mute ensuite `links`/`markdown` sur son exemplaire | |
| 59 | + _CACHE[key] = (time.time(), res.model_copy(deep=True)) | |
| 60 | + | |
| 61 | + | |
| 62 | +def _save_blob(data: bytes, ext: str) -> str: | |
| 63 | + s = get_settings() | |
| 64 | + name = hashlib.sha1(data).hexdigest()[:24] + ext | |
| 65 | + p = s.blob_dir / name | |
| 66 | + if not p.exists(): | |
| 67 | + p.write_bytes(data) | |
| 68 | + return f"{s.public_url.rstrip('/')}/v1/blobs/{name}" | |
| 69 | + | |
| 70 | + | |
| 71 | +async def scrape( | |
| 72 | + url: str, opts: ScrapeOptions | None = None, *, depth: int = 0, job_id: str | None = None | |
| 73 | +) -> PageResult: | |
| 74 | + opts = opts or ScrapeOptions() | |
| 75 | + t_start = time.perf_counter() | |
| 76 | + log_ = log.bind(url=url, job_id=job_id) | |
| 77 | + norm = normalize_url(url) | |
| 78 | + if not norm: | |
| 79 | + return PageResult.failed( | |
| 80 | + url, ErrorInfo.make(ErrorCode.INVALID_URL, "URL invalide ou schéma non http(s)"), depth=depth | |
| 81 | + ) | |
| 82 | + host = host_of(norm) | |
| 83 | + | |
| 84 | + # cache | |
| 85 | + key = _cache_key(norm, opts) | |
| 86 | + if opts.cache == "use" and not opts.extract: | |
| 87 | + hit = _cache_get(key, opts.max_age_s) | |
| 88 | + if hit is not None: | |
| 89 | + hit.depth = depth | |
| 90 | + return hit | |
| 91 | + | |
| 92 | + # SSRF (résolution DNS avant la requête) | |
| 93 | + try: | |
| 94 | + await check_url(norm) | |
| 95 | + except SsrfError as e: | |
| 96 | + return PageResult.failed(url, ErrorInfo.make(e.code, e.message), depth=depth) | |
| 97 | + | |
| 98 | + # robots | |
| 99 | + if opts.respect_robots: | |
| 100 | + try: | |
| 101 | + if not await robots_cache.allowed(norm): | |
| 102 | + return PageResult.failed( | |
| 103 | + url, ErrorInfo.make(ErrorCode.ROBOTS_DISALLOWED, "interdit par robots.txt"), depth=depth | |
| 104 | + ) | |
| 105 | + except Exception as e: | |
| 106 | + log_.info("robots.error", error=str(e)) | |
| 107 | + else: | |
| 108 | + log_.warning("robots.ignored", reason="respect_robots=false demandé") | |
| 109 | + if not opts.verify_ssl: | |
| 110 | + log_.warning("ssl.unverified", reason="verify_ssl=false demandé") | |
| 111 | + | |
| 112 | + # breaker | |
| 113 | + if not breaker.allow(host): | |
| 114 | + return PageResult.failed( | |
| 115 | + url, ErrorInfo.make(ErrorCode.CIRCUIT_OPEN, f"circuit ouvert pour {host}"), depth=depth | |
| 116 | + ) | |
| 117 | + | |
| 118 | + # fetch avec retry | |
| 119 | + trace: list[str] = [] | |
| 120 | + try: | |
| 121 | + | |
| 122 | + async def _do() -> tuple[FetchResult, list[str]]: | |
| 123 | + return await fetch_auto(norm, opts) | |
| 124 | + | |
| 125 | + (res, trace), attempts = await with_retry(_do, url=norm) | |
| 126 | + except FetchError as e: | |
| 127 | + breaker.record_failure(host, str(e.code)) | |
| 128 | + info = e.info | |
| 129 | + if trace: | |
| 130 | + info.details = {**(info.details or {}), "trace": trace} | |
| 131 | + log_.info("scrape.failed", code=info.code, attempts=info.attempts, message=info.message) | |
| 132 | + return PageResult.failed( | |
| 133 | + url, info, depth=depth, timings=Timings(total_ms=round((time.perf_counter() - t_start) * 1000, 1)) | |
| 134 | + ) | |
| 135 | + except Exception as e: # dernier filet | |
| 136 | + breaker.record_failure(host, "INTERNAL") | |
| 137 | + log_.error("scrape.internal", error=f"{type(e).__name__}: {e}") | |
| 138 | + return PageResult.failed( | |
| 139 | + url, ErrorInfo.make(ErrorCode.INTERNAL, f"{type(e).__name__}: {e}"), depth=depth | |
| 140 | + ) | |
| 141 | + breaker.record_success(host) | |
| 142 | + | |
| 143 | + # traitement | |
| 144 | + t_proc = time.perf_counter() | |
| 145 | + try: | |
| 146 | + page = await _process(url, res, opts, depth, attempts) | |
| 147 | + except Exception as e: | |
| 148 | + log_.error("process.failed", error=f"{type(e).__name__}: {e}") | |
| 149 | + page = PageResult.failed( | |
| 150 | + url, | |
| 151 | + ErrorInfo.make(ErrorCode.PARSE_FAILED, f"{type(e).__name__}: {e}"), | |
| 152 | + mode=res.mode, | |
| 153 | + http_status=res.status, | |
| 154 | + depth=depth, | |
| 155 | + ) | |
| 156 | + page.final_url = res.final_url | |
| 157 | + page.timings.process_ms = round((time.perf_counter() - t_proc) * 1000, 1) | |
| 158 | + page.timings.total_ms = round((time.perf_counter() - t_start) * 1000, 1) | |
| 159 | + if trace: | |
| 160 | + page.metadata.extra = {**page.metadata.extra, "fetch_trace": trace} | |
| 161 | + if page.status == "ok" and opts.cache != "bypass" and not opts.extract: | |
| 162 | + _cache_put(key, page) | |
| 163 | + return page | |
| 164 | + | |
| 165 | + | |
| 166 | +async def _process(url: str, res: FetchResult, opts: ScrapeOptions, depth: int, attempts: int) -> PageResult: | |
| 167 | + formats = set(opts.formats) | |
| 168 | + page = PageResult( | |
| 169 | + url=url, | |
| 170 | + final_url=res.final_url, | |
| 171 | + status="ok", | |
| 172 | + http_status=res.status, | |
| 173 | + fetch_mode_used=res.mode, | |
| 174 | + timings=res.timings, | |
| 175 | + depth=depth, | |
| 176 | + fetched_at=datetime.now(UTC), | |
| 177 | + ) | |
| 178 | + markdown = "" | |
| 179 | + html_text = res.text or "" | |
| 180 | + if res.is_pdf: | |
| 181 | + size_mb = len(res.body) / (1024 * 1024) | |
| 182 | + if size_mb > get_settings().max_size_mb: | |
| 183 | + return PageResult.failed( | |
| 184 | + url, | |
| 185 | + ErrorInfo.make(ErrorCode.TOO_LARGE, f"PDF {size_mb:.1f} MB"), | |
| 186 | + mode=res.mode, | |
| 187 | + http_status=res.status, | |
| 188 | + depth=depth, | |
| 189 | + ) | |
| 190 | + try: | |
| 191 | + pdf = pdf_to_markdown(res.body) | |
| 192 | + except ValueError as e: | |
| 193 | + return PageResult.failed( | |
| 194 | + url, | |
| 195 | + ErrorInfo.make(ErrorCode.PARSE_FAILED, str(e)), | |
| 196 | + mode=res.mode, | |
| 197 | + http_status=res.status, | |
| 198 | + depth=depth, | |
| 199 | + ) | |
| 200 | + markdown = pdf.markdown | |
| 201 | + page.metadata = PageMetadata( | |
| 202 | + title=pdf.title, | |
| 203 | + author=pdf.author, | |
| 204 | + published_at=pdf.created, | |
| 205 | + modified_at=pdf.modified, | |
| 206 | + content_type="application/pdf", | |
| 207 | + is_pdf=True, | |
| 208 | + pdf_pages=pdf.pages, | |
| 209 | + pdf_is_scanned=pdf.is_scanned, | |
| 210 | + word_count=len(markdown.split()), | |
| 211 | + extra={ | |
| 212 | + "tables": pdf.tables, | |
| 213 | + "ocr_pages": pdf.ocr_pages, | |
| 214 | + **({"warnings": pdf.warnings} if pdf.warnings else {}), | |
| 215 | + }, | |
| 216 | + ) | |
| 217 | + if "raw_html" in formats or "html" in formats: | |
| 218 | + page.raw_html = None | |
| 219 | + elif res.is_html: | |
| 220 | + md = html_to_markdown( | |
| 221 | + html_text, | |
| 222 | + res.final_url, | |
| 223 | + only_main_content=opts.only_main_content, | |
| 224 | + include_tags=opts.include_tags, | |
| 225 | + exclude_tags=opts.exclude_tags, | |
| 226 | + remove_base64_images=opts.remove_base64_images, | |
| 227 | + citations=opts.citations, | |
| 228 | + ) | |
| 229 | + markdown = md.markdown | |
| 230 | + page.metadata = extract_metadata( | |
| 231 | + html_text, res.final_url, res.content_type or "text/html", res.charset | |
| 232 | + ) | |
| 233 | + page.metadata.word_count = len(markdown.split()) | |
| 234 | + page.metadata.extra = {**page.metadata.extra, "main_content": md.main_selector_hint} | |
| 235 | + if "html" in formats: | |
| 236 | + page.html = md.clean_html | |
| 237 | + if "raw_html" in formats: | |
| 238 | + page.raw_html = html_text | |
| 239 | + if "links" in formats or True: # liens toujours calculés (crawl), exposés si demandés | |
| 240 | + page.links = extract_links(html_text, res.final_url) | |
| 241 | + elif res.is_text: | |
| 242 | + text, cs = (res.text or ""), res.charset | |
| 243 | + ct = res.content_type | |
| 244 | + markdown = ( | |
| 245 | + f"```{'json' if 'json' in ct else ''}\n{text.strip()}\n```\n" | |
| 246 | + if ("json" in ct or "xml" in ct) | |
| 247 | + else text | |
| 248 | + ) | |
| 249 | + page.metadata = PageMetadata(content_type=ct, charset=cs, word_count=len(text.split())) | |
| 250 | + if "raw_html" in formats: | |
| 251 | + page.raw_html = text | |
| 252 | + else: | |
| 253 | + return PageResult.failed( | |
| 254 | + url, | |
| 255 | + ErrorInfo.make( | |
| 256 | + ErrorCode.UNSUPPORTED_CONTENT, f"content-type non supporté: {res.content_type or 'inconnu'}" | |
| 257 | + ), | |
| 258 | + mode=res.mode, | |
| 259 | + http_status=res.status, | |
| 260 | + depth=depth, | |
| 261 | + ) | |
| 262 | + | |
| 263 | + if "markdown" in formats or not formats: | |
| 264 | + page.markdown = markdown | |
| 265 | + # page.links est toujours rempli (nécessaire au crawl) ; l'API le masque si "links" n'est pas demandé | |
| 266 | + if "chunks" in formats or opts.chunk: | |
| 267 | + page.chunks = make_chunks(markdown, res.final_url, opts.chunk or ChunkOptions()) | |
| 268 | + if res.screenshot and "screenshot" in formats: | |
| 269 | + page.screenshot_url = _save_blob(res.screenshot, ".jpg") | |
| 270 | + if opts.extract: | |
| 271 | + data: dict[str, Any] | None = None | |
| 272 | + errors: dict[str, str] = {} | |
| 273 | + if opts.extract.mode == "css" and opts.extract.css: | |
| 274 | + data, errors = extract_css(html_text, opts.extract.css, res.final_url) | |
| 275 | + elif opts.extract.mode == "llm" and opts.extract.schema_: | |
| 276 | + data, err, _tok = await extract_llm( | |
| 277 | + markdown, opts.extract.schema_, opts.extract.prompt, res.final_url | |
| 278 | + ) | |
| 279 | + if err: | |
| 280 | + errors["_llm"] = err.message | |
| 281 | + page.json_data = {"data": data, "errors": errors} if (data is not None or errors) else None | |
| 282 | + elif "json" in formats: | |
| 283 | + page.json_data = {"jsonld": page.metadata.jsonld} | |
| 284 | + page.error = None | |
| 285 | + if attempts > 1: | |
| 286 | + page.metadata.extra = {**page.metadata.extra, "attempts": attempts} | |
| 287 | + return page | |
added
trawls/core/security.py
+81 −0
@@ -0,0 +1,81 @@ | ||
| 1 | +"""Garde SSRF : refuse IP privées/loopback/link-local et schémas non http(s). | |
| 2 | + | |
| 3 | +Invariant : `check_url` résout le DNS AVANT toute requête ; le fetcher rappelle `check_url` | |
| 4 | +sur chaque URL finale après redirection. | |
| 5 | +""" | |
| 6 | + | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import asyncio | |
| 10 | +import ipaddress | |
| 11 | +import socket | |
| 12 | +from urllib.parse import urlsplit | |
| 13 | + | |
| 14 | +from trawls.config import get_settings | |
| 15 | +from trawls.models import ErrorCode | |
| 16 | + | |
| 17 | + | |
| 18 | +class SsrfError(Exception): | |
| 19 | + def __init__(self, code: ErrorCode, message: str) -> None: | |
| 20 | + super().__init__(message) | |
| 21 | + self.code = code | |
| 22 | + self.message = message | |
| 23 | + | |
| 24 | + | |
| 25 | +_BLOCKED_HOSTS = {"localhost", "metadata.google.internal", "169.254.169.254"} | |
| 26 | + | |
| 27 | + | |
| 28 | +def _ip_is_forbidden(ip: str) -> bool: | |
| 29 | + try: | |
| 30 | + a = ipaddress.ip_address(ip) | |
| 31 | + except ValueError: | |
| 32 | + return True | |
| 33 | + return ( | |
| 34 | + a.is_private | |
| 35 | + or a.is_loopback | |
| 36 | + or a.is_link_local | |
| 37 | + or a.is_multicast | |
| 38 | + or a.is_reserved | |
| 39 | + or a.is_unspecified | |
| 40 | + or (a.version == 6 and a.ipv4_mapped is not None and _ip_is_forbidden(str(a.ipv4_mapped))) | |
| 41 | + ) | |
| 42 | + | |
| 43 | + | |
| 44 | +async def resolve_host(host: str, timeout: float = 5.0) -> list[str]: | |
| 45 | + loop = asyncio.get_running_loop() | |
| 46 | + try: | |
| 47 | + infos = await asyncio.wait_for(loop.getaddrinfo(host, None, type=socket.SOCK_STREAM), timeout) | |
| 48 | + except TimeoutError as e: # noqa: F841 | |
| 49 | + raise SsrfError(ErrorCode.TIMEOUT_DNS, f"DNS timeout pour {host}") from None | |
| 50 | + except socket.gaierror as e: | |
| 51 | + raise SsrfError(ErrorCode.TIMEOUT_DNS, f"DNS échoué pour {host}: {e}") from None | |
| 52 | + return sorted({str(i[4][0]) for i in infos}) | |
| 53 | + | |
| 54 | + | |
| 55 | +async def check_url(url: str) -> str: | |
| 56 | + """Valide une URL cible. Retourne l'URL (inchangée) ou lève SsrfError.""" | |
| 57 | + settings = get_settings() | |
| 58 | + try: | |
| 59 | + parts = urlsplit(url) | |
| 60 | + except ValueError as e: | |
| 61 | + raise SsrfError(ErrorCode.INVALID_URL, f"URL invalide: {e}") from None | |
| 62 | + if parts.scheme not in ("http", "https"): | |
| 63 | + raise SsrfError(ErrorCode.INVALID_URL, f"schéma non supporté: {parts.scheme!r}") | |
| 64 | + host = (parts.hostname or "").rstrip(".").lower() | |
| 65 | + if not host: | |
| 66 | + raise SsrfError(ErrorCode.INVALID_URL, "hôte manquant") | |
| 67 | + if settings.allow_private_targets: | |
| 68 | + return url | |
| 69 | + if host in _BLOCKED_HOSTS or host.endswith(".localhost") or host.endswith(".internal"): | |
| 70 | + raise SsrfError(ErrorCode.SSRF_REFUSED, f"hôte refusé: {host}") | |
| 71 | + try: | |
| 72 | + ipaddress.ip_address(host) | |
| 73 | + ips = [host] | |
| 74 | + except ValueError: | |
| 75 | + ips = await resolve_host(host) | |
| 76 | + if not ips: | |
| 77 | + raise SsrfError(ErrorCode.TIMEOUT_DNS, f"aucune adresse pour {host}") | |
| 78 | + for ip in ips: | |
| 79 | + if _ip_is_forbidden(ip): | |
| 80 | + raise SsrfError(ErrorCode.SSRF_REFUSED, f"{host} résout vers une adresse interdite ({ip})") | |
| 81 | + return url | |
added
trawls/extract/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/extract/css.py
+94 −0
@@ -0,0 +1,94 @@ | ||
| 1 | +"""Extraction CSS : sélecteurs → JSON typé. Erreurs par champ, jamais d'échec global.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import re | |
| 6 | +from typing import Any | |
| 7 | +from urllib.parse import urljoin | |
| 8 | + | |
| 9 | +from selectolax.parser import HTMLParser, Node | |
| 10 | + | |
| 11 | +from trawls.models import CssField | |
| 12 | + | |
| 13 | +_NUM = re.compile(r"-?\d[\d\s,.]*") | |
| 14 | + | |
| 15 | + | |
| 16 | +def _coerce(raw: str, typ: str, base_url: str) -> Any: | |
| 17 | + v = raw.strip() | |
| 18 | + if typ == "str": | |
| 19 | + return re.sub(r"\s+", " ", v) | |
| 20 | + if typ == "int": | |
| 21 | + m = _NUM.search(v.replace(" ", "")) | |
| 22 | + if not m: | |
| 23 | + raise ValueError(f"pas d'entier dans {v!r}") | |
| 24 | + return int(re.sub(r"[^\d-]", "", m.group(0))) | |
| 25 | + if typ == "float": | |
| 26 | + m = _NUM.search(v.replace(" ", "")) | |
| 27 | + if not m: | |
| 28 | + raise ValueError(f"pas de nombre dans {v!r}") | |
| 29 | + s = m.group(0).replace(" ", "") | |
| 30 | + if s.count(",") == 1 and s.count(".") == 0: | |
| 31 | + s = s.replace(",", ".") | |
| 32 | + else: | |
| 33 | + s = s.replace(",", "") | |
| 34 | + return float(s) | |
| 35 | + if typ == "bool": | |
| 36 | + return v.lower() in ("1", "true", "yes", "oui", "on", "vrai", "✓") | |
| 37 | + if typ == "url": | |
| 38 | + return urljoin(base_url, v) if v else None | |
| 39 | + if typ == "date": | |
| 40 | + from dateutil import parser as dp | |
| 41 | + | |
| 42 | + return dp.parse(v, fuzzy=True).isoformat() | |
| 43 | + if typ == "list": | |
| 44 | + return [x.strip() for x in re.split(r"[,;|\n•]", v) if x.strip()] | |
| 45 | + return v | |
| 46 | + | |
| 47 | + | |
| 48 | +def _value(node: Node, attr: str) -> str: | |
| 49 | + if attr in ("text", ""): | |
| 50 | + return node.text(deep=True, separator=" ") | |
| 51 | + if attr == "html": | |
| 52 | + return node.html or "" | |
| 53 | + if attr == "inner_html": | |
| 54 | + return ( | |
| 55 | + "".join(c.html or "" for c in node.iter(include_text=True)) | |
| 56 | + if hasattr(node, "iter") | |
| 57 | + else node.html or "" | |
| 58 | + ) | |
| 59 | + return node.attributes.get(attr) or "" | |
| 60 | + | |
| 61 | + | |
| 62 | +def extract_css( | |
| 63 | + html: str, fields: dict[str, CssField], base_url: str = "" | |
| 64 | +) -> tuple[dict[str, Any], dict[str, str]]: | |
| 65 | + """Renvoie (données, erreurs par champ).""" | |
| 66 | + tree = HTMLParser(html or "") | |
| 67 | + data: dict[str, Any] = {} | |
| 68 | + errors: dict[str, str] = {} | |
| 69 | + for name, f in fields.items(): | |
| 70 | + try: | |
| 71 | + nodes = tree.css(f.selector) | |
| 72 | + except Exception as e: | |
| 73 | + errors[name] = f"sélecteur invalide: {e}" | |
| 74 | + data[name] = [] if f.multiple else None | |
| 75 | + continue | |
| 76 | + if not nodes: | |
| 77 | + data[name] = [] if f.multiple else None | |
| 78 | + errors[name] = "aucun élément" | |
| 79 | + continue | |
| 80 | + if f.multiple: | |
| 81 | + vals: list[Any] = [] | |
| 82 | + for n in nodes: | |
| 83 | + try: | |
| 84 | + vals.append(_coerce(_value(n, f.attr), f.type, base_url)) | |
| 85 | + except Exception: | |
| 86 | + continue | |
| 87 | + data[name] = vals | |
| 88 | + else: | |
| 89 | + try: | |
| 90 | + data[name] = _coerce(_value(nodes[0], f.attr), f.type, base_url) | |
| 91 | + except Exception as e: | |
| 92 | + data[name] = None | |
| 93 | + errors[name] = str(e)[:200] | |
| 94 | + return data, errors | |
added
trawls/extract/llm.py
+148 −0
@@ -0,0 +1,148 @@ | ||
| 1 | +"""Extraction LLM : JSON Schema + prompt → MD tronqué intelligemment → JSON validé (≤ 2 corrections).""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import json | |
| 6 | +from typing import Any | |
| 7 | + | |
| 8 | +import structlog | |
| 9 | + | |
| 10 | +from trawls.llm.client import LLMError, shared_llm | |
| 11 | +from trawls.models import ChunkOptions, ErrorCode, ErrorInfo | |
| 12 | +from trawls.processors.chunker import chunk, count_tokens | |
| 13 | + | |
| 14 | +log = structlog.get_logger(__name__) | |
| 15 | + | |
| 16 | +_SYSTEM = ( | |
| 17 | + "Tu es un extracteur de données. Réponds UNIQUEMENT avec un objet JSON valide, strictement conforme au " | |
| 18 | + "schéma fourni, sans texte autour, sans commentaires, sans markdown. Si une information est absente, " | |
| 19 | + "utilise null (ou [] pour une liste). N'invente rien." | |
| 20 | +) | |
| 21 | + | |
| 22 | + | |
| 23 | +def _validate(data: Any, schema: dict[str, Any]) -> list[str]: | |
| 24 | + """Validation JSON Schema légère (type, required, properties, items, enum).""" | |
| 25 | + errs: list[str] = [] | |
| 26 | + | |
| 27 | + def walk(d: Any, s: dict[str, Any], path: str) -> None: | |
| 28 | + t = s.get("type") | |
| 29 | + types = t if isinstance(t, list) else ([t] if t else []) | |
| 30 | + if d is None: | |
| 31 | + if "null" in types or not types: | |
| 32 | + return | |
| 33 | + errs.append(f"{path}: null non autorisé (attendu {t})") | |
| 34 | + return | |
| 35 | + if types: | |
| 36 | + ok = any( | |
| 37 | + (tt == "object" and isinstance(d, dict)) | |
| 38 | + or (tt == "array" and isinstance(d, list)) | |
| 39 | + or (tt == "string" and isinstance(d, str)) | |
| 40 | + or (tt == "integer" and isinstance(d, int) and not isinstance(d, bool)) | |
| 41 | + or (tt == "number" and isinstance(d, int | float) and not isinstance(d, bool)) | |
| 42 | + or (tt == "boolean" and isinstance(d, bool)) | |
| 43 | + or tt == "null" | |
| 44 | + and d is None | |
| 45 | + for tt in types | |
| 46 | + ) | |
| 47 | + if not ok: | |
| 48 | + errs.append(f"{path}: type {type(d).__name__} ≠ {t}") | |
| 49 | + return | |
| 50 | + if "enum" in s and d not in s["enum"]: | |
| 51 | + errs.append(f"{path}: {d!r} hors enum") | |
| 52 | + if isinstance(d, dict): | |
| 53 | + for req in s.get("required", []): | |
| 54 | + if req not in d: | |
| 55 | + errs.append(f"{path}.{req}: requis manquant") | |
| 56 | + for k, sub in (s.get("properties") or {}).items(): | |
| 57 | + if k in d: | |
| 58 | + walk(d[k], sub, f"{path}.{k}") | |
| 59 | + if isinstance(d, list) and isinstance(s.get("items"), dict): | |
| 60 | + for i, it in enumerate(d[:200]): | |
| 61 | + walk(it, s["items"], f"{path}[{i}]") | |
| 62 | + | |
| 63 | + walk(data, schema, "$") | |
| 64 | + return errs | |
| 65 | + | |
| 66 | + | |
| 67 | +def _select_context(markdown: str, prompt: str | None, max_tokens: int) -> str: | |
| 68 | + if count_tokens(markdown) <= max_tokens: | |
| 69 | + return markdown | |
| 70 | + chunks = chunk(markdown, "", ChunkOptions(strategy="by_heading", max_tokens=400, min_tokens=50)) | |
| 71 | + if not prompt: | |
| 72 | + out: list[str] = [] | |
| 73 | + tot = 0 | |
| 74 | + for c in chunks: | |
| 75 | + if tot + c.token_count > max_tokens: | |
| 76 | + break | |
| 77 | + out.append(c.text) | |
| 78 | + tot += c.token_count | |
| 79 | + return "\n\n".join(out) | |
| 80 | + # score BM25-light sur les mots du prompt | |
| 81 | + words = {w.lower() for w in prompt.split() if len(w) > 3} | |
| 82 | + scored = sorted( | |
| 83 | + chunks, key=lambda c: -(sum(c.text.lower().count(w) for w in words) + (0.5 if c.index < 3 else 0)) | |
| 84 | + ) | |
| 85 | + picked: list[Any] = [] | |
| 86 | + tot = 0 | |
| 87 | + for c in scored: | |
| 88 | + if tot + c.token_count > max_tokens: | |
| 89 | + continue | |
| 90 | + picked.append(c) | |
| 91 | + tot += c.token_count | |
| 92 | + picked.sort(key=lambda c: c.index) | |
| 93 | + return "\n\n[…]\n\n".join(c.text for c in picked) | |
| 94 | + | |
| 95 | + | |
| 96 | +async def extract_llm( | |
| 97 | + markdown: str, | |
| 98 | + schema: dict[str, Any], | |
| 99 | + prompt: str | None = None, | |
| 100 | + url: str = "", | |
| 101 | + max_context_tokens: int = 24_000, | |
| 102 | +) -> tuple[dict[str, Any] | None, ErrorInfo | None, int]: | |
| 103 | + """Renvoie (données, erreur, tokens consommés).""" | |
| 104 | + llm = shared_llm() | |
| 105 | + if not llm.configured: | |
| 106 | + return ( | |
| 107 | + None, | |
| 108 | + ErrorInfo.make( | |
| 109 | + ErrorCode.LLM_INVALID_OUTPUT, | |
| 110 | + "aucun LLM configuré (TRAWLS_LLM_PROVIDER / TRAWLS_LLM_BASE_URL)", | |
| 111 | + ), | |
| 112 | + 0, | |
| 113 | + ) | |
| 114 | + ctx = _select_context(markdown, prompt, max_context_tokens) | |
| 115 | + user = f"URL: {url}\n\nSCHÉMA JSON:\n{json.dumps(schema, ensure_ascii=False)}\n\n" | |
| 116 | + if prompt: | |
| 117 | + user += f"CONSIGNE: {prompt}\n\n" | |
| 118 | + user += f"CONTENU DE LA PAGE (Markdown):\n<<<\n{ctx}\n>>>\n\nRéponds avec le JSON conforme au schéma." | |
| 119 | + tokens = 0 | |
| 120 | + last_err = "" | |
| 121 | + for attempt in range(3): | |
| 122 | + try: | |
| 123 | + if attempt == 0: | |
| 124 | + data, r = await llm.complete_json(_SYSTEM, user, json_schema=schema) | |
| 125 | + else: | |
| 126 | + data, r = await llm.complete_json( | |
| 127 | + _SYSTEM, | |
| 128 | + user | |
| 129 | + + f"\n\nTa réponse précédente était invalide : {last_err}. Corrige et renvoie uniquement le JSON.", | |
| 130 | + json_schema=schema, | |
| 131 | + ) | |
| 132 | + tokens += r.input_tokens + r.output_tokens | |
| 133 | + except LLMError as e: | |
| 134 | + last_err = str(e) | |
| 135 | + log.info("extract.llm_invalid", url=url, attempt=attempt + 1, error=last_err) | |
| 136 | + continue | |
| 137 | + errs = _validate(data, schema) | |
| 138 | + if not errs: | |
| 139 | + return data if isinstance(data, dict) else {"result": data}, None, tokens | |
| 140 | + last_err = "; ".join(errs[:5]) | |
| 141 | + log.info("extract.llm_schema_errors", url=url, attempt=attempt + 1, errors=errs[:5]) | |
| 142 | + return ( | |
| 143 | + None, | |
| 144 | + ErrorInfo.make( | |
| 145 | + ErrorCode.LLM_INVALID_OUTPUT, f"sortie invalide après 3 essais: {last_err}", attempts=3 | |
| 146 | + ), | |
| 147 | + tokens, | |
| 148 | + ) | |
added
trawls/extract/merge.py
+54 −0
@@ -0,0 +1,54 @@ | ||
| 1 | +"""Fusion multi-pages : dédup d'entités par clé configurable, union des champs non nuls.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import re | |
| 6 | +from typing import Any | |
| 7 | + | |
| 8 | + | |
| 9 | +def _norm(v: Any) -> str: | |
| 10 | + return re.sub(r"\s+", " ", str(v)).strip().lower() | |
| 11 | + | |
| 12 | + | |
| 13 | +def merge_results(results: list[dict[str, Any]], key: str | None = None) -> dict[str, Any]: | |
| 14 | + """`results` = une extraction par URL. Si chaque résultat contient une liste d'entités (1er champ liste), | |
| 15 | + on les concatène et déduplique par `key` ; sinon on fusionne champ à champ (première valeur non nulle).""" | |
| 16 | + if not results: | |
| 17 | + return {} | |
| 18 | + list_field: str | None = None | |
| 19 | + for r in results: | |
| 20 | + for k, v in r.items(): | |
| 21 | + if isinstance(v, list) and v and isinstance(v[0], dict): | |
| 22 | + list_field = k | |
| 23 | + break | |
| 24 | + if list_field: | |
| 25 | + break | |
| 26 | + if list_field: | |
| 27 | + seen: dict[str, dict[str, Any]] = {} | |
| 28 | + order: list[str] = [] | |
| 29 | + for r in results: | |
| 30 | + for ent in r.get(list_field) or []: | |
| 31 | + if not isinstance(ent, dict): | |
| 32 | + continue | |
| 33 | + k = _norm(ent.get(key)) if key and ent.get(key) is not None else _norm(sorted(ent.items())) | |
| 34 | + if k in seen: | |
| 35 | + for f, v in ent.items(): | |
| 36 | + if seen[k].get(f) in (None, "", []) and v not in (None, "", []): | |
| 37 | + seen[k][f] = v | |
| 38 | + else: | |
| 39 | + seen[k] = dict(ent) | |
| 40 | + order.append(k) | |
| 41 | + merged: dict[str, Any] = {list_field: [seen[k] for k in order]} | |
| 42 | + for r in results: | |
| 43 | + for f, v in r.items(): | |
| 44 | + if f != list_field and f not in merged and v not in (None, "", []): | |
| 45 | + merged[f] = v | |
| 46 | + return merged | |
| 47 | + merged = {} | |
| 48 | + for r in results: | |
| 49 | + for f, v in r.items(): | |
| 50 | + if merged.get(f) in (None, "", []) and v not in (None, "", []): | |
| 51 | + merged[f] = v | |
| 52 | + elif f not in merged: | |
| 53 | + merged[f] = v | |
| 54 | + return merged | |
added
trawls/llm/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/llm/client.py
+192 −0
@@ -0,0 +1,192 @@ | ||
| 1 | +"""Interface LLM unique : Anthropic ou OpenAI-compatible (Ollama, vLLM, OpenRouter, OpenAI). | |
| 2 | +`complete_json` demande une sortie JSON stricte (mode structured output natif quand disponible).""" | |
| 3 | + | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import json | |
| 7 | +import re | |
| 8 | +from dataclasses import dataclass | |
| 9 | +from typing import Any | |
| 10 | + | |
| 11 | +import httpx | |
| 12 | +import structlog | |
| 13 | + | |
| 14 | +from trawls.config import get_settings | |
| 15 | + | |
| 16 | +log = structlog.get_logger(__name__) | |
| 17 | + | |
| 18 | + | |
| 19 | +class LLMError(Exception): | |
| 20 | + pass | |
| 21 | + | |
| 22 | + | |
| 23 | +@dataclass | |
| 24 | +class LLMResponse: | |
| 25 | + text: str | |
| 26 | + input_tokens: int = 0 | |
| 27 | + output_tokens: int = 0 | |
| 28 | + model: str = "" | |
| 29 | + | |
| 30 | + | |
| 31 | +def _extract_json(text: str) -> Any: | |
| 32 | + text = text.strip() | |
| 33 | + m = re.search(r"```(?:json)?\s*(.*?)```", text, re.S) | |
| 34 | + if m: | |
| 35 | + text = m.group(1).strip() | |
| 36 | + try: | |
| 37 | + return json.loads(text) | |
| 38 | + except json.JSONDecodeError: | |
| 39 | + pass | |
| 40 | + # premier objet/tableau équilibré | |
| 41 | + for open_, close in (("{", "}"), ("[", "]")): | |
| 42 | + s = text.find(open_) | |
| 43 | + e = text.rfind(close) | |
| 44 | + if s >= 0 and e > s: | |
| 45 | + try: | |
| 46 | + return json.loads(text[s : e + 1]) | |
| 47 | + except json.JSONDecodeError: | |
| 48 | + continue | |
| 49 | + raise LLMError("sortie non-JSON") | |
| 50 | + | |
| 51 | + | |
| 52 | +class LLMClient: | |
| 53 | + def __init__( | |
| 54 | + self, | |
| 55 | + provider: str | None = None, | |
| 56 | + api_key: str | None = None, | |
| 57 | + base_url: str | None = None, | |
| 58 | + model: str | None = None, | |
| 59 | + ) -> None: | |
| 60 | + s = get_settings() | |
| 61 | + self.provider = (provider or s.llm_provider).lower() | |
| 62 | + self.api_key = api_key or s.llm_api_key | |
| 63 | + self.base_url = (base_url or s.llm_base_url or "").rstrip("/") | |
| 64 | + self.model = model or s.llm_model | |
| 65 | + self._http = httpx.AsyncClient(timeout=120.0) | |
| 66 | + | |
| 67 | + @property | |
| 68 | + def configured(self) -> bool: | |
| 69 | + if self.provider == "none": | |
| 70 | + return False | |
| 71 | + if self.provider == "anthropic": | |
| 72 | + return bool(self.api_key) | |
| 73 | + return bool(self.base_url) | |
| 74 | + | |
| 75 | + async def close(self) -> None: | |
| 76 | + await self._http.aclose() | |
| 77 | + | |
| 78 | + async def complete( | |
| 79 | + self, | |
| 80 | + system: str, | |
| 81 | + user: str, | |
| 82 | + *, | |
| 83 | + max_tokens: int = 4096, | |
| 84 | + temperature: float = 0.0, | |
| 85 | + json_schema: dict[str, Any] | None = None, | |
| 86 | + ) -> LLMResponse: | |
| 87 | + if self.provider == "anthropic": | |
| 88 | + return await self._anthropic(system, user, max_tokens, temperature, json_schema) | |
| 89 | + return await self._openai(system, user, max_tokens, temperature, json_schema) | |
| 90 | + | |
| 91 | + async def complete_json( | |
| 92 | + self, system: str, user: str, *, json_schema: dict[str, Any] | None = None, max_tokens: int = 4096 | |
| 93 | + ) -> tuple[Any, LLMResponse]: | |
| 94 | + r = await self.complete(system, user, max_tokens=max_tokens, json_schema=json_schema) | |
| 95 | + return _extract_json(r.text), r | |
| 96 | + | |
| 97 | + # --- providers ----------------------------------------------------------- | |
| 98 | + | |
| 99 | + async def _openai( | |
| 100 | + self, system: str, user: str, max_tokens: int, temperature: float, schema: dict[str, Any] | None | |
| 101 | + ) -> LLMResponse: | |
| 102 | + url = f"{self.base_url}/chat/completions" | |
| 103 | + headers = {"Content-Type": "application/json"} | |
| 104 | + if self.api_key: | |
| 105 | + headers["Authorization"] = f"Bearer {self.api_key}" | |
| 106 | + body: dict[str, Any] = { | |
| 107 | + "model": self.model, | |
| 108 | + "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}], | |
| 109 | + "max_tokens": max_tokens, | |
| 110 | + "temperature": temperature, | |
| 111 | + } | |
| 112 | + if schema is not None: | |
| 113 | + body["response_format"] = { | |
| 114 | + "type": "json_schema", | |
| 115 | + "json_schema": {"name": "extraction", "schema": schema, "strict": False}, | |
| 116 | + } | |
| 117 | + try: | |
| 118 | + r = await self._http.post(url, headers=headers, json=body) | |
| 119 | + if r.status_code == 400 and schema is not None: | |
| 120 | + body["response_format"] = {"type": "json_object"} | |
| 121 | + r = await self._http.post(url, headers=headers, json=body) | |
| 122 | + if r.status_code == 400 and "response_format" in body: | |
| 123 | + del body["response_format"] | |
| 124 | + r = await self._http.post(url, headers=headers, json=body) | |
| 125 | + r.raise_for_status() | |
| 126 | + except httpx.HTTPError as e: | |
| 127 | + raise LLMError(f"LLM HTTP: {e}") from None | |
| 128 | + data = r.json() | |
| 129 | + try: | |
| 130 | + text = data["choices"][0]["message"]["content"] or "" | |
| 131 | + except (KeyError, IndexError) as e: | |
| 132 | + raise LLMError(f"réponse LLM inattendue: {e}") from None | |
| 133 | + usage = data.get("usage") or {} | |
| 134 | + return LLMResponse( | |
| 135 | + text=text, | |
| 136 | + input_tokens=usage.get("prompt_tokens", 0), | |
| 137 | + output_tokens=usage.get("completion_tokens", 0), | |
| 138 | + model=data.get("model", self.model), | |
| 139 | + ) | |
| 140 | + | |
| 141 | + async def _anthropic( | |
| 142 | + self, system: str, user: str, max_tokens: int, temperature: float, schema: dict[str, Any] | None | |
| 143 | + ) -> LLMResponse: | |
| 144 | + url = (self.base_url or "https://api.anthropic.com") + "/v1/messages" | |
| 145 | + headers = { | |
| 146 | + "x-api-key": self.api_key or "", | |
| 147 | + "anthropic-version": "2023-06-01", | |
| 148 | + "content-type": "application/json", | |
| 149 | + } | |
| 150 | + body: dict[str, Any] = { | |
| 151 | + "model": self.model, | |
| 152 | + "system": system, | |
| 153 | + "messages": [{"role": "user", "content": user}], | |
| 154 | + "max_tokens": max_tokens, | |
| 155 | + "temperature": temperature, | |
| 156 | + } | |
| 157 | + if schema is not None: | |
| 158 | + # tool use forcé = sortie structurée native | |
| 159 | + body["tools"] = [ | |
| 160 | + {"name": "emit", "description": "Émet le résultat structuré.", "input_schema": schema} | |
| 161 | + ] | |
| 162 | + body["tool_choice"] = {"type": "tool", "name": "emit"} | |
| 163 | + try: | |
| 164 | + r = await self._http.post(url, headers=headers, json=body) | |
| 165 | + r.raise_for_status() | |
| 166 | + except httpx.HTTPError as e: | |
| 167 | + raise LLMError(f"LLM HTTP: {e}") from None | |
| 168 | + data = r.json() | |
| 169 | + text = "" | |
| 170 | + for block in data.get("content", []): | |
| 171 | + if block.get("type") == "tool_use": | |
| 172 | + text = json.dumps(block.get("input", {})) | |
| 173 | + break | |
| 174 | + if block.get("type") == "text": | |
| 175 | + text += block.get("text", "") | |
| 176 | + usage = data.get("usage") or {} | |
| 177 | + return LLMResponse( | |
| 178 | + text=text, | |
| 179 | + input_tokens=usage.get("input_tokens", 0), | |
| 180 | + output_tokens=usage.get("output_tokens", 0), | |
| 181 | + model=data.get("model", self.model), | |
| 182 | + ) | |
| 183 | + | |
| 184 | + | |
| 185 | +_shared: LLMClient | None = None | |
| 186 | + | |
| 187 | + | |
| 188 | +def shared_llm() -> LLMClient: | |
| 189 | + global _shared | |
| 190 | + if _shared is None: | |
| 191 | + _shared = LLMClient() | |
| 192 | + return _shared | |
added
trawls/map/__init__.py
+221 −0
@@ -0,0 +1,221 @@ | ||
| 1 | +"""Map : agrège robots.txt → sitemaps (récursif, gzip, index) + crawl shallow http (depth 2) en parallèle. | |
| 2 | +Sortie dédupliquée avec sources, lastmod, depth ; tri BM25 sur URL+title si `search`.""" | |
| 3 | + | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import asyncio | |
| 7 | +import math | |
| 8 | +import re | |
| 9 | +import time | |
| 10 | +from collections import Counter | |
| 11 | +from urllib.parse import urlsplit | |
| 12 | + | |
| 13 | +import structlog | |
| 14 | +from selectolax.parser import HTMLParser | |
| 15 | + | |
| 16 | +from trawls.core.fetcher.http_fast import fetch_text, shared_http | |
| 17 | +from trawls.core.scheduler.dedup import normalize_url, same_site | |
| 18 | +from trawls.core.scheduler.robots import robots_cache | |
| 19 | +from trawls.models import MapOptions, MappedUrl, ScrapeOptions | |
| 20 | +from trawls.processors.encoding import decode_body | |
| 21 | +from trawls.processors.links import extract_links | |
| 22 | + | |
| 23 | +log = structlog.get_logger(__name__) | |
| 24 | + | |
| 25 | +_LOC = re.compile(r"<loc>\s*(.*?)\s*</loc>", re.I | re.S) | |
| 26 | +_URL_BLOCK = re.compile(r"<(url|sitemap)>(.*?)</\1>", re.I | re.S) | |
| 27 | +_LASTMOD = re.compile(r"<lastmod>\s*(.*?)\s*</lastmod>", re.I | re.S) | |
| 28 | + | |
| 29 | + | |
| 30 | +async def _read_sitemap( | |
| 31 | + url: str, out: dict[str, MappedUrl], seen_sm: set[str], root: str, opts: MapOptions, depth: int = 0 | |
| 32 | +) -> None: | |
| 33 | + if url in seen_sm or depth > 4 or len(out) >= _cap(opts): | |
| 34 | + return | |
| 35 | + seen_sm.add(url) | |
| 36 | + try: | |
| 37 | + status, text = await fetch_text(url, timeout_s=12, max_mb=30) | |
| 38 | + except Exception as e: | |
| 39 | + log.info("map.sitemap_failed", url=url, error=str(e)) | |
| 40 | + return | |
| 41 | + if status != 200 or not text: | |
| 42 | + return | |
| 43 | + is_index = "<sitemapindex" in text[:2000].lower() | |
| 44 | + blocks = _URL_BLOCK.findall(text) | |
| 45 | + if not blocks: | |
| 46 | + # texte brut (une URL par ligne) | |
| 47 | + for ln in text.splitlines(): | |
| 48 | + ln = ln.strip() | |
| 49 | + if ln.startswith("http"): | |
| 50 | + _add(out, ln, "sitemap", root, opts) | |
| 51 | + return | |
| 52 | + children: list[str] = [] | |
| 53 | + for kind, body in blocks: | |
| 54 | + m = _LOC.search(body) | |
| 55 | + if not m: | |
| 56 | + continue | |
| 57 | + loc = _unescape(m.group(1)) | |
| 58 | + lm = _LASTMOD.search(body) | |
| 59 | + if kind.lower() == "sitemap" or is_index: | |
| 60 | + children.append(loc) | |
| 61 | + else: | |
| 62 | + _add(out, loc, "sitemap", root, opts, lastmod=lm.group(1) if lm else None) | |
| 63 | + if children: | |
| 64 | + await asyncio.gather(*(_read_sitemap(c, out, seen_sm, root, opts, depth + 1) for c in children[:200])) | |
| 65 | + | |
| 66 | + | |
| 67 | +def _cap(opts: MapOptions) -> int: | |
| 68 | + """Plafond de collecte : plus large que `limit` quand on trie par pertinence (`search`).""" | |
| 69 | + return min(max(opts.limit * 10, 2000), 50_000) if opts.search else opts.limit | |
| 70 | + | |
| 71 | + | |
| 72 | +def _unescape(s: str) -> str: | |
| 73 | + return ( | |
| 74 | + s.replace("&", "&") | |
| 75 | + .replace("<", "<") | |
| 76 | + .replace(">", ">") | |
| 77 | + .replace(""", '"') | |
| 78 | + .replace("'", "'") | |
| 79 | + .strip() | |
| 80 | + ) | |
| 81 | + | |
| 82 | + | |
| 83 | +def _add( | |
| 84 | + out: dict[str, MappedUrl], | |
| 85 | + url: str, | |
| 86 | + source: str, | |
| 87 | + root: str, | |
| 88 | + opts: MapOptions, | |
| 89 | + lastmod: str | None = None, | |
| 90 | + depth: int | None = None, | |
| 91 | + title: str | None = None, | |
| 92 | +) -> None: | |
| 93 | + n = normalize_url(url) | |
| 94 | + if not n or not same_site(root, n, opts.include_subdomains): | |
| 95 | + return | |
| 96 | + if len(out) >= _cap(opts) and n not in out: | |
| 97 | + return | |
| 98 | + e = out.get(n) | |
| 99 | + if e is None: | |
| 100 | + out[n] = MappedUrl(url=n, sources=[source], lastmod=lastmod, depth=depth, title=title) | |
| 101 | + else: | |
| 102 | + if source not in e.sources: | |
| 103 | + e.sources.append(source) | |
| 104 | + e.lastmod = e.lastmod or lastmod | |
| 105 | + e.depth = ( | |
| 106 | + min(d for d in (e.depth, depth) if d is not None) | |
| 107 | + if (e.depth is not None or depth is not None) | |
| 108 | + else None | |
| 109 | + ) | |
| 110 | + e.title = e.title or title | |
| 111 | + | |
| 112 | + | |
| 113 | +async def _shallow_crawl(root: str, out: dict[str, MappedUrl], opts: MapOptions, deadline: float) -> None: | |
| 114 | + frontier: list[tuple[str, int]] = [(root, 0)] | |
| 115 | + seen: set[str] = {root} | |
| 116 | + sem = asyncio.Semaphore(16) | |
| 117 | + sopts = ScrapeOptions(timeout_ms=10_000, mode="http") | |
| 118 | + | |
| 119 | + async def one(u: str, d: int) -> list[tuple[str, int]]: | |
| 120 | + if time.monotonic() > deadline: | |
| 121 | + return [] | |
| 122 | + async with sem: | |
| 123 | + try: | |
| 124 | + res = await shared_http().fetch(u, sopts) | |
| 125 | + except Exception: | |
| 126 | + return [] | |
| 127 | + if not res.is_html or res.status >= 400: | |
| 128 | + return [] | |
| 129 | + text, _ = decode_body(res.body, res.headers.get("content-type")) | |
| 130 | + title = None | |
| 131 | + if opts.include_titles: | |
| 132 | + t = HTMLParser(text).css_first("title") | |
| 133 | + title = t.text().strip()[:200] if t else None | |
| 134 | + _add(out, res.final_url, "crawl", root, opts, depth=d, title=title) | |
| 135 | + nxt: list[tuple[str, int]] = [] | |
| 136 | + for lk in extract_links(text, res.final_url, opts.include_subdomains, max_links=2000): | |
| 137 | + if lk.kind == "internal" and lk.href not in seen: | |
| 138 | + seen.add(lk.href) | |
| 139 | + _add(out, lk.href, "crawl", root, opts, depth=d + 1) | |
| 140 | + if d + 1 < opts.crawl_depth: | |
| 141 | + nxt.append((lk.href, d + 1)) | |
| 142 | + return nxt | |
| 143 | + | |
| 144 | + while frontier and time.monotonic() < deadline and len(out) < _cap(opts): | |
| 145 | + batch, frontier = frontier[:64], frontier[64:] | |
| 146 | + results = await asyncio.gather(*(one(u, d) for u, d in batch)) | |
| 147 | + for r in results: | |
| 148 | + frontier.extend(r) | |
| 149 | + | |
| 150 | + | |
| 151 | +def _tokens(s: str) -> list[str]: | |
| 152 | + return [t for t in re.split(r"[^a-z0-9àâçéèêëîïôûùüÿœ]+", s.lower()) if len(t) > 1] | |
| 153 | + | |
| 154 | + | |
| 155 | +def rank(items: list[MappedUrl], query: str) -> list[MappedUrl]: | |
| 156 | + q = _tokens(query) | |
| 157 | + if not q: | |
| 158 | + return items | |
| 159 | + docs = [_tokens(urlsplit(i.url).path.replace("/", " ") + " " + (i.title or "")) for i in items] | |
| 160 | + n = len(docs) | |
| 161 | + df: Counter[str] = Counter() | |
| 162 | + for d in docs: | |
| 163 | + for t in set(d): | |
| 164 | + df[t] += 1 | |
| 165 | + avgdl = sum(len(d) for d in docs) / max(1, n) | |
| 166 | + k1, b = 1.5, 0.75 | |
| 167 | + for i, d in zip(items, docs, strict=False): | |
| 168 | + tf = Counter(d) | |
| 169 | + score = 0.0 | |
| 170 | + for t in q: | |
| 171 | + if t not in tf: | |
| 172 | + # correspondance partielle (préfixe) | |
| 173 | + part = [x for x in tf if x.startswith(t) or t.startswith(x)] | |
| 174 | + if not part: | |
| 175 | + continue | |
| 176 | + f = sum(tf[x] for x in part) * 0.5 | |
| 177 | + dfx = max(df[x] for x in part) | |
| 178 | + else: | |
| 179 | + f = tf[t] | |
| 180 | + dfx = df[t] | |
| 181 | + idf = math.log(1 + (n - dfx + 0.5) / (dfx + 0.5)) | |
| 182 | + score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * len(d) / max(1, avgdl))) | |
| 183 | + i.score = round(score, 4) | |
| 184 | + return sorted(items, key=lambda x: -(x.score or 0)) | |
| 185 | + | |
| 186 | + | |
| 187 | +async def map_site(url: str, opts: MapOptions) -> list[MappedUrl]: | |
| 188 | + root = normalize_url(url) or url | |
| 189 | + out: dict[str, MappedUrl] = {} | |
| 190 | + deadline = time.monotonic() + opts.timeout_s | |
| 191 | + tasks = [asyncio.create_task(_shallow_crawl(root, out, opts, deadline))] | |
| 192 | + if not opts.ignore_sitemap: | |
| 193 | + | |
| 194 | + async def sitemaps() -> None: | |
| 195 | + sms = await robots_cache.sitemaps(root) | |
| 196 | + p = urlsplit(root) | |
| 197 | + candidates = list( | |
| 198 | + dict.fromkeys( | |
| 199 | + sms | |
| 200 | + + [ | |
| 201 | + f"{p.scheme}://{p.netloc}/sitemap.xml", | |
| 202 | + f"{p.scheme}://{p.netloc}/sitemap_index.xml", | |
| 203 | + f"{p.scheme}://{p.netloc}/sitemap.xml.gz", | |
| 204 | + ] | |
| 205 | + ) | |
| 206 | + ) | |
| 207 | + seen_sm: set[str] = set() | |
| 208 | + await asyncio.gather(*(_read_sitemap(sm, out, seen_sm, root, opts) for sm in candidates[:10])) | |
| 209 | + | |
| 210 | + tasks.append(asyncio.create_task(sitemaps())) | |
| 211 | + try: | |
| 212 | + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=opts.timeout_s + 5) | |
| 213 | + except TimeoutError: | |
| 214 | + for t in tasks: | |
| 215 | + t.cancel() | |
| 216 | + items = list(out.values()) | |
| 217 | + if opts.search: | |
| 218 | + items = rank(items, opts.search) | |
| 219 | + else: | |
| 220 | + items.sort(key=lambda i: ((i.depth if i.depth is not None else 99), i.url)) | |
| 221 | + return items[: opts.limit] | |
added
trawls/models/__init__.py
+302 −0
@@ -0,0 +1,302 @@ | ||
| 1 | +"""Modèles Pydantic v2 — le contrat de l'API. Aucun dict non typé ne sort de l'API.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +from datetime import UTC, datetime | |
| 6 | +from enum import StrEnum | |
| 7 | +from typing import Any, Literal | |
| 8 | + | |
| 9 | +from pydantic import BaseModel, Field, field_validator | |
| 10 | + | |
| 11 | +Format = Literal["markdown", "html", "raw_html", "json", "links", "screenshot", "chunks", "metadata"] | |
| 12 | +FetchMode = Literal["auto", "http", "browser", "stealth"] | |
| 13 | +ResolvedMode = Literal["http", "browser", "stealth"] | |
| 14 | + | |
| 15 | + | |
| 16 | +class ErrorCode(StrEnum): | |
| 17 | + TIMEOUT_DNS = "TIMEOUT_DNS" | |
| 18 | + TIMEOUT_CONNECT = "TIMEOUT_CONNECT" | |
| 19 | + TIMEOUT_TTFB = "TIMEOUT_TTFB" | |
| 20 | + TIMEOUT_RENDER = "TIMEOUT_RENDER" | |
| 21 | + HTTP_4XX = "HTTP_4XX" | |
| 22 | + HTTP_5XX = "HTTP_5XX" | |
| 23 | + RATE_LIMITED = "RATE_LIMITED" | |
| 24 | + BLOCKED = "BLOCKED" | |
| 25 | + ROBOTS_DISALLOWED = "ROBOTS_DISALLOWED" | |
| 26 | + TOO_LARGE = "TOO_LARGE" | |
| 27 | + UNSUPPORTED_CONTENT = "UNSUPPORTED_CONTENT" | |
| 28 | + PARSE_FAILED = "PARSE_FAILED" | |
| 29 | + SSL_ERROR = "SSL_ERROR" | |
| 30 | + CIRCUIT_OPEN = "CIRCUIT_OPEN" | |
| 31 | + BUDGET_EXCEEDED = "BUDGET_EXCEEDED" | |
| 32 | + LLM_INVALID_OUTPUT = "LLM_INVALID_OUTPUT" | |
| 33 | + SSRF_REFUSED = "SSRF_REFUSED" | |
| 34 | + INVALID_URL = "INVALID_URL" | |
| 35 | + NETWORK = "NETWORK" | |
| 36 | + INTERNAL = "INTERNAL" | |
| 37 | + CANCELLED = "CANCELLED" | |
| 38 | + | |
| 39 | + | |
| 40 | +RETRYABLE: frozenset[ErrorCode] = frozenset( | |
| 41 | + { | |
| 42 | + ErrorCode.TIMEOUT_DNS, | |
| 43 | + ErrorCode.TIMEOUT_CONNECT, | |
| 44 | + ErrorCode.TIMEOUT_TTFB, | |
| 45 | + ErrorCode.TIMEOUT_RENDER, | |
| 46 | + ErrorCode.HTTP_5XX, | |
| 47 | + ErrorCode.RATE_LIMITED, | |
| 48 | + ErrorCode.CIRCUIT_OPEN, | |
| 49 | + ErrorCode.NETWORK, | |
| 50 | + ErrorCode.LLM_INVALID_OUTPUT, | |
| 51 | + } | |
| 52 | +) | |
| 53 | + | |
| 54 | + | |
| 55 | +class ErrorInfo(BaseModel): | |
| 56 | + code: ErrorCode | |
| 57 | + message: str | |
| 58 | + retryable: bool | |
| 59 | + attempts: int = 1 | |
| 60 | + details: dict[str, Any] | None = None | |
| 61 | + | |
| 62 | + @classmethod | |
| 63 | + def make(cls, code: ErrorCode, message: str, attempts: int = 1, **details: Any) -> ErrorInfo: | |
| 64 | + return cls( | |
| 65 | + code=code, | |
| 66 | + message=message[:1000], | |
| 67 | + retryable=code in RETRYABLE, | |
| 68 | + attempts=attempts, | |
| 69 | + details=details or None, | |
| 70 | + ) | |
| 71 | + | |
| 72 | + | |
| 73 | +class Cookie(BaseModel): | |
| 74 | + name: str | |
| 75 | + value: str | |
| 76 | + domain: str | None = None | |
| 77 | + path: str = "/" | |
| 78 | + | |
| 79 | + | |
| 80 | +class BrowserAction(BaseModel): | |
| 81 | + type: Literal["click", "scroll", "type", "wait", "press", "screenshot", "evaluate"] | |
| 82 | + selector: str | None = None | |
| 83 | + text: str | None = None | |
| 84 | + key: str | None = None | |
| 85 | + ms: int | None = None | |
| 86 | + direction: Literal["up", "down"] = "down" | |
| 87 | + amount: int = 1000 | |
| 88 | + script: str | None = None | |
| 89 | + | |
| 90 | + | |
| 91 | +class Location(BaseModel): | |
| 92 | + country: str = "CA" | |
| 93 | + languages: list[str] = ["fr-CA", "fr", "en"] | |
| 94 | + | |
| 95 | + | |
| 96 | +class ChunkOptions(BaseModel): | |
| 97 | + strategy: Literal["by_heading", "by_tokens"] = "by_heading" | |
| 98 | + size_tokens: int = 512 | |
| 99 | + overlap_tokens: int = 64 | |
| 100 | + min_tokens: int = 64 | |
| 101 | + max_tokens: int = 1024 | |
| 102 | + | |
| 103 | + | |
| 104 | +class CssField(BaseModel): | |
| 105 | + selector: str | |
| 106 | + attr: str = "text" | |
| 107 | + type: Literal["str", "int", "float", "date", "url", "list", "bool"] = "str" | |
| 108 | + multiple: bool = False | |
| 109 | + | |
| 110 | + | |
| 111 | +class ExtractOptions(BaseModel): | |
| 112 | + mode: Literal["css", "llm"] = "css" | |
| 113 | + schema_: dict[str, Any] | None = Field(default=None, alias="schema") | |
| 114 | + css: dict[str, CssField] | None = None | |
| 115 | + prompt: str | None = None | |
| 116 | + | |
| 117 | + model_config = {"populate_by_name": True} | |
| 118 | + | |
| 119 | + | |
| 120 | +class ScrapeOptions(BaseModel): | |
| 121 | + formats: list[Format] = ["markdown"] | |
| 122 | + only_main_content: bool = True | |
| 123 | + include_tags: list[str] = [] | |
| 124 | + exclude_tags: list[str] = [] | |
| 125 | + wait_for: str | int | None = None | |
| 126 | + timeout_ms: int = 30_000 | |
| 127 | + mode: FetchMode = "auto" | |
| 128 | + headers: dict[str, str] = {} | |
| 129 | + cookies: list[Cookie] = [] | |
| 130 | + proxy: str | None = None | |
| 131 | + actions: list[BrowserAction] = [] | |
| 132 | + location: Location | None = None | |
| 133 | + remove_base64_images: bool = True | |
| 134 | + chunk: ChunkOptions | None = None | |
| 135 | + extract: ExtractOptions | None = None | |
| 136 | + cache: Literal["use", "bypass", "refresh"] = "use" | |
| 137 | + max_age_s: int = 86_400 | |
| 138 | + citations: bool = False | |
| 139 | + verify_ssl: bool = True | |
| 140 | + respect_robots: bool = True | |
| 141 | + | |
| 142 | + @field_validator("timeout_ms") | |
| 143 | + @classmethod | |
| 144 | + def _clamp_timeout(cls, v: int) -> int: | |
| 145 | + return max(1_000, min(v, 180_000)) | |
| 146 | + | |
| 147 | + | |
| 148 | +class Link(BaseModel): | |
| 149 | + href: str | |
| 150 | + text: str = "" | |
| 151 | + rel: str | None = None | |
| 152 | + kind: Literal["internal", "external", "asset", "mailto", "tel", "other"] = "other" | |
| 153 | + nofollow: bool = False | |
| 154 | + | |
| 155 | + | |
| 156 | +class PageMetadata(BaseModel): | |
| 157 | + title: str | None = None | |
| 158 | + description: str | None = None | |
| 159 | + language: str | None = None | |
| 160 | + author: str | None = None | |
| 161 | + published_at: str | None = None | |
| 162 | + modified_at: str | None = None | |
| 163 | + canonical_url: str | None = None | |
| 164 | + site_name: str | None = None | |
| 165 | + og_image: str | None = None | |
| 166 | + og_type: str | None = None | |
| 167 | + keywords: list[str] = [] | |
| 168 | + favicon: str | None = None | |
| 169 | + content_type: str | None = None | |
| 170 | + charset: str | None = None | |
| 171 | + word_count: int = 0 | |
| 172 | + jsonld: list[dict[str, Any]] = [] | |
| 173 | + is_pdf: bool = False | |
| 174 | + pdf_pages: int | None = None | |
| 175 | + pdf_is_scanned: bool | None = None | |
| 176 | + extra: dict[str, Any] = {} | |
| 177 | + | |
| 178 | + | |
| 179 | +class Chunk(BaseModel): | |
| 180 | + index: int | |
| 181 | + text: str | |
| 182 | + heading_path: str = "" | |
| 183 | + token_count: int | |
| 184 | + char_range: tuple[int, int] | |
| 185 | + url: str | |
| 186 | + | |
| 187 | + | |
| 188 | +class Timings(BaseModel): | |
| 189 | + dns_ms: float | None = None | |
| 190 | + connect_ms: float | None = None | |
| 191 | + ttfb_ms: float | None = None | |
| 192 | + fetch_ms: float | None = None | |
| 193 | + render_ms: float | None = None | |
| 194 | + process_ms: float | None = None | |
| 195 | + total_ms: float | None = None | |
| 196 | + | |
| 197 | + | |
| 198 | +class PageResult(BaseModel): | |
| 199 | + url: str | |
| 200 | + final_url: str | |
| 201 | + status: Literal["ok", "failed", "skipped"] | |
| 202 | + http_status: int | None = None | |
| 203 | + fetch_mode_used: ResolvedMode | None = None | |
| 204 | + markdown: str | None = None | |
| 205 | + html: str | None = None | |
| 206 | + raw_html: str | None = None | |
| 207 | + json_data: dict[str, Any] | None = None | |
| 208 | + links: list[Link] = [] | |
| 209 | + metadata: PageMetadata = Field(default_factory=PageMetadata) | |
| 210 | + chunks: list[Chunk] | None = None | |
| 211 | + screenshot_url: str | None = None | |
| 212 | + error: ErrorInfo | None = None | |
| 213 | + timings: Timings = Field(default_factory=Timings) | |
| 214 | + fetched_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) | |
| 215 | + depth: int = 0 | |
| 216 | + from_cache: bool = False | |
| 217 | + | |
| 218 | + @classmethod | |
| 219 | + def failed(cls, url: str, err: ErrorInfo, mode: ResolvedMode | None = None, **kw: Any) -> PageResult: | |
| 220 | + return cls(url=url, final_url=url, status="failed", error=err, fetch_mode_used=mode, **kw) | |
| 221 | + | |
| 222 | + @classmethod | |
| 223 | + def skipped(cls, url: str, reason: str, **kw: Any) -> PageResult: | |
| 224 | + return cls( | |
| 225 | + url=url, | |
| 226 | + final_url=url, | |
| 227 | + status="skipped", | |
| 228 | + error=ErrorInfo(code=ErrorCode.INTERNAL, message=reason, retryable=False), | |
| 229 | + **kw, | |
| 230 | + ) | |
| 231 | + | |
| 232 | + | |
| 233 | +class CrawlOptions(BaseModel): | |
| 234 | + max_depth: int = 3 | |
| 235 | + max_pages: int = 100 | |
| 236 | + include_paths: list[str] = [] | |
| 237 | + exclude_paths: list[str] = [] | |
| 238 | + allow_subdomains: bool = False | |
| 239 | + allow_external_links: bool = False | |
| 240 | + ignore_sitemap: bool = False | |
| 241 | + respect_robots: bool = True | |
| 242 | + delay_ms: int = 0 | |
| 243 | + concurrency: int = 5 | |
| 244 | + strategy: Literal["bfs", "dfs", "best_first"] = "bfs" | |
| 245 | + search: str | None = None | |
| 246 | + max_duration_s: int = 1800 | |
| 247 | + | |
| 248 | + @field_validator("max_pages") | |
| 249 | + @classmethod | |
| 250 | + def _clamp_pages(cls, v: int) -> int: | |
| 251 | + return max(1, min(v, 20_000)) | |
| 252 | + | |
| 253 | + | |
| 254 | +class MapOptions(BaseModel): | |
| 255 | + search: str | None = None | |
| 256 | + include_subdomains: bool = False | |
| 257 | + limit: int = 5000 | |
| 258 | + ignore_sitemap: bool = False | |
| 259 | + include_titles: bool = False | |
| 260 | + crawl_depth: int = 2 | |
| 261 | + timeout_s: int = 30 | |
| 262 | + | |
| 263 | + | |
| 264 | +class MappedUrl(BaseModel): | |
| 265 | + url: str | |
| 266 | + sources: list[str] = [] | |
| 267 | + lastmod: str | None = None | |
| 268 | + depth: int | None = None | |
| 269 | + title: str | None = None | |
| 270 | + score: float | None = None | |
| 271 | + | |
| 272 | + | |
| 273 | +JobStatus = Literal["queued", "running", "paused", "completed", "failed", "cancelled"] | |
| 274 | +JobKind = Literal["crawl", "batch", "extract", "agent", "map"] | |
| 275 | + | |
| 276 | + | |
| 277 | +class JobSummary(BaseModel): | |
| 278 | + id: str | |
| 279 | + kind: JobKind | |
| 280 | + status: JobStatus | |
| 281 | + created_at: datetime | |
| 282 | + started_at: datetime | None = None | |
| 283 | + finished_at: datetime | None = None | |
| 284 | + total: int = 0 | |
| 285 | + completed: int = 0 | |
| 286 | + failed: int = 0 | |
| 287 | + skipped: int = 0 | |
| 288 | + root_url: str | None = None | |
| 289 | + error: ErrorInfo | None = None | |
| 290 | + credits_used: int = 0 | |
| 291 | + meta: dict[str, Any] = {} | |
| 292 | + | |
| 293 | + | |
| 294 | +class ApiError(BaseModel): | |
| 295 | + code: str | |
| 296 | + message: str | |
| 297 | + retryable: bool = False | |
| 298 | + details: dict[str, Any] | None = None | |
| 299 | + | |
| 300 | + | |
| 301 | +class ApiErrorEnvelope(BaseModel): | |
| 302 | + error: ApiError | |
added
trawls/processors/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/processors/chunker/__init__.py
+159 −0
@@ -0,0 +1,159 @@ | ||
| 1 | +"""Chunker : `by_heading` (sections h1-h3, fusion des petites, split des grandes) et `by_tokens` | |
| 2 | +(taille fixe + overlap, coupe sur frontières de phrases). Tokens comptés avec tiktoken cl100k. | |
| 3 | +""" | |
| 4 | + | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import re | |
| 8 | +from functools import lru_cache | |
| 9 | + | |
| 10 | +from trawls.models import Chunk, ChunkOptions | |
| 11 | + | |
| 12 | +_HEADING = re.compile(r"^(#{1,6})\s+(.*)$", re.M) | |
| 13 | +_SENT = re.compile(r"(?<=[.!?…»”\"])\s+(?=[A-ZÀ-ÝÉÈ0-9«“\"(])|\n{2,}") | |
| 14 | + | |
| 15 | + | |
| 16 | +@lru_cache | |
| 17 | +def _enc(): # type: ignore[no-untyped-def] | |
| 18 | + try: | |
| 19 | + import tiktoken | |
| 20 | + | |
| 21 | + return tiktoken.get_encoding("cl100k_base") | |
| 22 | + except Exception: # pragma: no cover | |
| 23 | + return None | |
| 24 | + | |
| 25 | + | |
| 26 | +def count_tokens(text: str) -> int: | |
| 27 | + enc = _enc() | |
| 28 | + if enc is None: | |
| 29 | + return max(1, len(text) // 4) | |
| 30 | + return len(enc.encode(text, disallowed_special=())) | |
| 31 | + | |
| 32 | + | |
| 33 | +def _sentences(text: str) -> list[str]: | |
| 34 | + parts = [p for p in _SENT.split(text) if p and p.strip()] | |
| 35 | + return parts or ([text] if text.strip() else []) | |
| 36 | + | |
| 37 | + | |
| 38 | +def _split_big(text: str, max_tokens: int, overlap: int) -> list[str]: | |
| 39 | + """Découpe un texte en morceaux ≤ max_tokens sur frontières de phrases, avec overlap.""" | |
| 40 | + sents = _sentences(text) | |
| 41 | + out: list[str] = [] | |
| 42 | + cur: list[str] = [] | |
| 43 | + cur_tok = 0 | |
| 44 | + for s in sents: | |
| 45 | + st = count_tokens(s) | |
| 46 | + if st > max_tokens: | |
| 47 | + # phrase géante : coupe brutale par mots | |
| 48 | + words = s.split() | |
| 49 | + step = max(1, int(len(words) * max_tokens / max(st, 1))) | |
| 50 | + for i in range(0, len(words), step): | |
| 51 | + piece = " ".join(words[i : i + step]) | |
| 52 | + if cur: | |
| 53 | + out.append(" ".join(cur)) | |
| 54 | + cur, cur_tok = [], 0 | |
| 55 | + out.append(piece) | |
| 56 | + continue | |
| 57 | + if cur_tok + st > max_tokens and cur: | |
| 58 | + out.append(" ".join(cur)) | |
| 59 | + # overlap : reprendre les dernières phrases jusqu'à `overlap` tokens | |
| 60 | + keep: list[str] = [] | |
| 61 | + kt = 0 | |
| 62 | + for prev in reversed(cur): | |
| 63 | + pt = count_tokens(prev) | |
| 64 | + if kt + pt > overlap: | |
| 65 | + break | |
| 66 | + keep.insert(0, prev) | |
| 67 | + kt += pt | |
| 68 | + cur, cur_tok = keep, kt | |
| 69 | + cur.append(s) | |
| 70 | + cur_tok += st | |
| 71 | + if cur: | |
| 72 | + out.append(" ".join(cur)) | |
| 73 | + return out | |
| 74 | + | |
| 75 | + | |
| 76 | +def chunk_by_tokens(md: str, url: str, opts: ChunkOptions) -> list[Chunk]: | |
| 77 | + pieces = _split_big(md, opts.size_tokens, opts.overlap_tokens) | |
| 78 | + return _finalize(pieces, url, md, heading_paths=None) | |
| 79 | + | |
| 80 | + | |
| 81 | +def chunk_by_heading(md: str, url: str, opts: ChunkOptions) -> list[Chunk]: | |
| 82 | + sections: list[tuple[list[str], str]] = [] # (heading_path, texte) | |
| 83 | + path: dict[int, str] = {} | |
| 84 | + pos = 0 | |
| 85 | + matches = list(_HEADING.finditer(md)) | |
| 86 | + if not matches: | |
| 87 | + return ( | |
| 88 | + chunk_by_tokens(md, url, opts) | |
| 89 | + if count_tokens(md) > opts.max_tokens | |
| 90 | + else _finalize([md], url, md, None) | |
| 91 | + ) | |
| 92 | + if matches[0].start() > 0: | |
| 93 | + sections.append(([], md[: matches[0].start()])) | |
| 94 | + for i, m in enumerate(matches): | |
| 95 | + lvl = len(m.group(1)) | |
| 96 | + title = m.group(2).strip() | |
| 97 | + for k in list(path): | |
| 98 | + if k >= lvl: | |
| 99 | + del path[k] | |
| 100 | + path[lvl] = title | |
| 101 | + end = matches[i + 1].start() if i + 1 < len(matches) else len(md) | |
| 102 | + body = md[m.end() : end] | |
| 103 | + sections.append(([path[k] for k in sorted(path)], m.group(0) + body)) | |
| 104 | + pos = end | |
| 105 | + del pos | |
| 106 | + # fusion des petites sections, split des grandes | |
| 107 | + merged: list[tuple[list[str], str]] = [] | |
| 108 | + for hp, txt in sections: | |
| 109 | + if merged and count_tokens(merged[-1][1]) < opts.min_tokens: | |
| 110 | + # la petite section précédente est absorbée : le chemin de titres devient celui de la section courante | |
| 111 | + _ph, pt = merged[-1] | |
| 112 | + merged[-1] = (hp if hp else _ph, pt.rstrip() + "\n\n" + txt.lstrip()) | |
| 113 | + else: | |
| 114 | + merged.append((hp, txt)) | |
| 115 | + pieces: list[str] = [] | |
| 116 | + paths: list[str] = [] | |
| 117 | + for hp, txt in merged: | |
| 118 | + hp_s = " > ".join(hp) | |
| 119 | + if count_tokens(txt) > opts.max_tokens: | |
| 120 | + for part in _split_big(txt, opts.max_tokens, opts.overlap_tokens): | |
| 121 | + pieces.append(part) | |
| 122 | + paths.append(hp_s) | |
| 123 | + else: | |
| 124 | + pieces.append(txt) | |
| 125 | + paths.append(hp_s) | |
| 126 | + return _finalize(pieces, url, md, paths) | |
| 127 | + | |
| 128 | + | |
| 129 | +def _finalize(pieces: list[str], url: str, md: str, heading_paths: list[str] | None) -> list[Chunk]: | |
| 130 | + out: list[Chunk] = [] | |
| 131 | + cursor = 0 | |
| 132 | + for i, p in enumerate(pieces): | |
| 133 | + text = p.strip() | |
| 134 | + if not text: | |
| 135 | + continue | |
| 136 | + start = md.find(text[:80], cursor) | |
| 137 | + if start < 0: | |
| 138 | + start = cursor | |
| 139 | + end = start + len(text) | |
| 140 | + cursor = max(cursor, start + 1) | |
| 141 | + out.append( | |
| 142 | + Chunk( | |
| 143 | + index=len(out), | |
| 144 | + text=text, | |
| 145 | + heading_path=heading_paths[i] if heading_paths else "", | |
| 146 | + token_count=count_tokens(text), | |
| 147 | + char_range=(start, end), | |
| 148 | + url=url, | |
| 149 | + ) | |
| 150 | + ) | |
| 151 | + return out | |
| 152 | + | |
| 153 | + | |
| 154 | +def chunk(md: str, url: str, opts: ChunkOptions) -> list[Chunk]: | |
| 155 | + if not md.strip(): | |
| 156 | + return [] | |
| 157 | + return ( | |
| 158 | + chunk_by_heading(md, url, opts) if opts.strategy == "by_heading" else chunk_by_tokens(md, url, opts) | |
| 159 | + ) | |
added
trawls/processors/encoding.py
+58 −0
@@ -0,0 +1,58 @@ | ||
| 1 | +"""Décodage robuste des corps HTTP : charset déclaré → meta HTML → charset-normalizer → latin-1. | |
| 2 | +Jamais d'exception.""" | |
| 3 | + | |
| 4 | +from __future__ import annotations | |
| 5 | + | |
| 6 | +import re | |
| 7 | + | |
| 8 | +_META_CHARSET = re.compile(rb"<meta[^>]+charset=[\"']?\s*([a-zA-Z0-9_\-]+)", re.I) | |
| 9 | +_XML_DECL = re.compile(rb"<\?xml[^>]+encoding=[\"']([a-zA-Z0-9_\-]+)", re.I) | |
| 10 | + | |
| 11 | + | |
| 12 | +def _from_content_type(ct: str | None) -> str | None: | |
| 13 | + if not ct: | |
| 14 | + return None | |
| 15 | + m = re.search(r"charset=[\"']?([a-zA-Z0-9_\-]+)", ct, re.I) | |
| 16 | + return m.group(1) if m else None | |
| 17 | + | |
| 18 | + | |
| 19 | +def _try(body: bytes, enc: str | None) -> str | None: | |
| 20 | + if not enc: | |
| 21 | + return None | |
| 22 | + try: | |
| 23 | + return body.decode(enc, errors="strict") | |
| 24 | + except (LookupError, UnicodeDecodeError): | |
| 25 | + return None | |
| 26 | + | |
| 27 | + | |
| 28 | +def decode_body(body: bytes, content_type: str | None = None) -> tuple[str, str]: | |
| 29 | + """Renvoie (texte, charset utilisé).""" | |
| 30 | + if not body: | |
| 31 | + return "", "utf-8" | |
| 32 | + if body.startswith(b"\xef\xbb\xbf"): | |
| 33 | + return body[3:].decode("utf-8", errors="replace"), "utf-8" | |
| 34 | + for enc in (_from_content_type(content_type),): | |
| 35 | + t = _try(body, enc) | |
| 36 | + if t is not None: | |
| 37 | + return t, enc or "utf-8" | |
| 38 | + head = body[:8192] | |
| 39 | + m = _META_CHARSET.search(head) or _XML_DECL.search(head) | |
| 40 | + if m: | |
| 41 | + enc = m.group(1).decode("ascii", "ignore") | |
| 42 | + t = _try(body, enc) | |
| 43 | + if t is not None: | |
| 44 | + return t, enc | |
| 45 | + t = _try(body, "utf-8") | |
| 46 | + if t is not None: | |
| 47 | + return t, "utf-8" | |
| 48 | + try: | |
| 49 | + from charset_normalizer import from_bytes | |
| 50 | + | |
| 51 | + best = from_bytes(body[:200_000]).best() | |
| 52 | + if best is not None and best.encoding: | |
| 53 | + t = _try(body, best.encoding) | |
| 54 | + if t is not None: | |
| 55 | + return t, best.encoding | |
| 56 | + except Exception: | |
| 57 | + pass | |
| 58 | + return body.decode("latin-1", errors="replace"), "latin-1" | |
added
trawls/processors/html_to_md/__init__.py
+60 −0
@@ -0,0 +1,60 @@ | ||
| 1 | +"""Pipeline HTML → Markdown (cœur du produit). Voir CLAUDE.md §4. | |
| 2 | + | |
| 3 | +`html_to_markdown(html, base_url, ...)` → (markdown, html_nettoyé) | |
| 4 | +""" | |
| 5 | + | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +from dataclasses import dataclass | |
| 9 | + | |
| 10 | +from trawls.processors.html_to_md.citations import add_citations | |
| 11 | +from trawls.processors.html_to_md.clean import clean | |
| 12 | +from trawls.processors.html_to_md.convert import Converter, postprocess | |
| 13 | +from trawls.processors.html_to_md.readability import find_main_content | |
| 14 | + | |
| 15 | + | |
| 16 | +@dataclass | |
| 17 | +class MdResult: | |
| 18 | + markdown: str | |
| 19 | + clean_html: str | |
| 20 | + main_selector_hint: str | |
| 21 | + | |
| 22 | + | |
| 23 | +def html_to_markdown( | |
| 24 | + html: str, | |
| 25 | + base_url: str = "", | |
| 26 | + *, | |
| 27 | + only_main_content: bool = True, | |
| 28 | + include_tags: list[str] | None = None, | |
| 29 | + exclude_tags: list[str] | None = None, | |
| 30 | + remove_base64_images: bool = True, | |
| 31 | + citations: bool = False, | |
| 32 | +) -> MdResult: | |
| 33 | + tree = clean( | |
| 34 | + html or "", | |
| 35 | + exclude_selectors=exclude_tags, | |
| 36 | + include_selectors=include_tags, | |
| 37 | + boilerplate=only_main_content, | |
| 38 | + ) | |
| 39 | + root = tree.body if tree.body is not None else tree.root | |
| 40 | + if root is None: | |
| 41 | + return MdResult("", "", "") | |
| 42 | + hint = "body" | |
| 43 | + if only_main_content: | |
| 44 | + main = find_main_content(tree) | |
| 45 | + if main is not None: | |
| 46 | + root = main | |
| 47 | + ident = " ".join( | |
| 48 | + filter(None, [main.tag, main.attributes.get("id"), main.attributes.get("class")]) | |
| 49 | + ) | |
| 50 | + hint = ident[:120] | |
| 51 | + conv = Converter(base_url=base_url, remove_base64_images=remove_base64_images) | |
| 52 | + md = conv.block(root) | |
| 53 | + md = postprocess(md) | |
| 54 | + if citations: | |
| 55 | + md = add_citations(md) | |
| 56 | + clean_html = root.html or "" | |
| 57 | + return MdResult(md, clean_html, hint) | |
| 58 | + | |
| 59 | + | |
| 60 | +__all__ = ["html_to_markdown", "MdResult"] | |
added
trawls/processors/html_to_md/citations.py
+22 −0
@@ -0,0 +1,22 @@ | ||
| 1 | +"""Étape 7 (option) : liens inline → références numérotées listées en fin de document.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import re | |
| 6 | + | |
| 7 | +_LINK = re.compile(r"(?<!!)\[([^\]]+)\]\((https?://[^)\s]+)(?: \"[^\"]*\")?\)") | |
| 8 | + | |
| 9 | + | |
| 10 | +def add_citations(md: str) -> str: | |
| 11 | + refs: dict[str, int] = {} | |
| 12 | + | |
| 13 | + def repl(m: re.Match[str]) -> str: | |
| 14 | + text, url = m.group(1), m.group(2) | |
| 15 | + n = refs.setdefault(url, len(refs) + 1) | |
| 16 | + return f"[{text}][{n}]" | |
| 17 | + | |
| 18 | + body = _LINK.sub(repl, md) | |
| 19 | + if not refs: | |
| 20 | + return md | |
| 21 | + lines = [f"[{n}]: {url}" for url, n in sorted(refs.items(), key=lambda kv: kv[1])] | |
| 22 | + return body.rstrip() + "\n\n" + "\n".join(lines) + "\n" | |
added
trawls/processors/html_to_md/clean.py
+176 −0
@@ -0,0 +1,176 @@ | ||
| 1 | +"""Étapes 1-3 du pipeline : parse (selectolax, fallback lxml), clean brut, suppression boilerplate. | |
| 2 | + | |
| 3 | +Chaque fonction est pure sur un arbre selectolax et testable isolément. | |
| 4 | +""" | |
| 5 | + | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import re | |
| 9 | + | |
| 10 | +from selectolax.parser import HTMLParser, Node | |
| 11 | + | |
| 12 | +_STRIP_TAGS = ( | |
| 13 | + "script", | |
| 14 | + "style", | |
| 15 | + "noscript", | |
| 16 | + "template", | |
| 17 | + "link", | |
| 18 | + "meta", | |
| 19 | + "object", | |
| 20 | + "embed", | |
| 21 | + "applet", | |
| 22 | + "canvas", | |
| 23 | + "map", | |
| 24 | +) | |
| 25 | +_HIDDEN_STYLE = re.compile(r"display\s*:\s*none|visibility\s*:\s*hidden", re.I) | |
| 26 | +_BOILER_CLASS = re.compile( | |
| 27 | + r"(^|[\s_-])(cookie|consent|gdpr|banner|popup|pop-up|modal|newsletter|subscribe|share|sharing|social|sidebar|" | |
| 28 | + r"related|recommend|advert|advertis|ads?|promo|breadcrumb|comments?|comment-list|disqus|footer|nav|navbar|menu|" | |
| 29 | + r"skip-link|toolbar|widget|sticky|toast|cta|paywall|login|signup|sign-up)([\s_-]|$)", | |
| 30 | + re.I, | |
| 31 | +) | |
| 32 | +_KEEP_CLASS = re.compile(r"(article|content|post|entry|story|body|main|text|prose|markdown)", re.I) | |
| 33 | +_SEMANTIC_BOILER = ("nav", "footer", "aside") | |
| 34 | +_ROLE_BOILER = {"navigation", "banner", "contentinfo", "complementary", "dialog", "alertdialog", "search"} | |
| 35 | + | |
| 36 | + | |
| 37 | +def parse_html(html: str) -> HTMLParser: | |
| 38 | + """Parse tolérant. Fallback lxml (recover) si l'arbre est irrécupérable.""" | |
| 39 | + try: | |
| 40 | + tree = HTMLParser(html) | |
| 41 | + if tree.body is not None or tree.root is not None: | |
| 42 | + return tree | |
| 43 | + except Exception: | |
| 44 | + pass | |
| 45 | + from lxml import html as lh | |
| 46 | + | |
| 47 | + doc = lh.fromstring(html or "<html><body></body></html>") | |
| 48 | + return HTMLParser(lh.tostring(doc, encoding="unicode")) | |
| 49 | + | |
| 50 | + | |
| 51 | +def _has_heading(node: Node) -> bool: | |
| 52 | + return node.css_first("h1") is not None | |
| 53 | + | |
| 54 | + | |
| 55 | +def _attr(node: Node, name: str) -> str: | |
| 56 | + v = node.attributes.get(name) | |
| 57 | + return v or "" | |
| 58 | + | |
| 59 | + | |
| 60 | +def strip_noise(tree: HTMLParser) -> None: | |
| 61 | + """Étape 2 : scripts, styles, éléments cachés, commentaires, iframes (sauf vidéo → lien).""" | |
| 62 | + for tag in _STRIP_TAGS: | |
| 63 | + for n in tree.css(tag): | |
| 64 | + n.decompose() | |
| 65 | + for n in tree.css("iframe"): | |
| 66 | + src = _attr(n, "src") | |
| 67 | + if re.search(r"youtube\.com|youtu\.be|vimeo\.com|dailymotion\.com", src, re.I): | |
| 68 | + n.replace_with(HTMLParser(f'<p><a href="{src}">[vidéo] {src}</a></p>').body.child) # type: ignore[union-attr] | |
| 69 | + else: | |
| 70 | + n.decompose() | |
| 71 | + for n in tree.css("svg"): | |
| 72 | + # svg décoratif (pas de <title>) → retiré | |
| 73 | + if n.css_first("title") is None: | |
| 74 | + n.decompose() | |
| 75 | + else: | |
| 76 | + t = n.css_first("title") | |
| 77 | + n.replace_with(HTMLParser(f"<span>{t.text() if t else ''}</span>").body.child) # type: ignore[union-attr] | |
| 78 | + for n in tree.css("[hidden], [aria-hidden='true'], [style]"): | |
| 79 | + if n.tag in ("html", "body"): | |
| 80 | + continue | |
| 81 | + st = _attr(n, "style") | |
| 82 | + if "hidden" in n.attributes or _attr(n, "aria-hidden") == "true" or _HIDDEN_STYLE.search(st): | |
| 83 | + n.decompose() | |
| 84 | + for n in tree.css("input[type=hidden], button, select, textarea, form > input, label > input"): | |
| 85 | + n.decompose() | |
| 86 | + # commentaires | |
| 87 | + root = tree.root | |
| 88 | + if root is not None: | |
| 89 | + for n in root.traverse(): | |
| 90 | + if n.tag == "_comment": | |
| 91 | + n.decompose() | |
| 92 | + | |
| 93 | + | |
| 94 | +def _text_len(node: Node) -> int: | |
| 95 | + return len(re.sub(r"\s+", " ", node.text(deep=True, separator=" ")).strip()) | |
| 96 | + | |
| 97 | + | |
| 98 | +def _link_density(node: Node) -> float: | |
| 99 | + total = _text_len(node) | |
| 100 | + if total == 0: | |
| 101 | + return 1.0 | |
| 102 | + links = sum(_text_len(a) for a in node.css("a")) | |
| 103 | + return min(1.0, links / total) | |
| 104 | + | |
| 105 | + | |
| 106 | +def strip_boilerplate( | |
| 107 | + tree: HTMLParser, exclude_selectors: list[str] | None = None, include_selectors: list[str] | None = None | |
| 108 | +) -> None: | |
| 109 | + """Étape 3 : nav/footer/aside/header, rôles ARIA, classes/ids suspects, densité de liens.""" | |
| 110 | + body = tree.body | |
| 111 | + if body is None: | |
| 112 | + return | |
| 113 | + protected: set[int] = set() | |
| 114 | + for sel in include_selectors or []: | |
| 115 | + try: | |
| 116 | + for n in tree.css(sel): | |
| 117 | + protected.add(id(n)) | |
| 118 | + for d in n.traverse(): | |
| 119 | + protected.add(id(d)) | |
| 120 | + except Exception: | |
| 121 | + pass | |
| 122 | + for sel in exclude_selectors or []: | |
| 123 | + try: | |
| 124 | + for n in tree.css(sel): | |
| 125 | + n.decompose() | |
| 126 | + except Exception: | |
| 127 | + pass | |
| 128 | + for tag in _SEMANTIC_BOILER: | |
| 129 | + for n in tree.css(tag): | |
| 130 | + if id(n) not in protected and (tag != "aside" or _text_len(n) < 2000 or _link_density(n) > 0.3): | |
| 131 | + n.decompose() | |
| 132 | + for n in tree.css("header"): | |
| 133 | + if id(n) not in protected and not _has_heading(n): | |
| 134 | + n.decompose() | |
| 135 | + for n in tree.css("[role]"): | |
| 136 | + if id(n) not in protected and _attr(n, "role").lower() in _ROLE_BOILER: | |
| 137 | + n.decompose() | |
| 138 | + # classes/ids suspects — on n'élimine que si le bloc n'a pas l'air d'être le contenu | |
| 139 | + candidates = list(tree.css("div, section, ul, ol, span, p, aside, table")) | |
| 140 | + for n in candidates: | |
| 141 | + if id(n) in protected or n.parent is None: | |
| 142 | + continue | |
| 143 | + ident = f"{_attr(n, 'class')} {_attr(n, 'id')}" | |
| 144 | + if not ident.strip() or not _BOILER_CLASS.search(ident): | |
| 145 | + continue | |
| 146 | + if _KEEP_CLASS.search(ident) and _text_len(n) > 400 and _link_density(n) < 0.3: | |
| 147 | + continue | |
| 148 | + if _text_len(n) > 3000 and _link_density(n) < 0.2: | |
| 149 | + continue | |
| 150 | + n.decompose() | |
| 151 | + # densité de liens : listes de liens / blocs très courts répétés | |
| 152 | + for n in tree.css("ul, ol, div, section"): | |
| 153 | + if id(n) in protected or n.parent is None: | |
| 154 | + continue | |
| 155 | + tl = _text_len(n) | |
| 156 | + if tl == 0: | |
| 157 | + if n.css_first("img") is None: | |
| 158 | + n.decompose() | |
| 159 | + continue | |
| 160 | + if tl < 25 and n.tag in ("div", "section") and n.css_first("img, h1, h2, h3, pre, code") is None: | |
| 161 | + continue | |
| 162 | + if _link_density(n) > 0.6 and len(n.css("a")) >= 4 and tl < 1500: | |
| 163 | + n.decompose() | |
| 164 | + | |
| 165 | + | |
| 166 | +def clean( | |
| 167 | + html: str, | |
| 168 | + exclude_selectors: list[str] | None = None, | |
| 169 | + include_selectors: list[str] | None = None, | |
| 170 | + boilerplate: bool = True, | |
| 171 | +) -> HTMLParser: | |
| 172 | + tree = parse_html(html) | |
| 173 | + strip_noise(tree) | |
| 174 | + if boilerplate: | |
| 175 | + strip_boilerplate(tree, exclude_selectors, include_selectors) | |
| 176 | + return tree | |
added
trawls/processors/html_to_md/convert.py
+385 −0
@@ -0,0 +1,385 @@ | ||
| 1 | +"""Étape 5 : DOM (selectolax) → Markdown. Gestion explicite de h1-h6, p, br, blockquote, pre/code, | |
| 2 | +listes imbriquées, strong/em, a (URLs absolutisées), img, tables, dl, details, figure, hr. | |
| 3 | +""" | |
| 4 | + | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import re | |
| 8 | +from urllib.parse import urljoin | |
| 9 | + | |
| 10 | +from selectolax.parser import Node | |
| 11 | + | |
| 12 | +from trawls.processors.html_to_md.tables import is_layout_table, table_to_md | |
| 13 | + | |
| 14 | +_BLOCK = { | |
| 15 | + "p", | |
| 16 | + "div", | |
| 17 | + "section", | |
| 18 | + "article", | |
| 19 | + "main", | |
| 20 | + "header", | |
| 21 | + "footer", | |
| 22 | + "aside", | |
| 23 | + "nav", | |
| 24 | + "ul", | |
| 25 | + "ol", | |
| 26 | + "li", | |
| 27 | + "table", | |
| 28 | + "blockquote", | |
| 29 | + "pre", | |
| 30 | + "h1", | |
| 31 | + "h2", | |
| 32 | + "h3", | |
| 33 | + "h4", | |
| 34 | + "h5", | |
| 35 | + "h6", | |
| 36 | + "hr", | |
| 37 | + "figure", | |
| 38 | + "figcaption", | |
| 39 | + "dl", | |
| 40 | + "dt", | |
| 41 | + "dd", | |
| 42 | + "details", | |
| 43 | + "summary", | |
| 44 | + "address", | |
| 45 | + "fieldset", | |
| 46 | + "form", | |
| 47 | + "body", | |
| 48 | + "html", | |
| 49 | + "tr", | |
| 50 | + "td", | |
| 51 | + "th", | |
| 52 | + "thead", | |
| 53 | + "tbody", | |
| 54 | + "tfoot", | |
| 55 | + "center", | |
| 56 | + "caption", | |
| 57 | + "video", | |
| 58 | + "audio", | |
| 59 | + "picture", | |
| 60 | +} | |
| 61 | +_SKIP = { | |
| 62 | + "script", | |
| 63 | + "style", | |
| 64 | + "noscript", | |
| 65 | + "template", | |
| 66 | + "head", | |
| 67 | + "title", | |
| 68 | + "meta", | |
| 69 | + "link", | |
| 70 | + "button", | |
| 71 | + "input", | |
| 72 | + "select", | |
| 73 | + "option", | |
| 74 | + "textarea", | |
| 75 | +} | |
| 76 | +_WS = re.compile(r"[ \t\r\f\v]+") | |
| 77 | +_NL = re.compile(r"\n{3,}") | |
| 78 | + | |
| 79 | + | |
| 80 | +class Converter: | |
| 81 | + def __init__(self, base_url: str = "", remove_base64_images: bool = True) -> None: | |
| 82 | + self.base = base_url | |
| 83 | + self.remove_b64 = remove_base64_images | |
| 84 | + self.h1_seen = False | |
| 85 | + | |
| 86 | + # ---- helpers ------------------------------------------------------------- | |
| 87 | + | |
| 88 | + def abs_url(self, u: str | None) -> str: | |
| 89 | + if not u: | |
| 90 | + return "" | |
| 91 | + u = u.strip() | |
| 92 | + if u.startswith(("javascript:", "#")): | |
| 93 | + return "" | |
| 94 | + return urljoin(self.base, u) if self.base else u | |
| 95 | + | |
| 96 | + def _children(self, node: Node) -> list[Node]: | |
| 97 | + out = [] | |
| 98 | + c = node.child | |
| 99 | + while c is not None: | |
| 100 | + out.append(c) | |
| 101 | + c = c.next | |
| 102 | + return out | |
| 103 | + | |
| 104 | + # ---- inline -------------------------------------------------------------- | |
| 105 | + | |
| 106 | + def inline(self, node: Node) -> str: | |
| 107 | + """Rend le contenu inline d'un nœud (sans ses bordures de bloc).""" | |
| 108 | + return "".join(self._inline_node(c) for c in self._children(node)) | |
| 109 | + | |
| 110 | + def _inline_node(self, n: Node) -> str: | |
| 111 | + tag = n.tag | |
| 112 | + if tag == "-text": | |
| 113 | + return _WS.sub(" ", n.text_content or "") | |
| 114 | + if tag in _SKIP or tag == "_comment": | |
| 115 | + return "" | |
| 116 | + if tag == "br": | |
| 117 | + return " \n" | |
| 118 | + if tag in ("strong", "b"): | |
| 119 | + inner = self.inline(n).strip() | |
| 120 | + return f"**{inner}**" if inner else "" | |
| 121 | + if tag in ("em", "i", "cite", "var", "dfn"): | |
| 122 | + inner = self.inline(n).strip() | |
| 123 | + return f"*{inner}*" if inner else "" | |
| 124 | + if tag in ("s", "del", "strike"): | |
| 125 | + inner = self.inline(n).strip() | |
| 126 | + return f"~~{inner}~~" if inner else "" | |
| 127 | + if tag in ("code", "kbd", "samp", "tt"): | |
| 128 | + inner = n.text(deep=True) | |
| 129 | + inner = inner.replace("\n", " ").strip() | |
| 130 | + if not inner: | |
| 131 | + return "" | |
| 132 | + fence = "``" if "`" in inner else "`" | |
| 133 | + return f"{fence}{inner}{fence}" | |
| 134 | + if tag == "a": | |
| 135 | + href = self.abs_url(n.attributes.get("href")) | |
| 136 | + inner = self.inline(n).strip() | |
| 137 | + if not inner: | |
| 138 | + img = n.css_first("img") | |
| 139 | + inner = (img.attributes.get("alt") or "").strip() if img else "" | |
| 140 | + if not href: | |
| 141 | + return inner | |
| 142 | + if not inner: | |
| 143 | + return "" | |
| 144 | + title = (n.attributes.get("title") or "").replace('"', "'").strip() | |
| 145 | + t = f' "{title}"' if title and title != inner else "" | |
| 146 | + return f"[{inner}]({href}{t})" | |
| 147 | + if tag == "img": | |
| 148 | + return self._img(n) | |
| 149 | + if tag in ("sup",): | |
| 150 | + inner = self.inline(n).strip() | |
| 151 | + return f"^{inner}^" if inner and len(inner) < 12 else inner | |
| 152 | + if tag in ("sub",): | |
| 153 | + inner = self.inline(n).strip() | |
| 154 | + return f"~{inner}~" if inner and len(inner) < 12 else inner | |
| 155 | + if tag in ( | |
| 156 | + "mark", | |
| 157 | + "u", | |
| 158 | + "span", | |
| 159 | + "abbr", | |
| 160 | + "time", | |
| 161 | + "small", | |
| 162 | + "big", | |
| 163 | + "font", | |
| 164 | + "label", | |
| 165 | + "q", | |
| 166 | + "wbr", | |
| 167 | + "bdi", | |
| 168 | + "bdo", | |
| 169 | + "data", | |
| 170 | + "ins", | |
| 171 | + "ruby", | |
| 172 | + "rt", | |
| 173 | + "output", | |
| 174 | + "svg", | |
| 175 | + "path", | |
| 176 | + "use", | |
| 177 | + "picture", | |
| 178 | + "source", | |
| 179 | + ): | |
| 180 | + if tag == "q": | |
| 181 | + return f"“{self.inline(n).strip()}”" | |
| 182 | + return self.inline(n) | |
| 183 | + if tag in _BLOCK: | |
| 184 | + # bloc à l'intérieur d'un inline : on rend en bloc, séparé par des retours | |
| 185 | + return "\n\n" + self.block(n).strip() + "\n\n" | |
| 186 | + return self.inline(n) | |
| 187 | + | |
| 188 | + def _img(self, n: Node) -> str: | |
| 189 | + src = ( | |
| 190 | + n.attributes.get("src") or n.attributes.get("data-src") or n.attributes.get("data-lazy-src") or "" | |
| 191 | + ) | |
| 192 | + if not src and n.attributes.get("srcset"): | |
| 193 | + src = n.attributes["srcset"].split(",")[0].split()[0] | |
| 194 | + if not src: | |
| 195 | + return "" | |
| 196 | + if src.startswith("data:"): | |
| 197 | + if self.remove_b64: | |
| 198 | + return "" | |
| 199 | + else: | |
| 200 | + src = self.abs_url(src) | |
| 201 | + alt = (n.attributes.get("alt") or "").strip().replace("]", ")") | |
| 202 | + title = (n.attributes.get("title") or "").replace('"', "'").strip() | |
| 203 | + t = f' "{title}"' if title else "" | |
| 204 | + return f"" | |
| 205 | + | |
| 206 | + # ---- blocks -------------------------------------------------------------- | |
| 207 | + | |
| 208 | + def block(self, node: Node, list_depth: int = 0) -> str: | |
| 209 | + parts: list[str] = [] | |
| 210 | + buf: list[str] = [] # inline en cours | |
| 211 | + | |
| 212 | + def flush() -> None: | |
| 213 | + if buf: | |
| 214 | + txt = "".join(buf) | |
| 215 | + txt = re.sub(r"[ \t]*\n[ \t]*", "\n", txt).strip() | |
| 216 | + if txt: | |
| 217 | + parts.append(txt) | |
| 218 | + buf.clear() | |
| 219 | + | |
| 220 | + for c in self._children(node): | |
| 221 | + tag = c.tag | |
| 222 | + if tag == "-text" or tag not in _BLOCK and tag not in _SKIP: | |
| 223 | + buf.append(self._inline_node(c)) | |
| 224 | + continue | |
| 225 | + if tag in _SKIP or tag == "_comment": | |
| 226 | + continue | |
| 227 | + flush() | |
| 228 | + rendered = self._block_node(c, list_depth) | |
| 229 | + if rendered.strip(): | |
| 230 | + parts.append(rendered.strip("\n")) | |
| 231 | + flush() | |
| 232 | + return "\n\n".join(p for p in parts if p.strip()) | |
| 233 | + | |
| 234 | + def _block_node(self, n: Node, list_depth: int) -> str: | |
| 235 | + tag = n.tag | |
| 236 | + if tag in ("h1", "h2", "h3", "h4", "h5", "h6"): | |
| 237 | + lvl = int(tag[1]) | |
| 238 | + if lvl == 1: | |
| 239 | + if self.h1_seen: | |
| 240 | + lvl = 2 | |
| 241 | + self.h1_seen = True | |
| 242 | + text = self.inline(n).strip().replace("\n", " ") | |
| 243 | + return f"{'#' * lvl} {text}" if text else "" | |
| 244 | + if tag == "p": | |
| 245 | + return self.inline(n).strip() | |
| 246 | + if tag == "hr": | |
| 247 | + return "---" | |
| 248 | + if tag == "br": | |
| 249 | + return "" | |
| 250 | + if tag in ("ul", "ol"): | |
| 251 | + return self._list(n, list_depth) | |
| 252 | + if tag == "li": | |
| 253 | + return self.block(n, list_depth) | |
| 254 | + if tag == "blockquote": | |
| 255 | + inner = self.block(n, list_depth).strip() | |
| 256 | + return "\n".join(f"> {ln}" if ln else ">" for ln in inner.split("\n")) if inner else "" | |
| 257 | + if tag == "pre": | |
| 258 | + code = n.css_first("code") | |
| 259 | + lang = "" | |
| 260 | + cls = ( | |
| 261 | + (code.attributes.get("class") if code is not None else None) | |
| 262 | + or n.attributes.get("class") | |
| 263 | + or "" | |
| 264 | + ) | |
| 265 | + m = re.search(r"(?:language|lang)-([\w+#-]+)", cls) or re.search( | |
| 266 | + r"\b(python|javascript|js|ts|typescript|bash|shell|json|yaml|html|css|sql|go|rust|java|c|cpp|ruby|php)\b", | |
| 267 | + cls, | |
| 268 | + ) | |
| 269 | + if m: | |
| 270 | + lang = m.group(1) | |
| 271 | + text = n.text(deep=True).strip("\n") | |
| 272 | + fence = "````" if "```" in text else "```" | |
| 273 | + return f"{fence}{lang}\n{text}\n{fence}" | |
| 274 | + if tag == "table": | |
| 275 | + if is_layout_table(n): | |
| 276 | + return self.block(n, list_depth) | |
| 277 | + return table_to_md(n, self.inline) | |
| 278 | + if tag in ("dl",): | |
| 279 | + out: list[str] = [] | |
| 280 | + for c in self._children(n): | |
| 281 | + if c.tag == "dt": | |
| 282 | + out.append(f"**{self.inline(c).strip()}**") | |
| 283 | + elif c.tag == "dd": | |
| 284 | + inner = self.block(c, list_depth).strip() | |
| 285 | + out.append(": " + inner.replace("\n", "\n ")) | |
| 286 | + elif c.tag == "div": | |
| 287 | + out.append(self.block(c, list_depth)) | |
| 288 | + return "\n".join(o for o in out if o.strip()) | |
| 289 | + if tag == "details": | |
| 290 | + summary = n.css_first("summary") | |
| 291 | + title = self.inline(summary).strip() if summary is not None else "Détails" | |
| 292 | + body_parts = [ | |
| 293 | + self._block_node(c, list_depth) if c.tag in _BLOCK else self._inline_node(c) | |
| 294 | + for c in self._children(n) | |
| 295 | + if c.tag != "summary" | |
| 296 | + ] | |
| 297 | + body = "\n\n".join(p.strip() for p in body_parts if p.strip()) | |
| 298 | + return f"**{title}**\n\n{body}" if body else f"**{title}**" | |
| 299 | + if tag == "figure": | |
| 300 | + cap = n.css_first("figcaption") | |
| 301 | + cap_txt = self.inline(cap).strip() if cap is not None else "" | |
| 302 | + imgs = [self._img(i) for i in n.css("img")] | |
| 303 | + body = "\n\n".join(i for i in imgs if i) or self.block(n, list_depth) | |
| 304 | + if cap_txt and cap_txt not in body: | |
| 305 | + body = f"{body}\n\n*{cap_txt}*" | |
| 306 | + return body | |
| 307 | + if tag == "figcaption": | |
| 308 | + return "" | |
| 309 | + if tag in ("video", "audio"): | |
| 310 | + src = n.attributes.get("src") or ( | |
| 311 | + n.css_first("source").attributes.get("src") if n.css_first("source") else "" | |
| 312 | + ) | |
| 313 | + return f"[{tag}]({self.abs_url(src)})" if src else "" | |
| 314 | + if tag in ("td", "th", "tr", "thead", "tbody", "tfoot", "caption"): | |
| 315 | + return self.block(n, list_depth) | |
| 316 | + # conteneurs génériques | |
| 317 | + return self.block(n, list_depth) | |
| 318 | + | |
| 319 | + def _list(self, n: Node, depth: int) -> str: | |
| 320 | + ordered = n.tag == "ol" | |
| 321 | + try: | |
| 322 | + start = int(n.attributes.get("start") or 1) | |
| 323 | + except ValueError: | |
| 324 | + start = 1 | |
| 325 | + items: list[str] = [] | |
| 326 | + idx = start | |
| 327 | + for c in self._children(n): | |
| 328 | + if c.tag != "li": | |
| 329 | + if c.tag in ("ul", "ol"): | |
| 330 | + items.append(self._indent(self._list(c, depth + 1), " ")) | |
| 331 | + continue | |
| 332 | + # séparer sous-listes du contenu inline du li | |
| 333 | + sub: list[str] = [] | |
| 334 | + inline_parts: list[str] = [] | |
| 335 | + for cc in self._children(c): | |
| 336 | + if cc.tag in ("ul", "ol"): | |
| 337 | + sub.append(self._list(cc, depth + 1)) | |
| 338 | + elif cc.tag in _BLOCK and cc.tag not in ("p",): | |
| 339 | + inline_parts.append("\n\n" + self._block_node(cc, depth + 1)) | |
| 340 | + elif cc.tag == "p": | |
| 341 | + inline_parts.append("\n\n" + self.inline(cc).strip()) | |
| 342 | + else: | |
| 343 | + inline_parts.append(self._inline_node(cc)) | |
| 344 | + text = "".join(inline_parts) | |
| 345 | + text = re.sub(r"[ \t]*\n[ \t]*", "\n", text).strip() | |
| 346 | + text = _NL.sub("\n\n", text) | |
| 347 | + bullet = f"{idx}." if ordered else "-" | |
| 348 | + pad = " " * (len(bullet) + 1) | |
| 349 | + body = text.replace("\n", "\n" + pad) | |
| 350 | + if not body and not sub: | |
| 351 | + continue | |
| 352 | + line = f"{bullet} {body}".rstrip() | |
| 353 | + if sub: | |
| 354 | + line += "\n" + "\n".join(self._indent(s, pad) for s in sub) | |
| 355 | + items.append(line) | |
| 356 | + idx += 1 | |
| 357 | + return "\n".join(items) | |
| 358 | + | |
| 359 | + @staticmethod | |
| 360 | + def _indent(text: str, pad: str) -> str: | |
| 361 | + return "\n".join(pad + ln if ln else ln for ln in text.split("\n")) | |
| 362 | + | |
| 363 | + | |
| 364 | +def postprocess(md: str) -> str: | |
| 365 | + """Étape 6 : lignes vides (>2 → 2), trim, NFC, zero-width, espaces en fin de ligne.""" | |
| 366 | + import unicodedata | |
| 367 | + | |
| 368 | + md = unicodedata.normalize("NFC", md) | |
| 369 | + md = re.sub(r"[]", "", md) | |
| 370 | + md = md.replace(" ", " ") | |
| 371 | + md = "\n".join(ln.rstrip() if not ln.endswith(" ") else ln for ln in md.split("\n")) | |
| 372 | + md = re.sub(r"^[ \t]+$", "", md, flags=re.M) | |
| 373 | + md = _NL.sub("\n\n", md) | |
| 374 | + # espaces multiples hors blocs de code | |
| 375 | + out: list[str] = [] | |
| 376 | + in_code = False | |
| 377 | + for ln in md.split("\n"): | |
| 378 | + if ln.startswith("```"): | |
| 379 | + in_code = not in_code | |
| 380 | + out.append(ln) | |
| 381 | + continue | |
| 382 | + if not in_code and not ln.startswith("|"): | |
| 383 | + ln = re.sub(r"(?<!^)(?<![-*\d.])[ ]{2,}(?! *$)", " ", ln) | |
| 384 | + out.append(ln) | |
| 385 | + return "\n".join(out).strip() + "\n" | |
added
trawls/processors/html_to_md/readability.py
+90 −0
@@ -0,0 +1,90 @@ | ||
| 1 | +"""Étape 4 : scoring du contenu principal (readability maison). | |
| 2 | + | |
| 3 | +score = longueur texte × (1 − densité liens) × bonus(ponctuation, paragraphes) × bonus(classes contenu) | |
| 4 | +Le bloc gagnant est renvoyé ; ses ancêtres inutiles sont ignorés (on travaille sur le sous-arbre). | |
| 5 | +""" | |
| 6 | + | |
| 7 | +from __future__ import annotations | |
| 8 | + | |
| 9 | +import math | |
| 10 | +import re | |
| 11 | + | |
| 12 | +from selectolax.parser import HTMLParser, Node | |
| 13 | + | |
| 14 | +_POS = re.compile( | |
| 15 | + r"(article|content|post|entry|story|body|main|text|prose|markdown|blog|page|document|detail)", re.I | |
| 16 | +) | |
| 17 | +_NEG = re.compile( | |
| 18 | + r"(comment|sidebar|footer|header|nav|menu|related|share|social|promo|widget|ad-|ads|banner|meta|tag|author-box)", | |
| 19 | + re.I, | |
| 20 | +) | |
| 21 | +_CANDIDATE_TAGS = ("article", "main", "section", "div", "td", "body") | |
| 22 | + | |
| 23 | + | |
| 24 | +def _txt(node: Node) -> str: | |
| 25 | + return re.sub(r"\s+", " ", node.text(deep=True, separator=" ")).strip() | |
| 26 | + | |
| 27 | + | |
| 28 | +def _score(node: Node) -> float: | |
| 29 | + text = _txt(node) | |
| 30 | + n = len(text) | |
| 31 | + if n < 140: | |
| 32 | + return 0.0 | |
| 33 | + links = sum(len(_txt(a)) for a in node.css("a")) | |
| 34 | + density = min(1.0, links / n) if n else 1.0 | |
| 35 | + paras = len(node.css("p")) | |
| 36 | + commas = text.count(",") + text.count(".") + text.count("。") + text.count("!") + text.count("?") | |
| 37 | + bonus = 1.0 + min(2.0, paras / 8.0) + min(1.0, commas / 50.0) | |
| 38 | + ident = f"{node.attributes.get('class') or ''} {node.attributes.get('id') or ''}" | |
| 39 | + if ( | |
| 40 | + node.tag in ("article", "main") | |
| 41 | + or node.attributes.get("role") == "main" | |
| 42 | + or node.attributes.get("itemprop") == "articleBody" | |
| 43 | + ): | |
| 44 | + bonus *= 1.6 | |
| 45 | + if _POS.search(ident): | |
| 46 | + bonus *= 1.25 | |
| 47 | + if _NEG.search(ident): | |
| 48 | + bonus *= 0.5 | |
| 49 | + headings = len(node.css("h1, h2, h3")) | |
| 50 | + bonus *= 1.0 + min(0.5, headings * 0.08) | |
| 51 | + # pénaliser les blocs géants "tout le body" : log pour amortir | |
| 52 | + return math.log1p(n) * n * (1.0 - density) ** 2 * bonus / (1.0 + 0.0002 * n) | |
| 53 | + | |
| 54 | + | |
| 55 | +def find_main_content(tree: HTMLParser) -> Node | None: | |
| 56 | + body = tree.body | |
| 57 | + if body is None: | |
| 58 | + return None | |
| 59 | + best: Node | None = None | |
| 60 | + best_score = 0.0 | |
| 61 | + body_len = len(_txt(body)) | |
| 62 | + for tag in _CANDIDATE_TAGS: | |
| 63 | + for node in tree.css(tag): | |
| 64 | + s = _score(node) | |
| 65 | + if s > best_score: | |
| 66 | + best, best_score = node, s | |
| 67 | + if best is None or best.tag == "body": | |
| 68 | + return body | |
| 69 | + # Le gagnant doit porter une part significative du texte du body, sinon on risque de perdre du contenu | |
| 70 | + # (ex. articles multi-sections). On remonte tant que le parent n'ajoute pas trop de bruit. | |
| 71 | + node = best | |
| 72 | + while node.parent is not None and node.parent.tag not in ("body", "html"): | |
| 73 | + parent = node.parent | |
| 74 | + pl, nl = len(_txt(parent)), len(_txt(node)) | |
| 75 | + if nl == 0: | |
| 76 | + break | |
| 77 | + gain = (pl - nl) / max(1, nl) | |
| 78 | + p_density = _link_density(parent) | |
| 79 | + if gain <= 0.6 and p_density < 0.25 and nl / max(1, body_len) < 0.7: | |
| 80 | + node = parent | |
| 81 | + else: | |
| 82 | + break | |
| 83 | + return node | |
| 84 | + | |
| 85 | + | |
| 86 | +def _link_density(node: Node) -> float: | |
| 87 | + n = len(_txt(node)) | |
| 88 | + if n == 0: | |
| 89 | + return 1.0 | |
| 90 | + return min(1.0, sum(len(_txt(a)) for a in node.css("a")) / n) | |
added
trawls/processors/html_to_md/tables.py
+108 −0
@@ -0,0 +1,108 @@ | ||
| 1 | +"""Tables HTML → Markdown : thead absent, rowspan/colspan (duplication), cellules multi-lignes aplaties.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import re | |
| 6 | +from collections.abc import Callable | |
| 7 | + | |
| 8 | +from selectolax.parser import Node | |
| 9 | + | |
| 10 | + | |
| 11 | +def _cell_text(cell: Node, inline: Callable[[Node], str]) -> str: | |
| 12 | + t = inline(cell) | |
| 13 | + t = re.sub(r"\s*\n\s*", " ", t) | |
| 14 | + t = re.sub(r"\s{2,}", " ", t).strip() | |
| 15 | + return t.replace("|", "\\|") | |
| 16 | + | |
| 17 | + | |
| 18 | +def _int_attr(cell: Node, name: str) -> int: | |
| 19 | + try: | |
| 20 | + return max(1, int(cell.attributes.get(name) or 1)) | |
| 21 | + except ValueError: | |
| 22 | + return 1 | |
| 23 | + | |
| 24 | + | |
| 25 | +def table_to_md(table: Node, inline: Callable[[Node], str]) -> str: | |
| 26 | + rows: list[list[str]] = [] | |
| 27 | + header_idx: int | None = None | |
| 28 | + pending: dict[int, tuple[str, int]] = {} # col → (texte, rowspan restant) | |
| 29 | + for tr in _direct_rows(table): | |
| 30 | + cells = [c for c in tr.iter() if c.tag in ("td", "th")] | |
| 31 | + row: list[str] = [] | |
| 32 | + col = 0 | |
| 33 | + ci = 0 | |
| 34 | + while ci < len(cells) or col in pending: | |
| 35 | + if col in pending: | |
| 36 | + txt, left = pending[col] | |
| 37 | + row.append(txt) | |
| 38 | + if left - 1 <= 0: | |
| 39 | + del pending[col] | |
| 40 | + else: | |
| 41 | + pending[col] = (txt, left - 1) | |
| 42 | + col += 1 | |
| 43 | + continue | |
| 44 | + cell = cells[ci] | |
| 45 | + ci += 1 | |
| 46 | + txt = _cell_text(cell, inline) | |
| 47 | + cs, rs = _int_attr(cell, "colspan"), _int_attr(cell, "rowspan") | |
| 48 | + for k in range(min(cs, 50)): | |
| 49 | + row.append(txt) | |
| 50 | + if rs > 1: | |
| 51 | + pending[col + k] = (txt, rs - 1) | |
| 52 | + col += cs | |
| 53 | + if not row: | |
| 54 | + continue | |
| 55 | + is_header = all(c.tag == "th" for c in cells) and cells and header_idx is None and not rows | |
| 56 | + rows.append(row) | |
| 57 | + if is_header: | |
| 58 | + header_idx = len(rows) - 1 | |
| 59 | + if not rows: | |
| 60 | + return "" | |
| 61 | + width = max(len(r) for r in rows) | |
| 62 | + rows = [r + [""] * (width - len(r)) for r in rows] | |
| 63 | + if header_idx is None: | |
| 64 | + # thead absent : première ligne = en-tête si elle est courte et sans chiffres dominants, sinon en-tête vide | |
| 65 | + first = rows[0] | |
| 66 | + if sum(1 for c in first if re.fullmatch(r"[\d\s.,%$€-]+", c or "x")) <= width // 2: | |
| 67 | + header, body = first, rows[1:] | |
| 68 | + else: | |
| 69 | + header, body = [""] * width, rows | |
| 70 | + else: | |
| 71 | + header, body = rows[header_idx], rows[:header_idx] + rows[header_idx + 1 :] | |
| 72 | + if not body and not any(header): | |
| 73 | + return "" | |
| 74 | + caption = table.css_first("caption") | |
| 75 | + out: list[str] = [] | |
| 76 | + if caption is not None: | |
| 77 | + out.append(f"**{_cell_text(caption, inline)}**\n") | |
| 78 | + out.append("| " + " | ".join(header) + " |") | |
| 79 | + out.append("| " + " | ".join("---" for _ in range(width)) + " |") | |
| 80 | + for r in body: | |
| 81 | + out.append("| " + " | ".join(r) + " |") | |
| 82 | + return "\n".join(out) | |
| 83 | + | |
| 84 | + | |
| 85 | +def _direct_rows(table: Node) -> list[Node]: | |
| 86 | + """Lignes <tr> appartenant à cette table (pas aux tables imbriquées) : enfants directs ou via thead/tbody/tfoot.""" | |
| 87 | + rows: list[Node] = [] | |
| 88 | + for c in table.iter(): | |
| 89 | + if c.tag == "tr": | |
| 90 | + rows.append(c) | |
| 91 | + elif c.tag in ("thead", "tbody", "tfoot"): | |
| 92 | + rows.extend(cc for cc in c.iter() if cc.tag == "tr") | |
| 93 | + return rows | |
| 94 | + | |
| 95 | + | |
| 96 | +def is_layout_table(table: Node) -> bool: | |
| 97 | + """Table de mise en page (1 colonne / pas de th / cellules géantes) → traiter comme des blocs.""" | |
| 98 | + trs = table.css("tr") | |
| 99 | + if not trs: | |
| 100 | + return True | |
| 101 | + if table.css_first("th") is not None: | |
| 102 | + return False | |
| 103 | + cols = max((len([c for c in tr.iter() if c.tag in ("td", "th")]) for tr in trs), default=0) | |
| 104 | + if cols <= 1: | |
| 105 | + return True | |
| 106 | + if len(trs) <= 2 and any(len(td.text(deep=True)) > 800 for td in table.css("td")): | |
| 107 | + return True | |
| 108 | + return table.css_first("table") is not None | |
added
trawls/processors/links.py
+58 −0
@@ -0,0 +1,58 @@ | ||
| 1 | +"""Extraction et classification des liens (interne / externe / asset / mailto / tel).""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import re | |
| 6 | +from urllib.parse import urlsplit | |
| 7 | + | |
| 8 | +from selectolax.parser import HTMLParser | |
| 9 | + | |
| 10 | +from trawls.core.scheduler.dedup import normalize_url, same_site | |
| 11 | +from trawls.models import Link | |
| 12 | + | |
| 13 | +_ASSET_EXT = re.compile( | |
| 14 | + r"\.(jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|mp4|webm|mov|avi|mkv|mp3|wav|ogg|flac|zip|rar|7z|tar|gz|bz2|dmg|exe|msi|apk|" | |
| 15 | + r"css|js|mjs|woff2?|ttf|otf|eot|xml|rss|atom|json|csv|xlsx?|docx?|pptx?)(\?|#|$)", | |
| 16 | + re.I, | |
| 17 | +) | |
| 18 | + | |
| 19 | + | |
| 20 | +def extract_links( | |
| 21 | + html: str, base_url: str, allow_subdomains: bool = False, max_links: int = 5000 | |
| 22 | +) -> list[Link]: | |
| 23 | + tree = HTMLParser(html or "") | |
| 24 | + base_el = tree.css_first("base[href]") | |
| 25 | + base = base_el.attributes.get("href") or base_url if base_el is not None else base_url | |
| 26 | + seen: set[str] = set() | |
| 27 | + out: list[Link] = [] | |
| 28 | + for a in tree.css("a[href]"): | |
| 29 | + raw = (a.attributes.get("href") or "").strip() | |
| 30 | + if not raw or raw.startswith("#"): | |
| 31 | + continue | |
| 32 | + low = raw.lower() | |
| 33 | + if low.startswith("mailto:"): | |
| 34 | + kind = "mailto" | |
| 35 | + href = raw | |
| 36 | + elif low.startswith("tel:"): | |
| 37 | + kind = "tel" | |
| 38 | + href = raw | |
| 39 | + elif low.startswith(("javascript:", "data:", "blob:", "about:")): | |
| 40 | + continue | |
| 41 | + else: | |
| 42 | + n = normalize_url(raw, base) | |
| 43 | + if not n: | |
| 44 | + continue | |
| 45 | + href = n | |
| 46 | + if _ASSET_EXT.search(urlsplit(href).path): | |
| 47 | + kind = "asset" | |
| 48 | + else: | |
| 49 | + kind = "internal" if same_site(base_url, href, allow_subdomains) else "external" | |
| 50 | + if href in seen: | |
| 51 | + continue | |
| 52 | + seen.add(href) | |
| 53 | + rel = a.attributes.get("rel") | |
| 54 | + text = re.sub(r"\s+", " ", a.text(deep=True)).strip()[:200] | |
| 55 | + out.append(Link(href=href, text=text, rel=rel, kind=kind, nofollow=bool(rel and "nofollow" in rel))) # type: ignore[arg-type] | |
| 56 | + if len(out) >= max_links: | |
| 57 | + break | |
| 58 | + return out | |
added
trawls/processors/pdf/__init__.py
+213 −0
@@ -0,0 +1,213 @@ | ||
| 1 | +"""PDF → Markdown : texte avec ordre de lecture (pymupdf sort=True), titres inférés par taille de police, | |
| 2 | +tables via find_tables(), détection scan (< 50 caractères/page) → OCR rapidocr si installé. | |
| 3 | +""" | |
| 4 | + | |
| 5 | +from __future__ import annotations | |
| 6 | + | |
| 7 | +import io | |
| 8 | +import statistics | |
| 9 | +from dataclasses import dataclass, field | |
| 10 | +from typing import Any | |
| 11 | + | |
| 12 | +import structlog | |
| 13 | + | |
| 14 | +from trawls.config import get_settings | |
| 15 | + | |
| 16 | +log = structlog.get_logger(__name__) | |
| 17 | + | |
| 18 | + | |
| 19 | +@dataclass | |
| 20 | +class PdfResult: | |
| 21 | + markdown: str | |
| 22 | + pages: int | |
| 23 | + is_scanned: bool | |
| 24 | + title: str | None = None | |
| 25 | + author: str | None = None | |
| 26 | + created: str | None = None | |
| 27 | + modified: str | None = None | |
| 28 | + tables: int = 0 | |
| 29 | + ocr_pages: int = 0 | |
| 30 | + warnings: list[str] = field(default_factory=list) | |
| 31 | + | |
| 32 | + | |
| 33 | +def _ocr_page(page: Any, dpi: int = 200) -> str: | |
| 34 | + try: | |
| 35 | + from rapidocr_onnxruntime import RapidOCR # type: ignore | |
| 36 | + except Exception: | |
| 37 | + return "" | |
| 38 | + try: | |
| 39 | + pix = page.get_pixmap(dpi=dpi) | |
| 40 | + engine = _ocr_engine(RapidOCR) | |
| 41 | + result, _ = engine(pix.tobytes("png")) | |
| 42 | + if not result: | |
| 43 | + return "" | |
| 44 | + lines = [r[1] for r in result if len(r) > 1] | |
| 45 | + return "\n".join(lines) | |
| 46 | + except Exception as e: # pragma: no cover | |
| 47 | + log.info("pdf.ocr_failed", error=str(e)) | |
| 48 | + return "" | |
| 49 | + | |
| 50 | + | |
| 51 | +_ENGINE: Any = None | |
| 52 | + | |
| 53 | + | |
| 54 | +def _ocr_engine(cls: Any) -> Any: | |
| 55 | + global _ENGINE | |
| 56 | + if _ENGINE is None: | |
| 57 | + _ENGINE = cls() | |
| 58 | + return _ENGINE | |
| 59 | + | |
| 60 | + | |
| 61 | +def _table_to_md(tbl: Any) -> str: | |
| 62 | + try: | |
| 63 | + rows = tbl.extract() | |
| 64 | + except Exception: | |
| 65 | + return "" | |
| 66 | + rows = [[(c or "").replace("\n", " ").replace("|", "\\|").strip() for c in r] for r in rows if r] | |
| 67 | + if not rows: | |
| 68 | + return "" | |
| 69 | + width = max(len(r) for r in rows) | |
| 70 | + rows = [r + [""] * (width - len(r)) for r in rows] | |
| 71 | + header, body = rows[0], rows[1:] | |
| 72 | + out = ["| " + " | ".join(header) + " |", "| " + " | ".join("---" for _ in range(width)) + " |"] | |
| 73 | + out += ["| " + " | ".join(r) + " |" for r in body] | |
| 74 | + return "\n".join(out) | |
| 75 | + | |
| 76 | + | |
| 77 | +def pdf_to_markdown(data: bytes, max_pages: int | None = None) -> PdfResult: | |
| 78 | + import pymupdf as fitz | |
| 79 | + | |
| 80 | + settings = get_settings() | |
| 81 | + max_pages = max_pages or settings.max_pages_pdf | |
| 82 | + warnings: list[str] = [] | |
| 83 | + try: | |
| 84 | + doc = fitz.open(stream=io.BytesIO(data), filetype="pdf") | |
| 85 | + except Exception as e: | |
| 86 | + raise ValueError(f"PDF irrécupérable: {e}") from None | |
| 87 | + if doc.is_encrypted: | |
| 88 | + try: | |
| 89 | + doc.authenticate("") | |
| 90 | + except Exception: | |
| 91 | + pass | |
| 92 | + meta = doc.metadata or {} | |
| 93 | + n_pages = doc.page_count | |
| 94 | + if n_pages > max_pages: | |
| 95 | + warnings.append(f"{n_pages} pages, tronqué à {max_pages}") | |
| 96 | + pages_md: list[str] = [] | |
| 97 | + total_chars = 0 | |
| 98 | + ocr_pages = 0 | |
| 99 | + tables_count = 0 | |
| 100 | + sizes: list[float] = [] | |
| 101 | + # 1re passe : tailles de police pour inférer les titres | |
| 102 | + spans_by_page: list[list[tuple[str, float, bool]]] = [] | |
| 103 | + for pno in range(min(n_pages, max_pages)): | |
| 104 | + page = doc[pno] | |
| 105 | + try: | |
| 106 | + d = page.get_text("dict", sort=True) | |
| 107 | + except Exception: | |
| 108 | + spans_by_page.append([]) | |
| 109 | + continue | |
| 110 | + spans: list[tuple[str, float, bool]] = [] | |
| 111 | + for block in d.get("blocks", []): | |
| 112 | + if block.get("type") != 0: | |
| 113 | + continue | |
| 114 | + for line in block.get("lines", []): | |
| 115 | + txt = "".join(s.get("text", "") for s in line.get("spans", [])).strip() | |
| 116 | + if not txt: | |
| 117 | + continue | |
| 118 | + sz = max((s.get("size", 0) for s in line.get("spans", [])), default=0) | |
| 119 | + bold = any((s.get("flags", 0) & 16) for s in line.get("spans", [])) | |
| 120 | + spans.append((txt, sz, bold)) | |
| 121 | + sizes.append(sz) | |
| 122 | + spans.append(("", 0.0, False)) # séparateur de bloc | |
| 123 | + spans_by_page.append(spans) | |
| 124 | + total_chars += sum(len(t) for t, _, _ in spans) | |
| 125 | + median = statistics.median(sizes) if sizes else 10.0 | |
| 126 | + is_scanned = total_chars / max(1, min(n_pages, max_pages)) < 50 | |
| 127 | + for pno in range(min(n_pages, max_pages)): | |
| 128 | + page = doc[pno] | |
| 129 | + spans = spans_by_page[pno] | |
| 130 | + if is_scanned or sum(len(t) for t, _, _ in spans) < 50: | |
| 131 | + ocr_txt = _ocr_page(page) | |
| 132 | + if ocr_txt: | |
| 133 | + ocr_pages += 1 | |
| 134 | + pages_md.append(ocr_txt) | |
| 135 | + continue | |
| 136 | + # tables | |
| 137 | + table_bboxes: list[Any] = [] | |
| 138 | + table_md: list[str] = [] | |
| 139 | + try: | |
| 140 | + tabs = page.find_tables() | |
| 141 | + for t in tabs.tables: | |
| 142 | + md = _table_to_md(t) | |
| 143 | + if md: | |
| 144 | + table_md.append(md) | |
| 145 | + table_bboxes.append(fitz.Rect(t.bbox)) | |
| 146 | + tables_count += 1 | |
| 147 | + except Exception: | |
| 148 | + pass | |
| 149 | + lines: list[str] = [] | |
| 150 | + prev_blank = True | |
| 151 | + for txt, sz, bold in spans: | |
| 152 | + if not txt: | |
| 153 | + if not prev_blank: | |
| 154 | + lines.append("") | |
| 155 | + prev_blank = True | |
| 156 | + continue | |
| 157 | + prev_blank = False | |
| 158 | + if sz > median * 1.5 and len(txt) < 120: | |
| 159 | + lines.append(f"# {txt}") | |
| 160 | + elif sz > median * 1.2 and len(txt) < 160: | |
| 161 | + lines.append(f"## {txt}") | |
| 162 | + elif bold and len(txt) < 100 and sz >= median: | |
| 163 | + lines.append(f"**{txt}**") | |
| 164 | + else: | |
| 165 | + lines.append(txt) | |
| 166 | + body = "\n".join(lines) | |
| 167 | + # rejoindre les lignes coupées d'un même paragraphe | |
| 168 | + body = _rejoin_lines(body) | |
| 169 | + if table_md: | |
| 170 | + body += "\n\n" + "\n\n".join(table_md) | |
| 171 | + pages_md.append(body.strip()) | |
| 172 | + md = "\n\n".join(p for p in pages_md if p) | |
| 173 | + if not md.strip(): | |
| 174 | + warnings.append( | |
| 175 | + "aucun texte extrait" | |
| 176 | + + ("" if ocr_pages else " (OCR indisponible : installer rapidocr-onnxruntime)") | |
| 177 | + ) | |
| 178 | + return PdfResult( | |
| 179 | + markdown=md.strip() + "\n", | |
| 180 | + pages=n_pages, | |
| 181 | + is_scanned=is_scanned, | |
| 182 | + title=(meta.get("title") or None), | |
| 183 | + author=(meta.get("author") or None), | |
| 184 | + created=(meta.get("creationDate") or None), | |
| 185 | + modified=(meta.get("modDate") or None), | |
| 186 | + tables=tables_count, | |
| 187 | + ocr_pages=ocr_pages, | |
| 188 | + warnings=warnings, | |
| 189 | + ) | |
| 190 | + | |
| 191 | + | |
| 192 | +def _rejoin_lines(text: str) -> str: | |
| 193 | + out: list[str] = [] | |
| 194 | + for para in text.split("\n\n"): | |
| 195 | + lines = para.split("\n") | |
| 196 | + buf = "" | |
| 197 | + for ln in lines: | |
| 198 | + if ln.startswith(("#", "**", "|", "- ", "• ")) or not buf: | |
| 199 | + if buf: | |
| 200 | + out.append(buf) | |
| 201 | + buf = ln | |
| 202 | + continue | |
| 203 | + if buf.endswith("-") and not buf.endswith(" -"): | |
| 204 | + buf = buf[:-1] + ln.lstrip() | |
| 205 | + elif buf.endswith((".", "!", "?", ":", ";")) and ln[:1].isupper(): | |
| 206 | + out.append(buf) | |
| 207 | + buf = ln | |
| 208 | + else: | |
| 209 | + buf = buf + " " + ln.strip() | |
| 210 | + if buf: | |
| 211 | + out.append(buf) | |
| 212 | + out.append("") | |
| 213 | + return "\n".join(out).strip() | |
added
trawls/processors/structured/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/processors/structured/metadata.py
+172 −0
@@ -0,0 +1,172 @@ | ||
| 1 | +"""Métadonnées : fusion title/description/lang/author/dates depuis <head>, OpenGraph, Twitter, JSON-LD, microdata.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +import json | |
| 6 | +import re | |
| 7 | +from typing import Any | |
| 8 | +from urllib.parse import urljoin | |
| 9 | + | |
| 10 | +from selectolax.parser import HTMLParser | |
| 11 | + | |
| 12 | +from trawls.models import PageMetadata | |
| 13 | + | |
| 14 | + | |
| 15 | +def _meta(tree: HTMLParser) -> dict[str, str]: | |
| 16 | + out: dict[str, str] = {} | |
| 17 | + for m in tree.css("meta"): | |
| 18 | + a = m.attributes | |
| 19 | + key = (a.get("property") or a.get("name") or a.get("itemprop") or "").strip().lower() | |
| 20 | + val = (a.get("content") or "").strip() | |
| 21 | + if key and val and key not in out: | |
| 22 | + out[key] = val | |
| 23 | + return out | |
| 24 | + | |
| 25 | + | |
| 26 | +def extract_jsonld(tree: HTMLParser) -> list[dict[str, Any]]: | |
| 27 | + out: list[dict[str, Any]] = [] | |
| 28 | + for s in tree.css('script[type="application/ld+json"]'): | |
| 29 | + raw = s.text() or "" | |
| 30 | + raw = raw.strip() | |
| 31 | + if not raw: | |
| 32 | + continue | |
| 33 | + try: | |
| 34 | + data = json.loads(raw) | |
| 35 | + except json.JSONDecodeError: | |
| 36 | + try: | |
| 37 | + data = json.loads(re.sub(r",\s*([}\]])", r"\1", raw)) | |
| 38 | + except json.JSONDecodeError: | |
| 39 | + continue | |
| 40 | + if isinstance(data, list): | |
| 41 | + out.extend(d for d in data if isinstance(d, dict)) | |
| 42 | + elif isinstance(data, dict): | |
| 43 | + if "@graph" in data and isinstance(data["@graph"], list): | |
| 44 | + out.extend(d for d in data["@graph"] if isinstance(d, dict)) | |
| 45 | + else: | |
| 46 | + out.append(data) | |
| 47 | + return out[:20] | |
| 48 | + | |
| 49 | + | |
| 50 | +def _first(*vals: str | None) -> str | None: | |
| 51 | + for v in vals: | |
| 52 | + if v and v.strip(): | |
| 53 | + return v.strip() | |
| 54 | + return None | |
| 55 | + | |
| 56 | + | |
| 57 | +def _jsonld_field(items: list[dict[str, Any]], *keys: str) -> str | None: | |
| 58 | + for it in items: | |
| 59 | + t = str(it.get("@type", "")).lower() | |
| 60 | + if any( | |
| 61 | + k in t | |
| 62 | + for k in ( | |
| 63 | + "article", | |
| 64 | + "newsarticle", | |
| 65 | + "blogposting", | |
| 66 | + "webpage", | |
| 67 | + "product", | |
| 68 | + "recipe", | |
| 69 | + "event", | |
| 70 | + "organization", | |
| 71 | + "person", | |
| 72 | + ) | |
| 73 | + ): | |
| 74 | + for k in keys: | |
| 75 | + v = it.get(k) | |
| 76 | + if isinstance(v, str) and v.strip(): | |
| 77 | + return v.strip() | |
| 78 | + if isinstance(v, dict): | |
| 79 | + n = v.get("name") | |
| 80 | + if isinstance(n, str) and n.strip(): | |
| 81 | + return n.strip() | |
| 82 | + if isinstance(v, list) and v: | |
| 83 | + x = v[0] | |
| 84 | + if isinstance(x, str): | |
| 85 | + return x | |
| 86 | + if isinstance(x, dict) and isinstance(x.get("name"), str): | |
| 87 | + return x["name"] | |
| 88 | + return None | |
| 89 | + | |
| 90 | + | |
| 91 | +def extract_metadata( | |
| 92 | + html: str, url: str, content_type: str | None = None, charset: str | None = None | |
| 93 | +) -> PageMetadata: | |
| 94 | + tree = HTMLParser(html or "") | |
| 95 | + meta = _meta(tree) | |
| 96 | + jsonld = extract_jsonld(tree) | |
| 97 | + title_el = tree.css_first("title") | |
| 98 | + h1 = tree.css_first("h1") | |
| 99 | + html_el = tree.css_first("html") | |
| 100 | + lang = ( | |
| 101 | + (html_el.attributes.get("lang") if html_el is not None else None) | |
| 102 | + or meta.get("og:locale") | |
| 103 | + or meta.get("content-language") | |
| 104 | + ) | |
| 105 | + canonical_el = tree.css_first('link[rel="canonical"]') | |
| 106 | + canonical = canonical_el.attributes.get("href") if canonical_el is not None else None | |
| 107 | + icon_el = ( | |
| 108 | + tree.css_first('link[rel~="icon"]') | |
| 109 | + or tree.css_first('link[rel="shortcut icon"]') | |
| 110 | + or tree.css_first('link[rel="apple-touch-icon"]') | |
| 111 | + ) | |
| 112 | + favicon = icon_el.attributes.get("href") if icon_el is not None else None | |
| 113 | + time_el = tree.css_first("time[datetime]") | |
| 114 | + keywords = [k.strip() for k in (meta.get("keywords") or "").split(",") if k.strip()][:30] | |
| 115 | + author_el = tree.css_first('[rel="author"], .author, .byline, [itemprop="author"]') | |
| 116 | + | |
| 117 | + md = PageMetadata( | |
| 118 | + title=_first( | |
| 119 | + meta.get("og:title"), | |
| 120 | + meta.get("twitter:title"), | |
| 121 | + title_el.text().strip() if title_el is not None else None, | |
| 122 | + _jsonld_field(jsonld, "headline", "name"), | |
| 123 | + h1.text().strip() if h1 is not None else None, | |
| 124 | + ), | |
| 125 | + description=_first( | |
| 126 | + meta.get("description"), | |
| 127 | + meta.get("og:description"), | |
| 128 | + meta.get("twitter:description"), | |
| 129 | + _jsonld_field(jsonld, "description"), | |
| 130 | + ), | |
| 131 | + language=(lang or "").split("_")[0].split("-")[0].lower() or None, | |
| 132 | + author=_first( | |
| 133 | + meta.get("author"), | |
| 134 | + meta.get("article:author"), | |
| 135 | + meta.get("twitter:creator"), | |
| 136 | + _jsonld_field(jsonld, "author"), | |
| 137 | + author_el.text().strip()[:120] if author_el is not None else None, | |
| 138 | + ), | |
| 139 | + published_at=_first( | |
| 140 | + meta.get("article:published_time"), | |
| 141 | + meta.get("datepublished"), | |
| 142 | + meta.get("date"), | |
| 143 | + meta.get("pubdate"), | |
| 144 | + _jsonld_field(jsonld, "datePublished"), | |
| 145 | + time_el.attributes.get("datetime") if time_el is not None else None, | |
| 146 | + ), | |
| 147 | + modified_at=_first( | |
| 148 | + meta.get("article:modified_time"), | |
| 149 | + meta.get("og:updated_time"), | |
| 150 | + _jsonld_field(jsonld, "dateModified"), | |
| 151 | + ), | |
| 152 | + canonical_url=urljoin(url, canonical) if canonical else None, | |
| 153 | + site_name=_first(meta.get("og:site_name"), meta.get("application-name")), | |
| 154 | + og_image=urljoin(url, meta["og:image"]) | |
| 155 | + if meta.get("og:image") | |
| 156 | + else (urljoin(url, meta["twitter:image"]) if meta.get("twitter:image") else None), | |
| 157 | + og_type=meta.get("og:type"), | |
| 158 | + keywords=keywords, | |
| 159 | + favicon=urljoin(url, favicon) if favicon else None, | |
| 160 | + content_type=content_type, | |
| 161 | + charset=charset, | |
| 162 | + jsonld=jsonld, | |
| 163 | + ) | |
| 164 | + extra = { | |
| 165 | + k: v | |
| 166 | + for k, v in meta.items() | |
| 167 | + if k.startswith(("og:", "twitter:", "article:")) | |
| 168 | + and k not in ("og:title", "og:description", "og:image", "og:type", "og:site_name") | |
| 169 | + } | |
| 170 | + if extra: | |
| 171 | + md.extra = dict(list(extra.items())[:40]) | |
| 172 | + return md | |
added
trawls/web/static/app.js
+322 −0
@@ -0,0 +1,322 @@ | ||
| 1 | +/* Trawls UI — vanilla JS, dense, dark. Routes : /playground /crawls /map /extract /keys /agent */ | |
| 2 | +(() => { | |
| 3 | + const $ = (s, r = document) => r.querySelector(s); | |
| 4 | + const $$ = (s, r = document) => [...r.querySelectorAll(s)]; | |
| 5 | + const view = $('#view'); | |
| 6 | + const esc = (s) => String(s ?? '').replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); | |
| 7 | + const fmtMs = (ms) => (ms == null ? '—' : ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(1)} s`); | |
| 8 | + const ago = (iso) => { if (!iso) return '—'; const d = (Date.now() - new Date(iso)) / 1000; return d < 60 ? `${d | 0}s` : d < 3600 ? `${(d / 60) | 0}m` : d < 86400 ? `${(d / 3600) | 0}h` : `${(d / 86400) | 0}j`; }; | |
| 9 | + const dur = (a, b) => (a ? fmtMs((new Date(b || Date.now()) - new Date(a))) : '—'); | |
| 10 | + | |
| 11 | + // ---- clé API (obfusquée localement) ------------------------------------------------ | |
| 12 | + const KEY = 'trawls.key'; | |
| 13 | + const enc = (s) => btoa(unescape(encodeURIComponent(s.split('').map((c, i) => String.fromCharCode(c.charCodeAt(0) ^ (i % 7 + 3))).join('')))); | |
| 14 | + const dec = (s) => { try { return decodeURIComponent(escape(atob(s))).split('').map((c, i) => String.fromCharCode(c.charCodeAt(0) ^ (i % 7 + 3))).join(''); } catch { return ''; } }; | |
| 15 | + const getKey = () => dec(localStorage.getItem(KEY) || ''); | |
| 16 | + const setKey = (k) => (k ? localStorage.setItem(KEY, enc(k)) : localStorage.removeItem(KEY)); | |
| 17 | + | |
| 18 | + const toast = (msg, err = false) => { const t = document.createElement('div'); t.className = 'toast' + (err ? ' err' : ''); t.textContent = msg; document.body.appendChild(t); setTimeout(() => t.remove(), 3200); }; | |
| 19 | + const copy = async (text, label = 'Copié') => { try { await navigator.clipboard.writeText(text); toast(label); } catch { toast('Copie impossible', true); } }; | |
| 20 | + | |
| 21 | + async function api(path, opts = {}) { | |
| 22 | + const headers = { 'Content-Type': 'application/json', ...(opts.headers || {}) }; | |
| 23 | + const k = getKey(); if (k) headers.Authorization = `Bearer ${k}`; | |
| 24 | + const r = await fetch(path, { ...opts, headers, body: opts.body && typeof opts.body !== 'string' ? JSON.stringify(opts.body) : opts.body }); | |
| 25 | + const ct = r.headers.get('content-type') || ''; | |
| 26 | + const data = ct.includes('json') ? await r.json() : await r.text(); | |
| 27 | + if (!r.ok) { const e = data && data.error ? data.error : { code: r.status, message: typeof data === 'string' ? data.slice(0, 200) : 'erreur' }; throw Object.assign(new Error(e.message), { code: e.code, details: e.details, status: r.status }); } | |
| 28 | + return data; | |
| 29 | + } | |
| 30 | + | |
| 31 | + const hl = (obj) => esc(JSON.stringify(obj, null, 2)).replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?)/g, (m) => { | |
| 32 | + let cls = 'json-n'; if (/^"/.test(m)) cls = /:$/.test(m) ? 'json-k' : 'json-s'; else if (/true|false/.test(m)) cls = 'json-b'; else if (/null/.test(m)) cls = 'json-null'; | |
| 33 | + return `<span class="${cls}">${m}</span>`; | |
| 34 | + }); | |
| 35 | + const md = (s) => (window.marked ? marked.parse(s || '', { mangle: false, headerIds: false }) : `<pre>${esc(s)}</pre>`); | |
| 36 | + | |
| 37 | + // ---- routeur ------------------------------------------------------------------------- | |
| 38 | + const routes = { playground: renderPlayground, crawls: renderCrawls, map: renderMap, extract: renderExtract, keys: renderKeys, agent: renderAgent }; | |
| 39 | + let cleanup = null; | |
| 40 | + function navigate(path, push = true) { | |
| 41 | + if (push) history.pushState({}, '', path); | |
| 42 | + const seg = path.split('/').filter(Boolean); | |
| 43 | + const name = routes[seg[0]] ? seg[0] : 'playground'; | |
| 44 | + $$('#nav a').forEach((a) => a.classList.toggle('on', a.dataset.route === name)); | |
| 45 | + if (cleanup) { try { cleanup(); } catch {} cleanup = null; } | |
| 46 | + view.innerHTML = ''; | |
| 47 | + view.appendChild($(`#tpl-${name}`).content.cloneNode(true)); | |
| 48 | + cleanup = routes[name](seg.slice(1)) || null; | |
| 49 | + } | |
| 50 | + document.addEventListener('click', (e) => { const a = e.target.closest('a[data-link]'); if (a) { e.preventDefault(); navigate(a.getAttribute('href')); } }); | |
| 51 | + window.addEventListener('popstate', () => navigate(location.pathname, false)); | |
| 52 | + | |
| 53 | + // ---- statut global ------------------------------------------------------------------------ | |
| 54 | + async function pollStatus() { | |
| 55 | + const el = $('#status'); | |
| 56 | + try { | |
| 57 | + const r = await fetch('/readyz'); const d = await r.json(); | |
| 58 | + el.className = 'status ' + (d.ok ? (d.checks.browser ? 'ok' : 'warn') : 'err'); | |
| 59 | + el.lastElementChild.textContent = d.ok ? (d.checks.browser ? 'ready' : 'ready · browser off') : 'degraded'; | |
| 60 | + const h = await (await fetch('/healthz')).json(); $('#ver').textContent = 'v' + h.version; $('#foot-info').textContent = `uptime ${fmtMs(h.uptime_s * 1000)}`; | |
| 61 | + } catch { el.className = 'status err'; el.lastElementChild.textContent = 'offline'; } | |
| 62 | + } | |
| 63 | + $('#btn-key').onclick = () => { $('#key-input').value = getKey(); $('#dlg-key').showModal(); }; | |
| 64 | + $('#key-save').onclick = () => { setKey($('#key-input').value.trim()); toast('Clé enregistrée'); }; | |
| 65 | + | |
| 66 | + // ---- snippets ----------------------------------------------------------------------------- | |
| 67 | + const base = location.origin; | |
| 68 | + const snippets = { | |
| 69 | + curl: (path, body) => `curl -s ${base}${path} \\\n -H 'Content-Type: application/json'${getKey() ? " \\\n -H 'Authorization: Bearer $TRAWLS_API_KEY'" : ''} \\\n -d '${JSON.stringify(body)}'`, | |
| 70 | + python: (path, body) => `import httpx\n\nr = httpx.post("${base}${path}", json=${JSON.stringify(body, null, 2).replace(/true/g, 'True').replace(/false/g, 'False').replace(/null/g, 'None')},${getKey() ? '\n headers={"Authorization": "Bearer " + TRAWLS_API_KEY},' : ''} timeout=120)\nprint(r.json()["markdown"])`, | |
| 71 | + ts: (path, body) => `const r = await fetch("${base}${path}", {\n method: "POST",\n headers: { "Content-Type": "application/json"${getKey() ? ', Authorization: `Bearer ${process.env.TRAWLS_API_KEY}`' : ''} },\n body: JSON.stringify(${JSON.stringify(body, null, 2)}),\n});\nconst page = await r.json();\nconsole.log(page.markdown);`, | |
| 72 | + }; | |
| 73 | + | |
| 74 | + // =================================================================================== | |
| 75 | + // PLAYGROUND | |
| 76 | + // =================================================================================== | |
| 77 | + function renderPlayground() { | |
| 78 | + const HIST = 'trawls.hist'; | |
| 79 | + let last = null, lastBody = null, tab = 'md'; | |
| 80 | + const hist = () => JSON.parse(localStorage.getItem(HIST) || '[]'); | |
| 81 | + const drawHist = () => { $('#pg-hist').innerHTML = hist().map((h, i) => `<li data-i="${i}"><span class="badge ${h.status}">${h.status}</span><span>${esc(h.url)}</span><span class="muted">${ago(h.at)}</span></li>`).join('') || '<li class="muted">—</li>'; }; | |
| 82 | + $('#pg-hist').onclick = (e) => { const li = e.target.closest('li[data-i]'); if (!li) return; const h = hist()[+li.dataset.i]; $('#pg-url').value = h.url; run(); }; | |
| 83 | + $('#pg-hist-clear').onclick = () => { localStorage.removeItem(HIST); drawHist(); }; | |
| 84 | + drawHist(); | |
| 85 | + | |
| 86 | + const buildBody = () => { | |
| 87 | + const formats = $$('#pg-formats input:checked').map((i) => i.value); | |
| 88 | + const wf = $('#pg-wait').value.trim(); | |
| 89 | + const body = { url: $('#pg-url').value.trim(), formats: formats.length ? formats : ['markdown'], mode: $('#pg-mode').value, only_main_content: $('#pg-main').checked, citations: $('#pg-cit').checked, remove_base64_images: $('#pg-b64').checked, cache: $('#pg-cache').checked ? 'use' : 'bypass', respect_robots: $('#pg-robots').checked, timeout_ms: +$('#pg-timeout').value || 30000 }; | |
| 90 | + if (wf) body.wait_for = /^\d+$/.test(wf) ? +wf : wf; | |
| 91 | + const ex = $('#pg-exclude').value.split(',').map((s) => s.trim()).filter(Boolean); if (ex.length) body.exclude_tags = ex; | |
| 92 | + const inc = $('#pg-include').value.split(',').map((s) => s.trim()).filter(Boolean); if (inc.length) body.include_tags = inc; | |
| 93 | + const acts = $('#pg-actions').value.trim(); if (acts) body.actions = JSON.parse(acts); | |
| 94 | + const exm = $('#pg-ex-mode').value; if (exm) { const eb = $('#pg-ex-body').value.trim(); body.extract = { mode: exm, prompt: $('#pg-ex-prompt').value || undefined }; if (exm === 'css') body.extract.css = eb ? JSON.parse(eb) : {}; else body.extract.schema = eb ? JSON.parse(eb) : {}; } | |
| 95 | + if (formats.includes('chunks')) body.chunk = { strategy: 'by_heading' }; | |
| 96 | + return body; | |
| 97 | + }; | |
| 98 | + | |
| 99 | + const draw = () => { | |
| 100 | + const r = $('#pg-result'); if (!last) return; | |
| 101 | + const p = last; | |
| 102 | + if (tab === 'md') r.innerHTML = p.markdown != null ? `<div class="prose">${md(p.markdown)}</div>` : '<div class="empty">markdown non demandé</div>'; | |
| 103 | + else if (tab === 'raw') r.innerHTML = `<pre class="code mono">${esc(p.markdown ?? '')}</pre>`; | |
| 104 | + else if (tab === 'json') r.innerHTML = `<pre class="code mono">${hl(p)}</pre>`; | |
| 105 | + else if (tab === 'html') r.innerHTML = `<pre class="code mono">${esc(p.html ?? p.raw_html ?? '(html non demandé)')}</pre>`; | |
| 106 | + else if (tab === 'links') r.innerHTML = p.links && p.links.length ? `<table class="tbl mono sm"><thead><tr><th>kind</th><th>text</th><th>href</th></tr></thead><tbody>${p.links.map((l) => `<tr><td><span class="badge">${l.kind}</span></td><td>${esc(l.text)}</td><td><a href="${esc(l.href)}" target="_blank" rel="noopener">${esc(l.href)}</a></td></tr>`).join('')}</tbody></table>` : '<div class="empty">aucun lien (format « links » non demandé ?)</div>'; | |
| 107 | + else if (tab === 'shot') r.innerHTML = p.screenshot_url ? `<img class="shot" src="${esc(p.screenshot_url)}" alt="screenshot">` : '<div class="empty">screenshot non demandé (mode navigateur requis)</div>'; | |
| 108 | + else if (tab === 'chunks') r.innerHTML = p.chunks && p.chunks.length ? p.chunks.map((c) => `<div class="chunk"><div class="h"><span>#${c.index}</span><span>${c.token_count} tok</span><span>${esc(c.heading_path || '—')}</span><span>[${c.char_range[0]}–${c.char_range[1]}]</span></div><div>${esc(c.text)}</div></div>`).join('') : '<div class="empty">chunks non demandés</div>'; | |
| 109 | + else if (tab === 'meta') r.innerHTML = `<pre class="code mono">${hl(p.metadata)}</pre>`; | |
| 110 | + }; | |
| 111 | + $('#pg-tabs').onclick = (e) => { | |
| 112 | + const b = e.target.closest('button[data-tab]'); if (b) { tab = b.dataset.tab; $$('#pg-tabs button[data-tab]').forEach((x) => x.classList.toggle('on', x === b)); draw(); return; } | |
| 113 | + const c = e.target.closest('button[data-copy]'); if (!c) return; | |
| 114 | + if (c.dataset.copy === 'out') { if (!last) return; copy(tab === 'md' || tab === 'raw' ? last.markdown || '' : JSON.stringify(last, null, 2), 'Sortie copiée'); return; } | |
| 115 | + const body = lastBody || buildBody(); copy(snippets[c.dataset.copy]('/v1/scrape', body), `Snippet ${c.dataset.copy} copié`); | |
| 116 | + }; | |
| 117 | + | |
| 118 | + async function run() { | |
| 119 | + let body; try { body = buildBody(); } catch (e) { toast('JSON invalide : ' + e.message, true); return; } | |
| 120 | + if (!body.url) return; | |
| 121 | + if (!/^https?:\/\//i.test(body.url)) { body.url = 'https://' + body.url; $('#pg-url').value = body.url; } | |
| 122 | + lastBody = body; | |
| 123 | + const btn = $('#pg-run'); btn.disabled = true; $('#pg-bar').innerHTML = `<span>⏳ ${esc(body.url)}</span>`; | |
| 124 | + const t0 = performance.now(); | |
| 125 | + try { | |
| 126 | + const p = await api('/v1/scrape', { method: 'POST', body }); | |
| 127 | + last = p; | |
| 128 | + const t = p.timings || {}; | |
| 129 | + $('#pg-bar').innerHTML = `<span class="${p.status === 'ok' ? 'ok' : 'err'}">● ${p.status}</span><span>HTTP <b>${p.http_status ?? '—'}</b></span><span>mode <b>${p.fetch_mode_used ?? '—'}</b></span><span><b>${p.metadata.word_count}</b> mots</span><span>fetch ${fmtMs(t.fetch_ms)}</span><span>process ${fmtMs(t.process_ms)}</span><span>total ${fmtMs(t.total_ms)}</span>${p.from_cache ? '<span class="warn">cache</span>' : ''}${p.error ? `<span class="err">${esc(p.error.code)}: ${esc(p.error.message)}</span>` : ''}${p.metadata.extra && p.metadata.extra.fetch_trace ? `<span class="muted">${esc(p.metadata.extra.fetch_trace.join(' → '))}</span>` : ''}`; | |
| 130 | + if (p.status !== 'ok' && tab === 'md') tab = 'json'; | |
| 131 | + if (p.status === 'ok' && body.extract && tab === 'md') tab = 'json'; | |
| 132 | + $$('#pg-tabs button[data-tab]').forEach((x) => x.classList.toggle('on', x.dataset.tab === tab)); | |
| 133 | + draw(); | |
| 134 | + const h = hist().filter((x) => x.url !== body.url); h.unshift({ url: body.url, status: p.status, at: new Date().toISOString() }); localStorage.setItem(HIST, JSON.stringify(h.slice(0, 20))); drawHist(); | |
| 135 | + } catch (e) { | |
| 136 | + $('#pg-bar').innerHTML = `<span class="err">✕ ${esc(e.code || 'ERREUR')}: ${esc(e.message)}</span>`; $('#pg-result').innerHTML = `<pre class="code mono">${hl({ error: { code: e.code, message: e.message, details: e.details } })}</pre>`; | |
| 137 | + } finally { btn.disabled = false; void (performance.now() - t0); } | |
| 138 | + } | |
| 139 | + $('#pg-form').onsubmit = (e) => { e.preventDefault(); run(); }; | |
| 140 | + const kd = (e) => { if ((e.metaKey || e.ctrlKey) && e.key === 'Enter') { e.preventDefault(); run(); } }; | |
| 141 | + document.addEventListener('keydown', kd); | |
| 142 | + const q = new URLSearchParams(location.search).get('url'); if (q) { $('#pg-url').value = q; run(); } | |
| 143 | + return () => document.removeEventListener('keydown', kd); | |
| 144 | + } | |
| 145 | + | |
| 146 | + // =================================================================================== | |
| 147 | + // CRAWLS | |
| 148 | + // =================================================================================== | |
| 149 | + function renderCrawls(seg) { | |
| 150 | + let es = null, current = null, timer = null; | |
| 151 | + const list = async () => { | |
| 152 | + try { | |
| 153 | + const jobs = await api('/v1/jobs?limit=100'); | |
| 154 | + $('#cr-jobs tbody').innerHTML = jobs.map((j) => `<tr data-id="${j.id}" class="${current === j.id ? 'sel' : ''}"><td class="mono">${j.id.slice(4, 14)}</td><td>${j.kind}</td><td><span class="badge ${j.status}">${j.status}</span></td><td class="mono">${j.completed}<span class="muted">/${j.total || '?'}</span>${j.failed ? ` <span class="badge failed">${j.failed}</span>` : ''}</td><td class="mono">${dur(j.started_at, j.finished_at)}</td><td><button class="link-btn" data-del="${j.id}" title="annuler">✕</button></td></tr>`).join('') || '<tr><td colspan="6" class="muted">aucun job</td></tr>'; | |
| 155 | + } catch (e) { toast(e.message, true); } | |
| 156 | + }; | |
| 157 | + $('#cr-jobs').onclick = (e) => { const d = e.target.closest('[data-del]'); if (d) { api(`/v1/jobs/${d.dataset.del}`, { method: 'DELETE' }).then(list); return; } const tr = e.target.closest('tr[data-id]'); if (tr) open(tr.dataset.id); }; | |
| 158 | + $('#cr-refresh').onclick = list; | |
| 159 | + $('#cr-form').onsubmit = async (e) => { | |
| 160 | + e.preventDefault(); | |
| 161 | + let body; | |
| 162 | + const raw = $('#cr-json').value.trim(); | |
| 163 | + if (raw) { try { body = JSON.parse(raw); } catch (er) { toast('JSON invalide', true); return; } } else { | |
| 164 | + const split = (s) => s.split(',').map((x) => x.trim()).filter(Boolean); | |
| 165 | + let url = $('#cr-url').value.trim(); if (!/^https?:\/\//i.test(url)) url = 'https://' + url; | |
| 166 | + body = { url, crawl: { max_pages: +$('#cr-pages').value, max_depth: +$('#cr-depth').value, concurrency: +$('#cr-conc').value, strategy: $('#cr-strat').value, include_paths: split($('#cr-inc').value), exclude_paths: split($('#cr-exc').value), search: $('#cr-search').value || null, delay_ms: +$('#cr-delay').value, allow_subdomains: $('#cr-sub').checked, allow_external_links: $('#cr-ext').checked, ignore_sitemap: $('#cr-nosm').checked, respect_robots: $('#cr-robots').checked }, scrape: { formats: $('#cr-links').checked ? ['markdown', 'links'] : ['markdown'] } }; | |
| 167 | + } | |
| 168 | + try { const r = await api('/v1/crawl', { method: 'POST', body }); toast('Job ' + r.job_id); await list(); open(r.job_id); } catch (er) { toast(er.message, true); } | |
| 169 | + }; | |
| 170 | + | |
| 171 | + function tree(pages) { | |
| 172 | + const root = {}; | |
| 173 | + for (const p of pages) { try { const u = new URL(p.final_url || p.url); const parts = [u.host, ...u.pathname.split('/').filter(Boolean)]; let n = root; for (const part of parts) n = n[part] = n[part] || {}; n.__page = p; } catch {} } | |
| 174 | + const render = (n, name) => { const kids = Object.keys(n).filter((k) => k !== '__page'); const page = n.__page; const leaf = page ? `<div class="leaf" data-url="${esc(page.url)}"><span class="badge ${page.status}">${page.status}</span><span>${esc(name)}</span><span class="muted">${page.title ? esc(page.title.slice(0, 60)) : ''}</span></div>` : ''; if (!kids.length) return leaf; return `<details open><summary>${esc(name)}/ <span class="muted">${kids.length}</span></summary>${leaf}${kids.sort().map((k) => render(n[k], k)).join('')}</details>`; }; | |
| 175 | + return `<div class="tree">${Object.keys(root).map((k) => render(root[k], k)).join('')}</div>`; | |
| 176 | + } | |
| 177 | + | |
| 178 | + async function open(id) { | |
| 179 | + current = id; $$('#cr-jobs tr').forEach((tr) => tr.classList.toggle('sel', tr.dataset.id === id)); | |
| 180 | + history.replaceState({}, '', `/crawls/${id}`); | |
| 181 | + if (es) { es.close(); es = null; } | |
| 182 | + const d = $('#cr-detail'); | |
| 183 | + d.innerHTML = `<div class="result-bar mono" id="cd-bar">chargement…</div><div class="progress"><i id="cd-prog" style="width:0%"></i></div> | |
| 184 | + <div class="row gap wrap" style="padding:6px 0"> | |
| 185 | + <select class="input" id="cd-filter" style="width:auto"><option value="">tous</option><option value="ok">ok</option><option value="failed">failed</option><option value="skipped">skipped</option></select> | |
| 186 | + <a class="btn ghost sm" href="/v1/jobs/${id}/export?format=zip" target="_blank">ZIP (md)</a><a class="btn ghost sm" href="/v1/jobs/${id}/export?format=jsonl" target="_blank">JSONL</a><a class="btn ghost sm" href="/v1/jobs/${id}/export?format=md" target="_blank">MD</a> | |
| 187 | + <button class="btn danger sm" id="cd-cancel">Annuler</button><span class="grow"></span><span class="muted mono" id="cd-count"></span></div> | |
| 188 | + <div class="split"><div id="cd-tree" class="tree"></div><div id="cd-page"><div class="log" id="cd-log"></div></div></div>`; | |
| 189 | + const pages = []; const events = []; | |
| 190 | + const drawBar = (j) => { $('#cd-bar').innerHTML = `<span class="badge ${j.status}">${j.status}</span><span>${esc(j.root_url || '')}</span><span><b>${j.completed}</b> ok</span><span class="${j.failed ? 'err' : ''}">${j.failed} failed</span><span>${j.skipped} skipped</span><span>total ${j.total}</span><span>${dur(j.started_at, j.finished_at)}</span>${j.meta && j.meta.stopped_reason ? `<span class="warn">${esc(j.meta.stopped_reason)}</span>` : ''}${j.error ? `<span class="err">${esc(j.error.message)}</span>` : ''}`; $('#cd-prog').style.width = (j.total ? Math.min(100, (100 * (j.completed + j.failed + j.skipped)) / j.total) : 0) + '%'; }; | |
| 191 | + const drawTree = () => { const f = $('#cd-filter').value; const rows = f ? pages.filter((p) => p.status === f) : pages; $('#cd-tree').innerHTML = tree(rows); $('#cd-count').textContent = `${rows.length} pages`; }; | |
| 192 | + $('#cd-filter').onchange = drawTree; | |
| 193 | + $('#cd-cancel').onclick = () => api(`/v1/jobs/${id}`, { method: 'DELETE' }).then(list); | |
| 194 | + $('#cd-tree').onclick = async (e) => { const l = e.target.closest('.leaf'); if (!l) return; const p = pages.find((x) => x.url === l.dataset.url); if (!p) return; let full = p; if (!p.markdown && p.status === 'ok') { const r = await api(`/v1/jobs/${id}?limit=500`); full = r.pages.find((x) => x.url === p.url) || p; } $('#cd-page').innerHTML = `<div class="result-bar mono"><span class="badge ${full.status}">${full.status}</span><span>${esc(full.final_url || full.url)}</span><span>${full.fetch_mode_used || ''}</span><span>${fmtMs(full.timings && full.timings.total_ms)}</span></div>${full.status === 'ok' ? `<div class="prose">${md(full.markdown)}</div>` : `<pre class="code mono">${hl(full.error)}</pre>`}`; }; | |
| 195 | + let cursor = 0; | |
| 196 | + try { | |
| 197 | + let j; | |
| 198 | + do { j = await api(`/v1/jobs/${id}?cursor=${cursor}&limit=500`); pages.push(...j.pages.map((p) => ({ ...p, title: p.metadata && p.metadata.title }))); cursor = j.next_cursor; } while (cursor); | |
| 199 | + drawBar(j); drawTree(); | |
| 200 | + if (['completed', 'failed', 'cancelled'].includes(j.status)) return; | |
| 201 | + } catch (e) { $('#cd-bar').innerHTML = `<span class="err">${esc(e.message)}</span>`; return; } | |
| 202 | + const k = getKey(); | |
| 203 | + es = new EventSource(`/v1/jobs/${id}/stream${k ? `?api_key=${encodeURIComponent(k)}` : ''}`); | |
| 204 | + const log = $('#cd-log'); | |
| 205 | + es.addEventListener('page', (ev) => { const p = JSON.parse(ev.data); pages.push({ ...p, metadata: { title: p.title } }); events.push(p); log.insertAdjacentHTML('afterbegin', `<div><span class="${p.status}">${p.status.padEnd(7)}</span> d${p.depth} ${(p.mode || '-').padEnd(7)} ${fmtMs(p.ms).padStart(7)} ${esc(p.url)}${p.error ? ` <span class="failed">${esc(p.error.code)}</span>` : ''}</div>`); if (events.length % 5 === 0) drawTree(); }); | |
| 206 | + es.addEventListener('progress', (ev) => { const p = JSON.parse(ev.data); $('#cd-prog').style.width = (p.total ? Math.min(100, (100 * ((p.completed || 0) + (p.failed || 0) + (p.skipped || 0))) / p.total) : 0) + '%'; $('#cd-count').textContent = `${pages.length} pages · ${p.queued} en file`; }); | |
| 207 | + es.addEventListener('status', (ev) => drawBar(JSON.parse(ev.data))); | |
| 208 | + es.addEventListener('done', async () => { es.close(); es = null; const j = await api(`/v1/jobs/${id}`); drawBar(j); drawTree(); list(); }); | |
| 209 | + es.addEventListener('error', () => { if (es && es.readyState === EventSource.CLOSED) { es = null; } }); | |
| 210 | + } | |
| 211 | + list(); timer = setInterval(list, 8000); | |
| 212 | + if (seg && seg[0]) open(seg[0]); | |
| 213 | + return () => { if (es) es.close(); clearInterval(timer); }; | |
| 214 | + } | |
| 215 | + | |
| 216 | + // =================================================================================== | |
| 217 | + // MAP | |
| 218 | + // =================================================================================== | |
| 219 | + function renderMap() { | |
| 220 | + let all = []; let sel = new Set(); | |
| 221 | + const listEl = $('#mp-list'); | |
| 222 | + const rows = () => { const f = $('#mp-filter').value.toLowerCase(); return f ? all.filter((u) => u.url.toLowerCase().includes(f) || (u.title || '').toLowerCase().includes(f)) : all; }; | |
| 223 | + const draw = () => { | |
| 224 | + const r = rows(); const group = $('#mp-group').checked; | |
| 225 | + if (!r.length) { listEl.innerHTML = '<div class="empty">—</div>'; return; } | |
| 226 | + // liste « virtualisée » simple : rendu par tranches | |
| 227 | + listEl.innerHTML = ''; | |
| 228 | + const frag = document.createDocumentFragment(); | |
| 229 | + let lastG = null; | |
| 230 | + const chunk = (start) => { | |
| 231 | + const end = Math.min(r.length, start + 400); | |
| 232 | + for (let i = start; i < end; i++) { | |
| 233 | + const u = r[i]; | |
| 234 | + if (group) { let g = '/'; try { g = '/' + (new URL(u.url).pathname.split('/').filter(Boolean)[0] || ''); } catch {} if (g !== lastG) { lastG = g; const h = document.createElement('div'); h.className = 'vgroup'; h.textContent = g; frag.appendChild(h); } } | |
| 235 | + const d = document.createElement('div'); d.className = 'vrow'; | |
| 236 | + d.innerHTML = `<input type="checkbox" data-u="${esc(u.url)}" ${sel.has(u.url) ? 'checked' : ''}><span title="${esc(u.title || '')}"><a href="${esc(u.url)}" target="_blank" rel="noopener">${esc(u.url)}</a>${u.title ? ` <span class="muted">— ${esc(u.title)}</span>` : ''}</span><span class="src">${(u.sources || []).join(',')}</span><span class="src">${u.score != null ? u.score.toFixed(2) : u.depth != null ? 'd' + u.depth : ''}</span>`; | |
| 237 | + frag.appendChild(d); | |
| 238 | + } | |
| 239 | + listEl.appendChild(frag); | |
| 240 | + if (end < r.length) requestAnimationFrame(() => chunk(end)); | |
| 241 | + }; | |
| 242 | + chunk(0); | |
| 243 | + }; | |
| 244 | + listEl.onchange = (e) => { const c = e.target.closest('input[data-u]'); if (!c) return; c.checked ? sel.add(c.dataset.u) : sel.delete(c.dataset.u); }; | |
| 245 | + $('#mp-filter').oninput = draw; $('#mp-group').onchange = draw; | |
| 246 | + $('#mp-form').onsubmit = async (e) => { | |
| 247 | + e.preventDefault(); let url = $('#mp-url').value.trim(); if (!/^https?:\/\//i.test(url)) url = 'https://' + url; | |
| 248 | + $('#mp-bar').textContent = '⏳ cartographie…'; sel = new Set(); | |
| 249 | + try { | |
| 250 | + const r = await api('/v1/map', { method: 'POST', body: { url, search: $('#mp-search').value || null, limit: +$('#mp-limit').value, include_subdomains: $('#mp-sub').checked, include_titles: $('#mp-titles').checked } }); | |
| 251 | + all = r.urls; draw(); | |
| 252 | + const bySrc = {}; all.forEach((u) => (u.sources || []).forEach((s) => (bySrc[s] = (bySrc[s] || 0) + 1))); | |
| 253 | + $('#mp-bar').innerHTML = `<span class="ok">● ${r.count} URLs</span><span>${fmtMs(r.took_ms)}</span>${Object.entries(bySrc).map(([k, v]) => `<span>${k}: <b>${v}</b></span>`).join('')}`; | |
| 254 | + } catch (er) { $('#mp-bar').innerHTML = `<span class="err">✕ ${esc(er.message)}</span>`; } | |
| 255 | + }; | |
| 256 | + const dl = (name, text, type) => { const a = document.createElement('a'); a.href = URL.createObjectURL(new Blob([text], { type })); a.download = name; a.click(); }; | |
| 257 | + $('#mp-csv').onclick = () => dl('map.csv', 'url,sources,lastmod,depth,title\n' + rows().map((u) => [u.url, (u.sources || []).join('|'), u.lastmod || '', u.depth ?? '', (u.title || '').replace(/"/g, '""')].map((x) => `"${x}"`).join(',')).join('\n'), 'text/csv'); | |
| 258 | + $('#mp-json').onclick = () => dl('map.json', JSON.stringify(rows(), null, 2), 'application/json'); | |
| 259 | + $('#mp-copy').onclick = () => copy(rows().map((u) => u.url).join('\n'), 'URLs copiées'); | |
| 260 | + $('#mp-crawl').onclick = async () => { | |
| 261 | + const urls = sel.size ? [...sel] : rows().map((u) => u.url); if (!urls.length) return; | |
| 262 | + try { const r = await api('/v1/batch/scrape', { method: 'POST', body: { urls, formats: ['markdown'] } }); toast(`Batch ${r.job_id}`); navigate(`/crawls/${r.job_id}`); } catch (e) { toast(e.message, true); } | |
| 263 | + }; | |
| 264 | + } | |
| 265 | + | |
| 266 | + // =================================================================================== | |
| 267 | + // EXTRACT | |
| 268 | + // =================================================================================== | |
| 269 | + function renderExtract() { | |
| 270 | + const fields = $('#ex-fields'); const json = $('#ex-json'); | |
| 271 | + const mode = () => $('input[name=ex-mode]:checked').value; | |
| 272 | + let rows = [{ name: 'title', selector: 'h1', type: 'str', attr: 'text', desc: '' }, { name: 'price', selector: '.price', type: 'float', attr: 'text', desc: '' }]; | |
| 273 | + const toJson = () => { const m = mode(); if (m === 'css') { const o = {}; rows.forEach((r) => { if (r.name) o[r.name] = { selector: r.selector, attr: r.attr || 'text', type: r.type || 'str', multiple: !!r.multiple }; }); return o; } const props = {}; rows.forEach((r) => { if (r.name) props[r.name] = { type: r.type === 'str' ? 'string' : r.type === 'int' ? 'integer' : r.type === 'float' ? 'number' : r.type === 'bool' ? 'boolean' : r.type === 'list' ? 'array' : 'string', ...(r.desc ? { description: r.desc } : {}), ...(r.type === 'list' ? { items: { type: 'string' } } : {}) }; }); return { type: 'object', properties: props, required: rows.filter((r) => r.required && r.name).map((r) => r.name) }; }; | |
| 274 | + const drawFields = () => { | |
| 275 | + const m = mode(); | |
| 276 | + fields.innerHTML = rows.map((r, i) => m === 'css' | |
| 277 | + ? `<div class="ex-field"><input class="input mono" data-i="${i}" data-k="name" value="${esc(r.name)}" placeholder="champ"><input class="input mono" data-i="${i}" data-k="selector" value="${esc(r.selector || '')}" placeholder="sélecteur CSS"><select class="input" data-i="${i}" data-k="type">${['str', 'int', 'float', 'date', 'url', 'list', 'bool'].map((t) => `<option ${r.type === t ? 'selected' : ''}>${t}</option>`).join('')}</select><button class="link-btn" data-del="${i}">✕</button><div class="row gap" style="grid-column:1/-1"><input class="input mono" data-i="${i}" data-k="attr" value="${esc(r.attr || 'text')}" placeholder="attr (text|href|src…)" style="width:10rem"><label class="check"><input type="checkbox" data-i="${i}" data-k="multiple" ${r.multiple ? 'checked' : ''}> multiple</label></div></div>` | |
| 278 | + : `<div class="ex-field llm"><input class="input mono" data-i="${i}" data-k="name" value="${esc(r.name)}" placeholder="champ"><select class="input" data-i="${i}" data-k="type">${['str', 'int', 'float', 'bool', 'list'].map((t) => `<option ${r.type === t ? 'selected' : ''}>${t}</option>`).join('')}</select><input class="input" data-i="${i}" data-k="desc" value="${esc(r.desc || '')}" placeholder="description"><button class="link-btn" data-del="${i}">✕</button><label class="check" style="grid-column:1/-1"><input type="checkbox" data-i="${i}" data-k="required" ${r.required ? 'checked' : ''}> requis</label></div>`).join(''); | |
| 279 | + json.value = JSON.stringify(toJson(), null, 2); | |
| 280 | + }; | |
| 281 | + fields.oninput = fields.onchange = (e) => { const t = e.target; if (t.dataset.i == null) return; rows[+t.dataset.i][t.dataset.k] = t.type === 'checkbox' ? t.checked : t.value; json.value = JSON.stringify(toJson(), null, 2); }; | |
| 282 | + fields.onclick = (e) => { const d = e.target.closest('[data-del]'); if (d) { rows.splice(+d.dataset.del, 1); drawFields(); } }; | |
| 283 | + $('#ex-add').onclick = () => { rows.push({ name: '', selector: '', type: 'str', attr: 'text' }); drawFields(); }; | |
| 284 | + $$('input[name=ex-mode]').forEach((r) => (r.onchange = drawFields)); | |
| 285 | + json.onchange = () => { try { const o = JSON.parse(json.value); if (mode() === 'css') rows = Object.entries(o).map(([k, v]) => ({ name: k, ...v })); else rows = Object.entries(o.properties || {}).map(([k, v]) => ({ name: k, type: v.type === 'integer' ? 'int' : v.type === 'number' ? 'float' : v.type === 'boolean' ? 'bool' : v.type === 'array' ? 'list' : 'str', desc: v.description || '', required: (o.required || []).includes(k) })); drawFields(); } catch { toast('JSON invalide', true); } }; | |
| 286 | + drawFields(); | |
| 287 | + $('#ex-form').onsubmit = async (e) => { | |
| 288 | + e.preventDefault(); | |
| 289 | + const lines = $('#ex-urls').value.split('\n').map((s) => s.trim()).filter(Boolean); | |
| 290 | + const body = { mode: mode(), prompt: $('#ex-prompt').value || null, merge_key: $('#ex-merge').value || null }; | |
| 291 | + const pat = lines.find((l) => l.includes('*')); if (pat) body.pattern = pat; body.urls = lines.filter((l) => !l.includes('*')); | |
| 292 | + try { const schema = JSON.parse(json.value); if (mode() === 'css') body.css = schema; else body.schema = schema; } catch { toast('JSON invalide', true); return; } | |
| 293 | + $('#ex-bar').textContent = '⏳ extraction…'; $('#ex-result').innerHTML = ''; | |
| 294 | + try { | |
| 295 | + const r = await api('/v1/extract', { method: 'POST', body }); | |
| 296 | + if (r.status === 'queued') { $('#ex-bar').innerHTML = `<span class="warn">job ${r.job_id} en file (> 5 URLs)</span>`; navigate(`/crawls/${r.job_id}`); return; } | |
| 297 | + const okN = (r.per_url || []).filter((p) => p.status === 'ok').length; | |
| 298 | + $('#ex-bar').innerHTML = `<span class="ok">● ${okN}/${(r.per_url || []).length} pages</span><span>job ${r.job_id}</span>`; | |
| 299 | + $('#ex-result').innerHTML = `<h4>Fusion</h4><pre class="code mono">${hl(r.data)}</pre><h4>Par URL</h4>${(r.per_url || []).map((p) => `<div class="chunk"><div class="h"><span class="badge ${p.status}">${p.status}</span><span>${esc(p.url)}</span></div><pre class="code mono">${hl(p.data)}</pre>${p.errors && Object.keys(p.errors).length ? `<div class="muted mono">erreurs : ${esc(JSON.stringify(p.errors))}</div>` : ''}${p.error ? `<div class="err mono">${esc(p.error.code)}: ${esc(p.error.message)}</div>` : ''}</div>`).join('')}`; | |
| 300 | + } catch (er) { $('#ex-bar').innerHTML = `<span class="err">✕ ${esc(er.message)}</span>`; } | |
| 301 | + }; | |
| 302 | + } | |
| 303 | + | |
| 304 | + // =================================================================================== | |
| 305 | + // KEYS | |
| 306 | + // =================================================================================== | |
| 307 | + function renderKeys() { | |
| 308 | + const load = async () => { | |
| 309 | + try { const s = await api('/v1/system'); $('#ky-system').innerHTML = hl(s); } catch (e) { $('#ky-system').textContent = e.message; } | |
| 310 | + try { const u = await api('/v1/usage?days=30'); const max = Math.max(1, ...u.days.map((d) => d.credits)); $('#ky-usage').innerHTML = u.days.length ? u.days.map((d) => `<i style="height:${(100 * d.credits) / max}%" title="${d.day}: ${d.requests} req · ${d.credits} crédits"></i>`).join('') : '<span class="muted">aucun usage</span>'; } catch {} | |
| 311 | + try { const ks = await api('/v1/keys'); $('#ky-table tbody').innerHTML = ks.map((k) => `<tr><td class="mono">${k.id}</td><td>${esc(k.name)}</td><td class="mono">${k.prefix}…</td><td>${k.quota_per_day ?? '∞'}</td><td>${ago(k.last_used)}</td><td>${k.disabled ? '<span class="badge cancelled">révoquée</span>' : `<button class="link-btn" data-rev="${k.id}">révoquer</button>`}</td></tr>`).join('') || '<tr><td colspan="6" class="muted">aucune clé</td></tr>'; } catch (e) { $('#ky-table tbody').innerHTML = `<tr><td colspan="6" class="muted">${esc(e.message)} — définir TRAWLS_ADMIN_KEY et l'entrer via 🔑</td></tr>`; } | |
| 312 | + try { const rq = await api('/v1/requests?limit=60'); $('#ky-req tbody').innerHTML = rq.map((r) => `<tr><td>${ago(r.at)}</td><td>${r.method}</td><td>${esc(r.path)}</td><td><span class="badge ${r.status < 400 ? 'ok' : 'failed'}">${r.status}</span></td><td>${r.ms}</td></tr>`).join(''); } catch {} | |
| 313 | + }; | |
| 314 | + $('#ky-table').onclick = (e) => { const b = e.target.closest('[data-rev]'); if (b && confirm('Révoquer cette clé ?')) api(`/v1/keys/${b.dataset.rev}`, { method: 'DELETE' }).then(load); }; | |
| 315 | + $('#ky-form').onsubmit = async (e) => { e.preventDefault(); try { const r = await api('/v1/keys', { method: 'POST', body: { name: $('#ky-name').value, quota_per_day: $('#ky-quota').value ? +$('#ky-quota').value : null } }); const el = $('#ky-new'); el.classList.remove('hidden'); el.innerHTML = `Clé créée (affichée une seule fois) : <b>${esc(r.key)}</b> <button class="btn ghost sm" id="ky-copy">copier</button>`; $('#ky-copy').onclick = () => copy(r.key); load(); } catch (er) { toast(er.message, true); } }; | |
| 316 | + load(); | |
| 317 | + } | |
| 318 | + function renderAgent() {} | |
| 319 | + | |
| 320 | + pollStatus(); setInterval(pollStatus, 15000); | |
| 321 | + navigate(location.pathname === '/' ? '/playground' : location.pathname, false); | |
| 322 | +})(); | |
added
trawls/web/static/favicon.svg
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="7" fill="#0b0c0e"/><path d="M4 10c6 0 8 6 12 6s6-6 12-6" fill="none" stroke="#f5a524" stroke-width="2.6" stroke-linecap="round"/><path d="M4 17c6 0 8 6 12 6s6-6 12-6" fill="none" stroke="#f5a524" stroke-width="2.6" stroke-linecap="round" opacity=".65"/><path d="M4 24c6 0 8 6 12 6s6-6 12-6" fill="none" stroke="#f5a524" stroke-width="2.6" stroke-linecap="round" opacity=".35"/></svg> | |
added
trawls/web/static/index.html
+245 −0
@@ -0,0 +1,245 @@ | ||
| 1 | +<!doctype html> | |
| 2 | +<html lang="fr" data-theme="dark"> | |
| 3 | +<head> | |
| 4 | +<meta charset="utf-8"> | |
| 5 | +<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"> | |
| 6 | +<title>Trawls — crawl, extract, navigate</title> | |
| 7 | +<meta name="description" content="Trawls : moteur de crawling, d'extraction et de navigation web, sortie Markdown LLM-ready. Self-host first."> | |
| 8 | +<meta property="og:title" content="Trawls — crawl, extract, navigate"> | |
| 9 | +<meta property="og:description" content="Un chalut ratisse le fond et remonte tout : on cartographie et on extrait n'importe quel site. Markdown propre, escalade http → navigateur → stealth, API OpenAPI."> | |
| 10 | +<meta property="og:image" content="https://www.trawls.dev/static/og.svg"> | |
| 11 | +<meta property="og:url" content="https://www.trawls.dev/"> | |
| 12 | +<link rel="icon" href="/static/favicon.svg" type="image/svg+xml"> | |
| 13 | +<link rel="preconnect" href="https://fonts.googleapis.com"> | |
| 14 | +<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet"> | |
| 15 | +<link rel="stylesheet" href="/static/style.css"> | |
| 16 | +</head> | |
| 17 | +<body> | |
| 18 | +<header class="top"> | |
| 19 | + <a class="brand" href="/playground" data-link> | |
| 20 | + <svg width="22" height="22" viewBox="0 0 32 32" aria-hidden="true"><path d="M3 9c6 0 8 6 13 6s7-6 13-6" fill="none" stroke="var(--accent)" stroke-width="2.4" stroke-linecap="round"/><path d="M3 16c6 0 8 6 13 6s7-6 13-6" fill="none" stroke="var(--accent)" stroke-width="2.4" stroke-linecap="round" opacity=".65"/><path d="M3 23c6 0 8 6 13 6s7-6 13-6" fill="none" stroke="var(--accent)" stroke-width="2.4" stroke-linecap="round" opacity=".35"/></svg> | |
| 21 | + <span>Trawls</span><span class="ver" id="ver"></span> | |
| 22 | + </a> | |
| 23 | + <nav class="nav" id="nav"> | |
| 24 | + <a href="/playground" data-link data-route="playground">Playground</a> | |
| 25 | + <a href="/crawls" data-link data-route="crawls">Crawls</a> | |
| 26 | + <a href="/map" data-link data-route="map">Map</a> | |
| 27 | + <a href="/extract" data-link data-route="extract">Extract</a> | |
| 28 | + <a href="/keys" data-link data-route="keys">Keys</a> | |
| 29 | + <a href="/docs" target="_blank" rel="noopener">API docs ↗</a> | |
| 30 | + </nav> | |
| 31 | + <div class="top-right"> | |
| 32 | + <span class="status" id="status"><i></i><span>…</span></span> | |
| 33 | + <button class="btn ghost sm" id="btn-key" title="Clé API">🔑</button> | |
| 34 | + </div> | |
| 35 | +</header> | |
| 36 | + | |
| 37 | +<main id="view" class="view"></main> | |
| 38 | + | |
| 39 | +<footer class="foot"> | |
| 40 | + <span>Trawls · www.trawls.dev · self-host first · <a href="/docs" target="_blank" rel="noopener">OpenAPI</a> · <a href="/metrics" target="_blank" rel="noopener">metrics</a></span> | |
| 41 | + <span class="mono" id="foot-info"></span> | |
| 42 | +</footer> | |
| 43 | + | |
| 44 | +<dialog id="dlg-key" class="dlg"> | |
| 45 | + <form method="dialog"> | |
| 46 | + <h3>Clé API</h3> | |
| 47 | + <p class="muted">Stockée localement dans ce navigateur (chiffrée légèrement). Laisser vide en self-host sans auth.</p> | |
| 48 | + <input id="key-input" class="input mono" placeholder="trw_… ou clé admin" autocomplete="off"> | |
| 49 | + <div class="row end gap"> | |
| 50 | + <button class="btn ghost" value="cancel">Annuler</button> | |
| 51 | + <button class="btn" id="key-save" value="ok">Enregistrer</button> | |
| 52 | + </div> | |
| 53 | + </form> | |
| 54 | +</dialog> | |
| 55 | + | |
| 56 | +<template id="tpl-playground"> | |
| 57 | +<section class="page playground"> | |
| 58 | + <div class="panel left"> | |
| 59 | + <form id="pg-form" class="stack"> | |
| 60 | + <label class="field"> | |
| 61 | + <span>URL</span> | |
| 62 | + <div class="row gap"> | |
| 63 | + <input class="input mono grow" id="pg-url" placeholder="https://exemple.com/article" required autocomplete="off" spellcheck="false"> | |
| 64 | + <button class="btn" id="pg-run" type="submit">Run <kbd>⌘↵</kbd></button> | |
| 65 | + </div> | |
| 66 | + </label> | |
| 67 | + <div class="grid2"> | |
| 68 | + <label class="field"><span>Formats</span> | |
| 69 | + <div class="chips" id="pg-formats"> | |
| 70 | + <label><input type="checkbox" value="markdown" checked>markdown</label> | |
| 71 | + <label><input type="checkbox" value="html">html</label> | |
| 72 | + <label><input type="checkbox" value="raw_html">raw_html</label> | |
| 73 | + <label><input type="checkbox" value="json">json</label> | |
| 74 | + <label><input type="checkbox" value="links">links</label> | |
| 75 | + <label><input type="checkbox" value="screenshot">screenshot</label> | |
| 76 | + <label><input type="checkbox" value="chunks">chunks</label> | |
| 77 | + </div> | |
| 78 | + </label> | |
| 79 | + <label class="field"><span>Mode</span> | |
| 80 | + <select class="input" id="pg-mode"><option value="auto">auto (http → browser → stealth)</option><option value="http">http</option><option value="browser">browser</option><option value="stealth">stealth</option></select> | |
| 81 | + </label> | |
| 82 | + <label class="field"><span>wait_for (sélecteur ou ms)</span><input class="input mono" id="pg-wait" placeholder=".article-body ou 2000"></label> | |
| 83 | + <label class="field"><span>timeout_ms</span><input class="input mono" id="pg-timeout" type="number" value="30000" min="1000" step="1000"></label> | |
| 84 | + <label class="field"><span>exclude_tags (CSS, virgules)</span><input class="input mono" id="pg-exclude" placeholder=".ads, #comments"></label> | |
| 85 | + <label class="field"><span>include_tags</span><input class="input mono" id="pg-include" placeholder="article"></label> | |
| 86 | + </div> | |
| 87 | + <div class="row gap wrap"> | |
| 88 | + <label class="check"><input type="checkbox" id="pg-main" checked> only_main_content</label> | |
| 89 | + <label class="check"><input type="checkbox" id="pg-cit"> citations</label> | |
| 90 | + <label class="check"><input type="checkbox" id="pg-b64" checked> remove_base64_images</label> | |
| 91 | + <label class="check"><input type="checkbox" id="pg-cache" checked> cache</label> | |
| 92 | + <label class="check"><input type="checkbox" id="pg-robots" checked> respect_robots</label> | |
| 93 | + </div> | |
| 94 | + <details class="adv"> | |
| 95 | + <summary>Actions navigateur (JSON)</summary> | |
| 96 | + <textarea class="input mono" id="pg-actions" rows="4" placeholder='[{"type":"click","selector":"#accept"},{"type":"scroll","amount":2000},{"type":"wait","ms":1000}]'></textarea> | |
| 97 | + </details> | |
| 98 | + <details class="adv"> | |
| 99 | + <summary>Extraction (JSON schema → LLM, ou CSS)</summary> | |
| 100 | + <div class="row gap"><select class="input" id="pg-ex-mode"><option value="">désactivée</option><option value="css">css</option><option value="llm">llm</option></select></div> | |
| 101 | + <textarea class="input mono" id="pg-ex-body" rows="5" placeholder='css: {"title":{"selector":"h1"},"price":{"selector":".price","type":"float"}} | |
| 102 | +llm: {"type":"object","properties":{"title":{"type":"string"},"price":{"type":"number"}}}'></textarea> | |
| 103 | + <input class="input" id="pg-ex-prompt" placeholder="prompt optionnel (llm)"> | |
| 104 | + </details> | |
| 105 | + </form> | |
| 106 | + <div class="hist"> | |
| 107 | + <div class="row between"><h4>Historique</h4><button class="btn ghost sm" id="pg-hist-clear">vider</button></div> | |
| 108 | + <ul id="pg-hist" class="list"></ul> | |
| 109 | + </div> | |
| 110 | + </div> | |
| 111 | + <div class="panel right"> | |
| 112 | + <div class="tabs" id="pg-tabs"> | |
| 113 | + <button data-tab="md" class="on">Markdown</button> | |
| 114 | + <button data-tab="raw">MD brut</button> | |
| 115 | + <button data-tab="json">JSON</button> | |
| 116 | + <button data-tab="html">HTML</button> | |
| 117 | + <button data-tab="links">Links</button> | |
| 118 | + <button data-tab="shot">Screenshot</button> | |
| 119 | + <button data-tab="chunks">Chunks</button> | |
| 120 | + <button data-tab="meta">Metadata</button> | |
| 121 | + <span class="grow"></span> | |
| 122 | + <div class="copy-group"> | |
| 123 | + <button class="btn ghost sm" data-copy="curl">cURL</button> | |
| 124 | + <button class="btn ghost sm" data-copy="python">Python</button> | |
| 125 | + <button class="btn ghost sm" data-copy="ts">TS</button> | |
| 126 | + <button class="btn ghost sm" data-copy="out">Copier sortie</button> | |
| 127 | + </div> | |
| 128 | + </div> | |
| 129 | + <div class="result-bar mono" id="pg-bar">Prêt. Entrez une URL.</div> | |
| 130 | + <div class="result" id="pg-result"><div class="empty">Le résultat s'affiche ici.</div></div> | |
| 131 | + </div> | |
| 132 | +</section> | |
| 133 | +</template> | |
| 134 | + | |
| 135 | +<template id="tpl-crawls"> | |
| 136 | +<section class="page crawls"> | |
| 137 | + <div class="panel left"> | |
| 138 | + <form id="cr-form" class="stack"> | |
| 139 | + <label class="field"><span>URL racine</span><input class="input mono" id="cr-url" placeholder="https://docs.exemple.com" required></label> | |
| 140 | + <div class="grid2"> | |
| 141 | + <label class="field"><span>max_pages</span><input class="input mono" id="cr-pages" type="number" value="50" min="1" max="20000"></label> | |
| 142 | + <label class="field"><span>max_depth</span><input class="input mono" id="cr-depth" type="number" value="3" min="0" max="20"></label> | |
| 143 | + <label class="field"><span>concurrency</span><input class="input mono" id="cr-conc" type="number" value="5" min="1" max="32"></label> | |
| 144 | + <label class="field"><span>strategy</span><select class="input" id="cr-strat"><option>bfs</option><option>dfs</option><option>best_first</option></select></label> | |
| 145 | + <label class="field"><span>include_paths (globs, virgules)</span><input class="input mono" id="cr-inc" placeholder="/docs/*, /blog/*"></label> | |
| 146 | + <label class="field"><span>exclude_paths</span><input class="input mono" id="cr-exc" placeholder="/tag/*, *.pdf"></label> | |
| 147 | + <label class="field"><span>search (best_first)</span><input class="input" id="cr-search" placeholder="pricing api"></label> | |
| 148 | + <label class="field"><span>delay_ms</span><input class="input mono" id="cr-delay" type="number" value="0" min="0"></label> | |
| 149 | + </div> | |
| 150 | + <div class="row gap wrap"> | |
| 151 | + <label class="check"><input type="checkbox" id="cr-sub"> allow_subdomains</label> | |
| 152 | + <label class="check"><input type="checkbox" id="cr-ext"> allow_external_links</label> | |
| 153 | + <label class="check"><input type="checkbox" id="cr-nosm"> ignore_sitemap</label> | |
| 154 | + <label class="check"><input type="checkbox" id="cr-robots" checked> respect_robots</label> | |
| 155 | + <label class="check"><input type="checkbox" id="cr-links"> links dans la sortie</label> | |
| 156 | + </div> | |
| 157 | + <details class="adv"><summary>Requête JSON brute (écrase le formulaire)</summary><textarea class="input mono" id="cr-json" rows="6" placeholder='{"url":"https://…","crawl":{"max_pages":100},"scrape":{"formats":["markdown"]}}'></textarea></details> | |
| 158 | + <div class="row gap"><button class="btn" type="submit">Lancer le crawl</button></div> | |
| 159 | + </form> | |
| 160 | + <div class="row between"><h4>Jobs</h4><button class="btn ghost sm" id="cr-refresh">↻</button></div> | |
| 161 | + <table class="tbl" id="cr-jobs"><thead><tr><th>id</th><th>type</th><th>statut</th><th>pages</th><th>durée</th><th></th></tr></thead><tbody></tbody></table> | |
| 162 | + </div> | |
| 163 | + <div class="panel right" id="cr-detail"> | |
| 164 | + <div class="empty">Sélectionnez un job ou lancez un crawl.</div> | |
| 165 | + </div> | |
| 166 | +</section> | |
| 167 | +</template> | |
| 168 | + | |
| 169 | +<template id="tpl-map"> | |
| 170 | +<section class="page mapp"> | |
| 171 | + <form id="mp-form" class="row gap wrap top-form"> | |
| 172 | + <input class="input mono grow" id="mp-url" placeholder="https://exemple.com" required> | |
| 173 | + <input class="input" id="mp-search" placeholder="search (BM25, optionnel)"> | |
| 174 | + <input class="input mono" id="mp-limit" type="number" value="2000" min="1" max="20000" style="width:8rem" title="limit"> | |
| 175 | + <label class="check"><input type="checkbox" id="mp-sub"> sous-domaines</label> | |
| 176 | + <label class="check"><input type="checkbox" id="mp-titles"> titres</label> | |
| 177 | + <button class="btn" type="submit">Map</button> | |
| 178 | + </form> | |
| 179 | + <div class="result-bar mono" id="mp-bar">—</div> | |
| 180 | + <div class="row gap wrap tools"> | |
| 181 | + <input class="input mono grow" id="mp-filter" placeholder="filtre instantané…"> | |
| 182 | + <label class="check"><input type="checkbox" id="mp-group"> grouper par path</label> | |
| 183 | + <button class="btn ghost sm" id="mp-csv">CSV</button> | |
| 184 | + <button class="btn ghost sm" id="mp-json">JSON</button> | |
| 185 | + <button class="btn ghost sm" id="mp-copy">Copier</button> | |
| 186 | + <button class="btn sm" id="mp-crawl">Crawler la sélection</button> | |
| 187 | + </div> | |
| 188 | + <div class="vlist" id="mp-list"></div> | |
| 189 | +</section> | |
| 190 | +</template> | |
| 191 | + | |
| 192 | +<template id="tpl-extract"> | |
| 193 | +<section class="page extract"> | |
| 194 | + <div class="panel left"> | |
| 195 | + <form id="ex-form" class="stack"> | |
| 196 | + <label class="field"><span>URLs (une par ligne) ou pattern (https://site.com/produits/*)</span><textarea class="input mono" id="ex-urls" rows="4" required></textarea></label> | |
| 197 | + <div class="row gap wrap"> | |
| 198 | + <label class="check"><input type="radio" name="ex-mode" value="css" checked> CSS (déterministe)</label> | |
| 199 | + <label class="check"><input type="radio" name="ex-mode" value="llm"> LLM (schéma JSON)</label> | |
| 200 | + </div> | |
| 201 | + <div class="row between"><h4>Schéma</h4><button class="btn ghost sm" id="ex-add" type="button">+ champ</button></div> | |
| 202 | + <div id="ex-fields" class="stack sm"></div> | |
| 203 | + <label class="field"><span>JSON synchronisé</span><textarea class="input mono" id="ex-json" rows="7"></textarea></label> | |
| 204 | + <label class="field"><span>prompt (llm)</span><input class="input" id="ex-prompt" placeholder="Extrais le produit principal de la page"></label> | |
| 205 | + <label class="field"><span>merge_key (dédup multi-pages)</span><input class="input mono" id="ex-merge" placeholder="name"></label> | |
| 206 | + <button class="btn" type="submit">Tester l'extraction</button> | |
| 207 | + </form> | |
| 208 | + </div> | |
| 209 | + <div class="panel right"> | |
| 210 | + <div class="result-bar mono" id="ex-bar">—</div> | |
| 211 | + <div class="result" id="ex-result"><div class="empty">Résultat par URL et fusion.</div></div> | |
| 212 | + </div> | |
| 213 | +</section> | |
| 214 | +</template> | |
| 215 | + | |
| 216 | +<template id="tpl-keys"> | |
| 217 | +<section class="page keys"> | |
| 218 | + <div class="panel left"> | |
| 219 | + <h4>Système</h4> | |
| 220 | + <pre class="code mono" id="ky-system">…</pre> | |
| 221 | + <h4>Usage (30 j)</h4> | |
| 222 | + <div id="ky-usage" class="bars"></div> | |
| 223 | + </div> | |
| 224 | + <div class="panel right"> | |
| 225 | + <div class="row between"><h4>Clés API</h4> | |
| 226 | + <form id="ky-form" class="row gap"><input class="input" id="ky-name" placeholder="nom" required><input class="input mono" id="ky-quota" type="number" placeholder="quota/jour" style="width:8rem"><button class="btn sm" type="submit">Créer</button></form> | |
| 227 | + </div> | |
| 228 | + <div id="ky-new" class="callout hidden"></div> | |
| 229 | + <table class="tbl" id="ky-table"><thead><tr><th>id</th><th>nom</th><th>préfixe</th><th>quota</th><th>dernier usage</th><th></th></tr></thead><tbody></tbody></table> | |
| 230 | + <h4>Dernières requêtes</h4> | |
| 231 | + <table class="tbl mono sm" id="ky-req"><thead><tr><th>quand</th><th>méthode</th><th>path</th><th>statut</th><th>ms</th></tr></thead><tbody></tbody></table> | |
| 232 | + </div> | |
| 233 | +</section> | |
| 234 | +</template> | |
| 235 | + | |
| 236 | +<template id="tpl-agent"> | |
| 237 | +<section class="page"> | |
| 238 | + <div class="panel"><h3>Agent</h3><p class="muted">La console agent (observe → décide → agit → vérifie) arrive en phase 7 de la feuille de route. Les phases 1 à 6 (fetch, Markdown, API, jobs, map, PDF, extract, UI) sont livrées.</p></div> | |
| 239 | +</section> | |
| 240 | +</template> | |
| 241 | + | |
| 242 | +<script src="https://cdn.jsdelivr.net/npm/marked@12/marked.min.js"></script> | |
| 243 | +<script src="/static/app.js" defer></script> | |
| 244 | +</body> | |
| 245 | +</html> | |
added
trawls/web/static/og.svg
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | +<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="630" viewBox="0 0 1200 630"><rect width="1200" height="630" fill="#0b0c0e"/><g fill="none" stroke="#f5a524" stroke-width="10" stroke-linecap="round"><path d="M80 240c120 0 160 120 240 120s120-120 240-120 160 120 240 120 120-120 240-120"/><path d="M80 330c120 0 160 120 240 120s120-120 240-120 160 120 240 120 120-120 240-120" opacity=".6"/><path d="M80 420c120 0 160 120 240 120s120-120 240-120 160 120 240 120 120-120 240-120" opacity=".3"/></g><text x="80" y="150" font-family="Inter,system-ui,sans-serif" font-weight="700" font-size="96" fill="#e6e8eb">Trawls</text><text x="400" y="150" font-family="JetBrains Mono,monospace" font-size="34" fill="#a2a8b3">www.trawls.dev</text><text x="80" y="580" font-family="Inter,system-ui,sans-serif" font-size="34" fill="#a2a8b3">Crawl · Extract · Navigate — Markdown LLM-ready, self-host first</text></svg> | |
added
trawls/web/static/style.css
+125 −0
@@ -0,0 +1,125 @@ | ||
| 1 | +:root{ | |
| 2 | + --bg:#0b0c0e;--bg2:#111317;--bg3:#171a1f;--line:#23272e;--line2:#2e333b; | |
| 3 | + --fg:#e6e8eb;--fg2:#a2a8b3;--fg3:#6b7280; | |
| 4 | + --accent:#f5a524;--accent2:#c98416;--ok:#3fb950;--warn:#d29922;--err:#f85149;--info:#58a6ff; | |
| 5 | + --mono:"JetBrains Mono",ui-monospace,SFMono-Regular,Menlo,monospace;--sans:Inter,system-ui,-apple-system,Segoe UI,Roboto,sans-serif; | |
| 6 | + --r:6px;--top:44px;--foot:28px; | |
| 7 | +} | |
| 8 | +*{box-sizing:border-box} | |
| 9 | +html,body{height:100%} | |
| 10 | +body{margin:0;background:var(--bg);color:var(--fg);font:13px/1.45 var(--sans);-webkit-font-smoothing:antialiased;display:flex;flex-direction:column;min-height:100dvh} | |
| 11 | +a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline} | |
| 12 | +.mono,code,pre,kbd,.input.mono,textarea.mono{font-family:var(--mono);font-size:12px} | |
| 13 | +.muted{color:var(--fg2)}.hidden{display:none!important}.grow{flex:1;min-width:0} | |
| 14 | +.row{display:flex;align-items:center}.row.gap{gap:8px}.row.between{justify-content:space-between}.row.end{justify-content:flex-end}.row.wrap{flex-wrap:wrap} | |
| 15 | +.stack{display:flex;flex-direction:column;gap:10px}.stack.sm{gap:6px} | |
| 16 | +.grid2{display:grid;grid-template-columns:1fr 1fr;gap:8px 12px} | |
| 17 | +h3,h4{margin:6px 0;font-weight:600}h4{font-size:12px;color:var(--fg2);text-transform:uppercase;letter-spacing:.04em} | |
| 18 | +kbd{background:var(--bg3);border:1px solid var(--line2);border-radius:3px;padding:0 4px;font-size:10px;color:var(--fg2);margin-left:6px} | |
| 19 | + | |
| 20 | +.top{position:sticky;top:0;z-index:10;height:var(--top);display:flex;align-items:center;gap:18px;padding:0 14px;background:var(--bg2);border-bottom:1px solid var(--line)} | |
| 21 | +.brand{display:flex;align-items:center;gap:8px;font-weight:700;font-size:14px;color:var(--fg)}.brand:hover{text-decoration:none} | |
| 22 | +.brand .ver{font-family:var(--mono);font-size:10px;color:var(--fg3);font-weight:400} | |
| 23 | +.nav{display:flex;gap:2px;flex:1} | |
| 24 | +.nav a{padding:5px 10px;border-radius:var(--r);color:var(--fg2);font-weight:500} | |
| 25 | +.nav a:hover{background:var(--bg3);text-decoration:none;color:var(--fg)}.nav a.on{color:var(--accent);background:rgba(245,165,36,.08)} | |
| 26 | +.top-right{display:flex;align-items:center;gap:8px} | |
| 27 | +.status{display:flex;align-items:center;gap:6px;font-family:var(--mono);font-size:11px;color:var(--fg2)} | |
| 28 | +.status i{width:8px;height:8px;border-radius:50%;background:var(--fg3);display:inline-block} | |
| 29 | +.status.ok i{background:var(--ok)}.status.err i{background:var(--err)}.status.warn i{background:var(--warn)} | |
| 30 | + | |
| 31 | +.view{flex:1;display:flex;min-height:0} | |
| 32 | +.page{flex:1;display:grid;grid-template-columns:minmax(340px,420px) 1fr;min-height:calc(100dvh - var(--top) - var(--foot))} | |
| 33 | +.page.mapp,.page.keys{grid-template-columns:1fr} | |
| 34 | +.page.keys{grid-template-columns:minmax(320px,400px) 1fr} | |
| 35 | +.panel{padding:14px;overflow:auto;min-width:0} | |
| 36 | +.panel.left{border-right:1px solid var(--line);background:var(--bg2)} | |
| 37 | +.panel.right{display:flex;flex-direction:column;min-height:0} | |
| 38 | + | |
| 39 | +.field{display:flex;flex-direction:column;gap:4px} | |
| 40 | +.field>span{font-size:11px;color:var(--fg2);font-weight:500} | |
| 41 | +.input{background:var(--bg);border:1px solid var(--line2);color:var(--fg);border-radius:var(--r);padding:6px 8px;font:inherit;outline:none;min-width:0;width:100%} | |
| 42 | +.input:focus{border-color:var(--accent);box-shadow:0 0 0 2px rgba(245,165,36,.18)} | |
| 43 | +textarea.input{resize:vertical;line-height:1.4} | |
| 44 | +select.input{appearance:none;background-image:linear-gradient(45deg,transparent 50%,var(--fg2) 50%),linear-gradient(135deg,var(--fg2) 50%,transparent 50%);background-position:calc(100% - 14px) 50%,calc(100% - 9px) 50%;background-size:5px 5px;background-repeat:no-repeat;padding-right:26px} | |
| 45 | +.btn{background:var(--accent);color:#161000;border:1px solid var(--accent2);border-radius:var(--r);padding:6px 12px;font:inherit;font-weight:600;cursor:pointer;white-space:nowrap} | |
| 46 | +.btn:hover{background:#ffb640}.btn:disabled{opacity:.5;cursor:progress} | |
| 47 | +.btn.ghost{background:transparent;color:var(--fg2);border-color:var(--line2)}.btn.ghost:hover{color:var(--fg);background:var(--bg3)} | |
| 48 | +.btn.sm{padding:3px 8px;font-size:11px} | |
| 49 | +.btn.danger{background:transparent;color:var(--err);border-color:var(--line2)} | |
| 50 | +.check{display:flex;align-items:center;gap:5px;color:var(--fg2);font-size:12px;cursor:pointer;user-select:none} | |
| 51 | +.check input{accent-color:var(--accent)} | |
| 52 | +.chips{display:flex;flex-wrap:wrap;gap:4px} | |
| 53 | +.chips label{display:flex;align-items:center;gap:4px;border:1px solid var(--line2);border-radius:999px;padding:2px 8px;font-size:11px;color:var(--fg2);cursor:pointer;font-family:var(--mono)} | |
| 54 | +.chips label:has(input:checked){border-color:var(--accent);color:var(--accent);background:rgba(245,165,36,.08)} | |
| 55 | +.chips input{display:none} | |
| 56 | +details.adv{border:1px solid var(--line);border-radius:var(--r);padding:6px 8px;background:var(--bg)} | |
| 57 | +details.adv summary{cursor:pointer;color:var(--fg2);font-size:12px;font-weight:500} | |
| 58 | +details.adv[open] summary{margin-bottom:8px} | |
| 59 | +details.adv .input{margin-top:6px} | |
| 60 | + | |
| 61 | +.tabs{display:flex;align-items:center;gap:2px;border-bottom:1px solid var(--line);padding:0 0 6px;flex-wrap:wrap} | |
| 62 | +.tabs>button{background:none;border:0;color:var(--fg2);padding:5px 10px;border-radius:var(--r);cursor:pointer;font:inherit;font-weight:500} | |
| 63 | +.tabs>button:hover{color:var(--fg);background:var(--bg3)}.tabs>button.on{color:var(--accent);background:rgba(245,165,36,.08)} | |
| 64 | +.copy-group{display:flex;gap:4px} | |
| 65 | +.result-bar{padding:6px 0;color:var(--fg2);font-size:11px;border-bottom:1px solid var(--line);display:flex;gap:12px;flex-wrap:wrap} | |
| 66 | +.result-bar .ok{color:var(--ok)}.result-bar .err{color:var(--err)}.result-bar .warn{color:var(--warn)}.result-bar b{color:var(--fg)} | |
| 67 | +.result{flex:1;overflow:auto;padding:12px 4px;min-height:0} | |
| 68 | +.empty{color:var(--fg3);padding:40px;text-align:center} | |
| 69 | +.prose{max-width:860px;line-height:1.6;font-size:14px} | |
| 70 | +.prose h1{font-size:22px;margin:.6em 0 .4em}.prose h2{font-size:18px;margin:1em 0 .4em;border-bottom:1px solid var(--line);padding-bottom:4px}.prose h3{font-size:15px} | |
| 71 | +.prose pre{background:var(--bg2);border:1px solid var(--line);padding:10px;border-radius:var(--r);overflow:auto}.prose code{background:var(--bg3);padding:1px 4px;border-radius:3px}.prose pre code{background:none;padding:0} | |
| 72 | +.prose table{border-collapse:collapse;font-size:13px}.prose td,.prose th{border:1px solid var(--line2);padding:4px 8px}.prose th{background:var(--bg3)} | |
| 73 | +.prose img{max-width:100%;border-radius:var(--r)}.prose blockquote{border-left:3px solid var(--accent);margin:0;padding:2px 12px;color:var(--fg2)} | |
| 74 | +pre.code{background:var(--bg2);border:1px solid var(--line);border-radius:var(--r);padding:10px;overflow:auto;white-space:pre-wrap;word-break:break-word;margin:0;font-size:12px;line-height:1.45} | |
| 75 | +.json-k{color:#79c0ff}.json-s{color:#a5d6ff}.json-n{color:#f5a524}.json-b{color:#ff7b72}.json-null{color:var(--fg3)} | |
| 76 | + | |
| 77 | +.tbl{width:100%;border-collapse:collapse;font-size:12px} | |
| 78 | +.tbl th{text-align:left;color:var(--fg2);font-weight:500;padding:6px 8px;border-bottom:1px solid var(--line);font-size:11px;text-transform:uppercase;letter-spacing:.03em} | |
| 79 | +.tbl td{padding:5px 8px;border-bottom:1px solid var(--line);vertical-align:top} | |
| 80 | +.tbl tr:hover td{background:var(--bg3)}.tbl tr.sel td{background:rgba(245,165,36,.07)} | |
| 81 | +.tbl.sm td,.tbl.sm th{padding:3px 6px;font-size:11px} | |
| 82 | +.badge{display:inline-block;padding:1px 6px;border-radius:999px;font-size:10px;font-family:var(--mono);border:1px solid var(--line2);color:var(--fg2)} | |
| 83 | +.badge.ok{color:var(--ok);border-color:rgba(63,185,80,.4)}.badge.failed,.badge.err{color:var(--err);border-color:rgba(248,81,73,.4)} | |
| 84 | +.badge.running,.badge.queued{color:var(--accent);border-color:rgba(245,165,36,.4)}.badge.skipped{color:var(--warn)}.badge.completed{color:var(--info);border-color:rgba(88,166,255,.4)}.badge.cancelled{color:var(--fg3)} | |
| 85 | +.link-btn{background:none;border:0;color:var(--accent);cursor:pointer;font:inherit;padding:0} | |
| 86 | + | |
| 87 | +.list{list-style:none;margin:0;padding:0;display:flex;flex-direction:column;gap:2px} | |
| 88 | +.list li{display:flex;gap:8px;align-items:center;padding:4px 6px;border-radius:var(--r);cursor:pointer;color:var(--fg2);font-family:var(--mono);font-size:11px;overflow:hidden} | |
| 89 | +.list li:hover{background:var(--bg3);color:var(--fg)}.list li span{white-space:nowrap;overflow:hidden;text-overflow:ellipsis} | |
| 90 | +.hist{margin-top:16px} | |
| 91 | + | |
| 92 | +.progress{height:6px;background:var(--bg3);border-radius:3px;overflow:hidden;margin:6px 0} | |
| 93 | +.progress>i{display:block;height:100%;background:var(--accent);transition:width .3s} | |
| 94 | +.tree{font-family:var(--mono);font-size:11px;line-height:1.7} | |
| 95 | +.tree details{margin-left:10px}.tree summary{cursor:pointer;color:var(--fg2)}.tree summary:hover{color:var(--fg)} | |
| 96 | +.tree .leaf{margin-left:24px;display:flex;gap:6px;align-items:center;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;cursor:pointer} | |
| 97 | +.tree .leaf:hover{color:var(--accent)} | |
| 98 | +.split{display:grid;grid-template-columns:minmax(260px,38%) 1fr;gap:12px;flex:1;min-height:0} | |
| 99 | +.split>div{min-height:0;overflow:auto} | |
| 100 | +.log{font-family:var(--mono);font-size:11px;line-height:1.6;white-space:pre-wrap} | |
| 101 | +.log .ok{color:var(--ok)}.log .failed{color:var(--err)}.log .skipped{color:var(--warn)} | |
| 102 | + | |
| 103 | +.top-form{padding:12px 14px;border-bottom:1px solid var(--line);background:var(--bg2)} | |
| 104 | +.tools{padding:8px 14px;border-bottom:1px solid var(--line)} | |
| 105 | +.mapp .result-bar{padding:6px 14px} | |
| 106 | +.vlist{flex:1;overflow:auto;font-family:var(--mono);font-size:11px;padding:4px 14px} | |
| 107 | +.vrow{display:grid;grid-template-columns:20px 1fr 90px 70px;gap:8px;align-items:center;padding:3px 4px;border-bottom:1px solid var(--line);height:26px;white-space:nowrap;overflow:hidden} | |
| 108 | +.vrow:hover{background:var(--bg3)}.vrow span{overflow:hidden;text-overflow:ellipsis}.vrow .src{color:var(--fg3)} | |
| 109 | +.vgroup{padding:6px 4px 2px;color:var(--accent);font-weight:600;border-bottom:1px solid var(--line2)} | |
| 110 | + | |
| 111 | +.ex-field{display:grid;grid-template-columns:1fr 1.4fr 90px 24px;gap:6px;align-items:center} | |
| 112 | +.ex-field.llm{grid-template-columns:1fr 100px 1.4fr 24px} | |
| 113 | +.callout{background:rgba(245,165,36,.08);border:1px solid var(--accent2);border-radius:var(--r);padding:10px;margin:8px 0;font-family:var(--mono);font-size:12px;word-break:break-all} | |
| 114 | +.bars{display:flex;align-items:flex-end;gap:2px;height:80px;border-bottom:1px solid var(--line);padding:0 2px} | |
| 115 | +.bars>i{flex:1;background:var(--accent);opacity:.8;border-radius:2px 2px 0 0;min-height:1px} | |
| 116 | +.bars>i:hover{opacity:1} | |
| 117 | +.foot{height:var(--foot);display:flex;align-items:center;justify-content:space-between;padding:0 14px;border-top:1px solid var(--line);color:var(--fg3);font-size:11px;background:var(--bg2)} | |
| 118 | +.dlg{background:var(--bg2);color:var(--fg);border:1px solid var(--line2);border-radius:8px;padding:16px;width:min(460px,92vw)} | |
| 119 | +.dlg::backdrop{background:rgba(0,0,0,.6)}.dlg form{display:flex;flex-direction:column;gap:10px} | |
| 120 | +.shot{max-width:100%;border:1px solid var(--line2);border-radius:var(--r)} | |
| 121 | +.chunk{border:1px solid var(--line);border-radius:var(--r);padding:8px;margin-bottom:8px;background:var(--bg2)} | |
| 122 | +.chunk .h{font-family:var(--mono);font-size:10px;color:var(--fg3);margin-bottom:4px;display:flex;gap:10px} | |
| 123 | +.toast{position:fixed;bottom:40px;right:14px;background:var(--bg3);border:1px solid var(--line2);padding:8px 12px;border-radius:var(--r);font-size:12px;z-index:20;box-shadow:0 6px 20px rgba(0,0,0,.4)} | |
| 124 | +.toast.err{border-color:var(--err)} | |
| 125 | +@media (max-width:900px){.page{grid-template-columns:1fr;min-height:auto}.panel.left{border-right:0;border-bottom:1px solid var(--line)}.split{grid-template-columns:1fr}.nav a{padding:5px 7px}.grid2{grid-template-columns:1fr}.top{gap:10px;overflow-x:auto}.page.keys{grid-template-columns:1fr}} | |
added
trawls/worker/__init__.py
+1 −0
@@ -0,0 +1 @@ | ||
| 1 | + | |
added
trawls/worker/crawl.py
+489 −0
@@ -0,0 +1,489 @@ | ||
| 1 | +"""Moteur de jobs en processus (asyncio) : crawl (frontier BFS/DFS/best-first, dédup, politesse, | |
| 2 | +budget, reprise), batch scrape, extract multi-URL. Événements publiés pour le SSE. | |
| 3 | + | |
| 4 | +Invariants : une page échouée ne stoppe jamais le job ; chaque page est persistée dès qu'elle est prête ; | |
| 5 | +la frontier est sauvegardée périodiquement (reprise après crash). | |
| 6 | +""" | |
| 7 | + | |
| 8 | +from __future__ import annotations | |
| 9 | + | |
| 10 | +import asyncio | |
| 11 | +import fnmatch | |
| 12 | +import hashlib | |
| 13 | +import hmac | |
| 14 | +import json | |
| 15 | +import time | |
| 16 | +from collections import defaultdict | |
| 17 | +from typing import Any | |
| 18 | +from urllib.parse import urlsplit | |
| 19 | + | |
| 20 | +import httpx | |
| 21 | +import structlog | |
| 22 | + | |
| 23 | +from trawls.config import get_settings | |
| 24 | +from trawls.core.resilience.budget import Budget | |
| 25 | +from trawls.core.scheduler.dedup import ContentDeduper, host_of, normalize_url, same_site | |
| 26 | +from trawls.core.scheduler.politeness import HostLimiter | |
| 27 | +from trawls.core.scheduler.robots import robots_cache | |
| 28 | +from trawls.core.scrape import scrape | |
| 29 | +from trawls.extract.merge import merge_results | |
| 30 | +from trawls.map import map_site, rank | |
| 31 | +from trawls.models import CrawlOptions, ErrorCode, ErrorInfo, MapOptions, MappedUrl, PageResult, ScrapeOptions | |
| 32 | +from trawls.worker.store import Store, get_store | |
| 33 | + | |
| 34 | +log = structlog.get_logger(__name__) | |
| 35 | + | |
| 36 | + | |
| 37 | +class EventBus: | |
| 38 | + """Abonnés SSE par job : file d'événements JSON.""" | |
| 39 | + | |
| 40 | + def __init__(self) -> None: | |
| 41 | + self._subs: dict[str, list[asyncio.Queue[dict[str, Any] | None]]] = defaultdict(list) | |
| 42 | + | |
| 43 | + def subscribe(self, jid: str) -> asyncio.Queue[dict[str, Any] | None]: | |
| 44 | + q: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue(maxsize=1000) | |
| 45 | + self._subs[jid].append(q) | |
| 46 | + return q | |
| 47 | + | |
| 48 | + def unsubscribe(self, jid: str, q: asyncio.Queue[dict[str, Any] | None]) -> None: | |
| 49 | + try: | |
| 50 | + self._subs[jid].remove(q) | |
| 51 | + except ValueError: | |
| 52 | + pass | |
| 53 | + if not self._subs[jid]: | |
| 54 | + del self._subs[jid] | |
| 55 | + | |
| 56 | + def publish(self, jid: str, event: str, data: Any) -> None: | |
| 57 | + for q in list(self._subs.get(jid, [])): | |
| 58 | + try: | |
| 59 | + q.put_nowait({"event": event, "data": data}) | |
| 60 | + except asyncio.QueueFull: | |
| 61 | + pass | |
| 62 | + | |
| 63 | + def close(self, jid: str) -> None: | |
| 64 | + for q in list(self._subs.get(jid, [])): | |
| 65 | + try: | |
| 66 | + q.put_nowait(None) | |
| 67 | + except asyncio.QueueFull: | |
| 68 | + pass | |
| 69 | + | |
| 70 | + | |
| 71 | +bus = EventBus() | |
| 72 | + | |
| 73 | + | |
| 74 | +def _match_paths(url: str, include: list[str], exclude: list[str]) -> bool: | |
| 75 | + path = urlsplit(url).path or "/" | |
| 76 | + if include and not any( | |
| 77 | + fnmatch.fnmatch(path, g) or fnmatch.fnmatch(url, g) or path.startswith(g.rstrip("*")) for g in include | |
| 78 | + ): | |
| 79 | + return False | |
| 80 | + if exclude and any( | |
| 81 | + fnmatch.fnmatch(path, g) or fnmatch.fnmatch(url, g) or path.startswith(g.rstrip("*")) for g in exclude | |
| 82 | + ): | |
| 83 | + return False | |
| 84 | + return True | |
| 85 | + | |
| 86 | + | |
| 87 | +class Frontier: | |
| 88 | + """File d'URLs priorisée. BFS = FIFO par profondeur ; DFS = LIFO ; best_first = score BM25 décroissant.""" | |
| 89 | + | |
| 90 | + def __init__(self, strategy: str, search: str | None) -> None: | |
| 91 | + self.strategy = strategy | |
| 92 | + self.search = search | |
| 93 | + self._items: list[tuple[float, int, str]] = [] # (priorité, profondeur, url) | |
| 94 | + self.seen: set[str] = set() | |
| 95 | + self._n = 0 | |
| 96 | + | |
| 97 | + def push(self, url: str, depth: int) -> bool: | |
| 98 | + if url in self.seen: | |
| 99 | + return False | |
| 100 | + self.seen.add(url) | |
| 101 | + self._n += 1 | |
| 102 | + if self.strategy == "dfs": | |
| 103 | + prio = -float(self._n) | |
| 104 | + elif self.strategy == "best_first" and self.search: | |
| 105 | + prio = -(rank([MappedUrl(url=url)], self.search)[0].score or 0.0) + depth * 0.01 | |
| 106 | + else: | |
| 107 | + prio = depth * 1_000_000 + self._n | |
| 108 | + self._items.append((prio, depth, url)) | |
| 109 | + return True | |
| 110 | + | |
| 111 | + def pop_batch(self, n: int) -> list[tuple[str, int]]: | |
| 112 | + self._items.sort(key=lambda t: t[0]) | |
| 113 | + batch, self._items = self._items[:n], self._items[n:] | |
| 114 | + return [(u, d) for _, d, u in batch] | |
| 115 | + | |
| 116 | + def __len__(self) -> int: | |
| 117 | + return len(self._items) | |
| 118 | + | |
| 119 | + def dump(self) -> dict[str, Any]: | |
| 120 | + return {"pending": [(u, d) for _, d, u in self._items], "seen": sorted(self.seen)[:50_000]} | |
| 121 | + | |
| 122 | + def load(self, d: dict[str, Any]) -> None: | |
| 123 | + self.seen = set(d.get("seen", [])) | |
| 124 | + for u, depth in d.get("pending", []): | |
| 125 | + self._n += 1 | |
| 126 | + self._items.append( | |
| 127 | + (depth * 1_000_000 + self._n if self.strategy != "dfs" else -float(self._n), depth, u) | |
| 128 | + ) | |
| 129 | + | |
| 130 | + | |
| 131 | +async def run_crawl(jid: str, req: dict[str, Any], store: Store) -> None: | |
| 132 | + root = normalize_url(req["url"]) or req["url"] | |
| 133 | + copts = CrawlOptions.model_validate(req.get("crawl") or {}) | |
| 134 | + sopts = ScrapeOptions.model_validate(req.get("scrape") or {}) | |
| 135 | + sopts.respect_robots = copts.respect_robots | |
| 136 | + settings = get_settings() | |
| 137 | + budget = Budget(max_pages=copts.max_pages, max_duration_s=copts.max_duration_s) | |
| 138 | + limiter = HostLimiter( | |
| 139 | + per_host=min(copts.concurrency, settings.max_concurrency_per_host), | |
| 140 | + global_limit=settings.max_concurrency_global, | |
| 141 | + default_delay_s=copts.delay_ms / 1000, | |
| 142 | + ) | |
| 143 | + deduper = ContentDeduper() | |
| 144 | + frontier = Frontier(copts.strategy, copts.search) | |
| 145 | + if req.get("_frontier"): | |
| 146 | + frontier.load(req["_frontier"]) | |
| 147 | + log.info("crawl.resume", job_id=jid, pending=len(frontier)) | |
| 148 | + # pages déjà traitées | |
| 149 | + for u, _s, _d in await store.page_urls(jid): | |
| 150 | + frontier.seen.add(u) | |
| 151 | + else: | |
| 152 | + frontier.push(root, 0) | |
| 153 | + if not copts.ignore_sitemap: | |
| 154 | + try: | |
| 155 | + mapped = await asyncio.wait_for( | |
| 156 | + map_site( | |
| 157 | + root, | |
| 158 | + MapOptions( | |
| 159 | + limit=min(copts.max_pages * 3, 5000), | |
| 160 | + include_subdomains=copts.allow_subdomains, | |
| 161 | + timeout_s=15, | |
| 162 | + crawl_depth=0, | |
| 163 | + ), | |
| 164 | + ), | |
| 165 | + timeout=25, | |
| 166 | + ) | |
| 167 | + added = 0 | |
| 168 | + for m in mapped: | |
| 169 | + if ( | |
| 170 | + "sitemap" in m.sources | |
| 171 | + and _match_paths(m.url, copts.include_paths, copts.exclude_paths) | |
| 172 | + and frontier.push(m.url, 1) | |
| 173 | + ): | |
| 174 | + added += 1 | |
| 175 | + log.info("crawl.sitemap_seeded", job_id=jid, added=added) | |
| 176 | + except Exception as e: | |
| 177 | + log.info("crawl.sitemap_skip", job_id=jid, error=str(e)) | |
| 178 | + if copts.respect_robots: | |
| 179 | + d = await robots_cache.crawl_delay(root) | |
| 180 | + if d: | |
| 181 | + limiter.set_delay(host_of(root), min(d, 10.0)) | |
| 182 | + | |
| 183 | + await store.set_total(jid, min(copts.max_pages, len(frontier) + 1)) | |
| 184 | + bus.publish(jid, "progress", {"queued": len(frontier), "completed": 0}) | |
| 185 | + last_save = time.monotonic() | |
| 186 | + cancelled = False | |
| 187 | + | |
| 188 | + async def one(url: str, depth: int) -> None: | |
| 189 | + host = host_of(url) | |
| 190 | + async with limiter.slot(host): | |
| 191 | + page = await scrape(url, sopts, depth=depth, job_id=jid) | |
| 192 | + if page.status == "ok" and page.markdown: | |
| 193 | + dup = deduper.is_duplicate(page.markdown, page.final_url) | |
| 194 | + if dup: | |
| 195 | + page = PageResult.skipped( | |
| 196 | + url, | |
| 197 | + f"doublon de {dup}", | |
| 198 | + depth=depth, | |
| 199 | + fetch_mode_used=page.fetch_mode_used, | |
| 200 | + http_status=page.http_status, | |
| 201 | + ) | |
| 202 | + page.final_url = url | |
| 203 | + # découverte | |
| 204 | + if page.status == "ok" and depth < copts.max_depth: | |
| 205 | + for lk in page.links: | |
| 206 | + if lk.kind == "asset" or lk.kind in ("mailto", "tel", "other") or lk.nofollow: | |
| 207 | + continue | |
| 208 | + if lk.kind == "external" and not copts.allow_external_links: | |
| 209 | + continue | |
| 210 | + if lk.kind == "internal" and not same_site(root, lk.href, copts.allow_subdomains): | |
| 211 | + continue | |
| 212 | + if not _match_paths(lk.href, copts.include_paths, copts.exclude_paths): | |
| 213 | + continue | |
| 214 | + if budget.max_pages and len(frontier.seen) >= copts.max_pages * 5: | |
| 215 | + break | |
| 216 | + frontier.push(lk.href, depth + 1) | |
| 217 | + if "links" not in sopts.formats: | |
| 218 | + page.links = [] | |
| 219 | + budget.consume_page() | |
| 220 | + await store.add_page(jid, page) | |
| 221 | + bus.publish(jid, "page", _page_event(page)) | |
| 222 | + | |
| 223 | + try: | |
| 224 | + while len(frontier) and budget.can_fetch_page(): | |
| 225 | + if await store.status_of(jid) == "cancelled": | |
| 226 | + cancelled = True | |
| 227 | + break | |
| 228 | + remaining = (budget.max_pages or 10**9) - budget.pages | |
| 229 | + batch = frontier.pop_batch(min(copts.concurrency, remaining)) | |
| 230 | + await asyncio.gather(*(one(u, d) for u, d in batch)) | |
| 231 | + j = await store.get_job(jid) | |
| 232 | + if j: | |
| 233 | + total = min(copts.max_pages, j.completed + j.failed + j.skipped + len(frontier)) | |
| 234 | + await store.set_total(jid, total) | |
| 235 | + bus.publish( | |
| 236 | + jid, | |
| 237 | + "progress", | |
| 238 | + { | |
| 239 | + "queued": len(frontier), | |
| 240 | + "completed": j.completed, | |
| 241 | + "failed": j.failed, | |
| 242 | + "skipped": j.skipped, | |
| 243 | + "total": total, | |
| 244 | + "elapsed_s": round(budget.elapsed(), 1), | |
| 245 | + }, | |
| 246 | + ) | |
| 247 | + if time.monotonic() - last_save > 5: | |
| 248 | + await store.save_frontier(jid, frontier.dump()) | |
| 249 | + last_save = time.monotonic() | |
| 250 | + if cancelled: | |
| 251 | + await store.set_status(jid, "cancelled") | |
| 252 | + bus.publish(jid, "done", {"status": "cancelled"}) | |
| 253 | + else: | |
| 254 | + meta = { | |
| 255 | + "pages_seen": len(frontier.seen), | |
| 256 | + "pending": len(frontier), | |
| 257 | + "elapsed_s": round(budget.elapsed(), 1), | |
| 258 | + } | |
| 259 | + if not budget.can_fetch_page() and len(frontier): | |
| 260 | + meta["stopped_reason"] = budget.why_exhausted() | |
| 261 | + await store.set_status(jid, "completed", meta=meta) | |
| 262 | + bus.publish(jid, "done", {"status": "completed", **meta}) | |
| 263 | + except Exception as e: | |
| 264 | + log.error("crawl.crashed", job_id=jid, error=f"{type(e).__name__}: {e}") | |
| 265 | + await store.set_status( | |
| 266 | + jid, "failed", error=ErrorInfo.make(ErrorCode.INTERNAL, f"{type(e).__name__}: {e}") | |
| 267 | + ) | |
| 268 | + bus.publish(jid, "error", {"code": "INTERNAL", "message": str(e)}) | |
| 269 | + finally: | |
| 270 | + await store.save_frontier(jid, {"pending": [], "seen": []}) | |
| 271 | + bus.close(jid) | |
| 272 | + await _webhook(jid, req.get("_webhook"), store) | |
| 273 | + | |
| 274 | + | |
| 275 | +async def run_batch(jid: str, req: dict[str, Any], store: Store) -> None: | |
| 276 | + urls = [normalize_url(u) or u for u in req.get("urls", [])] | |
| 277 | + sopts = ScrapeOptions.model_validate( | |
| 278 | + {k: v for k, v in req.items() if k not in ("urls", "_webhook", "_frontier", "concurrency")} | |
| 279 | + ) | |
| 280 | + settings = get_settings() | |
| 281 | + limiter = HostLimiter( | |
| 282 | + per_host=settings.max_concurrency_per_host, global_limit=settings.max_concurrency_global | |
| 283 | + ) | |
| 284 | + conc = min(int(req.get("concurrency") or 8), settings.max_concurrency_global) | |
| 285 | + sem = asyncio.Semaphore(conc) | |
| 286 | + done_urls = {u for u, _s, _d in await store.page_urls(jid)} | |
| 287 | + await store.set_total(jid, len(urls)) | |
| 288 | + | |
| 289 | + async def one(u: str) -> None: | |
| 290 | + if u in done_urls: | |
| 291 | + return | |
| 292 | + async with sem, limiter.slot(host_of(u)): | |
| 293 | + page = await scrape(u, sopts, job_id=jid) | |
| 294 | + if "links" not in sopts.formats: | |
| 295 | + page.links = [] | |
| 296 | + await store.add_page(jid, page) | |
| 297 | + bus.publish(jid, "page", _page_event(page)) | |
| 298 | + | |
| 299 | + try: | |
| 300 | + tasks = [asyncio.create_task(one(u)) for u in urls] | |
| 301 | + for t in asyncio.as_completed(tasks): | |
| 302 | + await t | |
| 303 | + if await store.status_of(jid) == "cancelled": | |
| 304 | + for x in tasks: | |
| 305 | + x.cancel() | |
| 306 | + await store.set_status(jid, "cancelled") | |
| 307 | + bus.publish(jid, "done", {"status": "cancelled"}) | |
| 308 | + return | |
| 309 | + await store.set_status(jid, "completed") | |
| 310 | + bus.publish(jid, "done", {"status": "completed"}) | |
| 311 | + except Exception as e: | |
| 312 | + await store.set_status(jid, "failed", error=ErrorInfo.make(ErrorCode.INTERNAL, str(e))) | |
| 313 | + bus.publish(jid, "error", {"code": "INTERNAL", "message": str(e)}) | |
| 314 | + finally: | |
| 315 | + bus.close(jid) | |
| 316 | + await _webhook(jid, req.get("_webhook"), store) | |
| 317 | + | |
| 318 | + | |
| 319 | +async def resolve_extract_urls(req: dict[str, Any]) -> list[str]: | |
| 320 | + urls = list(req.get("urls") or []) | |
| 321 | + pattern = req.get("pattern") | |
| 322 | + if pattern: | |
| 323 | + base = pattern.split("*")[0] | |
| 324 | + root = normalize_url(base) or base | |
| 325 | + mapped = await map_site(root, MapOptions(limit=int(req.get("limit") or 200), timeout_s=20)) | |
| 326 | + urls += [m.url for m in mapped if fnmatch.fnmatch(m.url, pattern) or m.url.startswith(base)] | |
| 327 | + return list(dict.fromkeys(u for u in urls if u))[: int(req.get("limit") or 200)] | |
| 328 | + | |
| 329 | + | |
| 330 | +async def run_extract( | |
| 331 | + jid: str, req: dict[str, Any], store: Store, urls: list[str] | None = None | |
| 332 | +) -> dict[str, Any]: | |
| 333 | + urls = urls or await resolve_extract_urls(req) | |
| 334 | + from trawls.models import CssField, ExtractOptions | |
| 335 | + | |
| 336 | + eo = ExtractOptions( | |
| 337 | + mode=req.get("mode", "css" if req.get("css") else "llm"), | |
| 338 | + schema=req.get("schema"), | |
| 339 | + css={k: CssField.model_validate(v) for k, v in (req.get("css") or {}).items()} or None, | |
| 340 | + prompt=req.get("prompt"), | |
| 341 | + ) | |
| 342 | + sopts = ScrapeOptions.model_validate(req.get("scrape") or {}) | |
| 343 | + sopts.extract = eo | |
| 344 | + sopts.formats = ["markdown"] | |
| 345 | + await store.set_total(jid, len(urls)) | |
| 346 | + sem = asyncio.Semaphore(6) | |
| 347 | + results: list[dict[str, Any]] = [] | |
| 348 | + per_url: list[dict[str, Any]] = [] | |
| 349 | + | |
| 350 | + async def one(u: str) -> None: | |
| 351 | + async with sem: | |
| 352 | + page = await scrape(u, sopts, job_id=jid) | |
| 353 | + page.links = [] | |
| 354 | + page.markdown = None | |
| 355 | + await store.add_page(jid, page) | |
| 356 | + bus.publish(jid, "page", _page_event(page)) | |
| 357 | + d = (page.json_data or {}).get("data") if page.json_data else None | |
| 358 | + per_url.append( | |
| 359 | + { | |
| 360 | + "url": u, | |
| 361 | + "status": page.status, | |
| 362 | + "data": d, | |
| 363 | + "errors": (page.json_data or {}).get("errors") if page.json_data else None, | |
| 364 | + "error": page.error.model_dump() if page.error else None, | |
| 365 | + } | |
| 366 | + ) | |
| 367 | + if isinstance(d, dict): | |
| 368 | + results.append(d) | |
| 369 | + | |
| 370 | + try: | |
| 371 | + await asyncio.gather(*(one(u) for u in urls)) | |
| 372 | + merged = merge_results(results, key=req.get("merge_key")) | |
| 373 | + meta = {"merged": merged, "per_url": per_url} | |
| 374 | + await store.set_status(jid, "completed", meta=meta) | |
| 375 | + bus.publish(jid, "done", {"status": "completed"}) | |
| 376 | + return meta | |
| 377 | + except Exception as e: | |
| 378 | + await store.set_status(jid, "failed", error=ErrorInfo.make(ErrorCode.INTERNAL, str(e))) | |
| 379 | + bus.publish(jid, "error", {"code": "INTERNAL", "message": str(e)}) | |
| 380 | + return {"per_url": per_url} | |
| 381 | + finally: | |
| 382 | + bus.close(jid) | |
| 383 | + await _webhook(jid, req.get("_webhook"), store) | |
| 384 | + | |
| 385 | + | |
| 386 | +def _page_event(page: PageResult) -> dict[str, Any]: | |
| 387 | + return { | |
| 388 | + "url": page.url, | |
| 389 | + "final_url": page.final_url, | |
| 390 | + "status": page.status, | |
| 391 | + "http_status": page.http_status, | |
| 392 | + "mode": page.fetch_mode_used, | |
| 393 | + "depth": page.depth, | |
| 394 | + "title": page.metadata.title, | |
| 395 | + "words": page.metadata.word_count, | |
| 396 | + "error": page.error.model_dump() if page.error else None, | |
| 397 | + "ms": page.timings.total_ms, | |
| 398 | + } | |
| 399 | + | |
| 400 | + | |
| 401 | +async def _webhook(jid: str, url: str | None, store: Store) -> None: | |
| 402 | + if not url: | |
| 403 | + return | |
| 404 | + job = await store.get_job(jid) | |
| 405 | + if not job: | |
| 406 | + return | |
| 407 | + body = json.dumps({"job": job.model_dump(mode="json")}, default=str).encode() | |
| 408 | + secret = (get_settings().admin_key or "trawls").encode() | |
| 409 | + sig = hmac.new(secret, body, hashlib.sha256).hexdigest() | |
| 410 | + async with httpx.AsyncClient(follow_redirects=False, timeout=15) as c: | |
| 411 | + for attempt in range(3): | |
| 412 | + try: | |
| 413 | + r = await c.post( | |
| 414 | + url, | |
| 415 | + content=body, | |
| 416 | + headers={ | |
| 417 | + "Content-Type": "application/json", | |
| 418 | + "X-Trawls-Signature": f"sha256={sig}", | |
| 419 | + "X-Trawls-Job": jid, | |
| 420 | + }, | |
| 421 | + ) | |
| 422 | + if r.status_code < 400: | |
| 423 | + return | |
| 424 | + except Exception as e: | |
| 425 | + log.info("webhook.failed", job_id=jid, attempt=attempt + 1, error=str(e)) | |
| 426 | + await asyncio.sleep(2 * (attempt + 1)) | |
| 427 | + | |
| 428 | + | |
| 429 | +class Worker: | |
| 430 | + """Boucle en processus : prend les jobs `queued`, les exécute avec `worker_concurrency` en parallèle.""" | |
| 431 | + | |
| 432 | + def __init__(self, store: Store | None = None) -> None: | |
| 433 | + self.store = store or get_store() | |
| 434 | + self._tasks: set[asyncio.Task[None]] = set() | |
| 435 | + self._stop = asyncio.Event() | |
| 436 | + self.concurrency = get_settings().worker_concurrency | |
| 437 | + | |
| 438 | + async def _run_one(self, jid: str) -> None: | |
| 439 | + req = await self.store.get_job_request(jid) | |
| 440 | + job = await self.store.get_job(jid) | |
| 441 | + if not req or not job: | |
| 442 | + return | |
| 443 | + log.info("job.start", job_id=jid, kind=job.kind) | |
| 444 | + bus.publish(jid, "status", {"status": "running"}) | |
| 445 | + try: | |
| 446 | + if job.kind == "crawl": | |
| 447 | + await run_crawl(jid, req, self.store) | |
| 448 | + elif job.kind == "batch": | |
| 449 | + await run_batch(jid, req, self.store) | |
| 450 | + elif job.kind == "extract": | |
| 451 | + await run_extract(jid, req, self.store) | |
| 452 | + else: | |
| 453 | + await self.store.set_status( | |
| 454 | + jid, "failed", error=ErrorInfo.make(ErrorCode.INTERNAL, f"kind inconnu {job.kind}") | |
| 455 | + ) | |
| 456 | + except Exception as e: | |
| 457 | + log.error("job.crashed", job_id=jid, error=str(e)) | |
| 458 | + await self.store.set_status(jid, "failed", error=ErrorInfo.make(ErrorCode.INTERNAL, str(e))) | |
| 459 | + log.info("job.end", job_id=jid) | |
| 460 | + | |
| 461 | + async def run(self) -> None: | |
| 462 | + last_purge = 0.0 | |
| 463 | + while not self._stop.is_set(): | |
| 464 | + self._tasks = {t for t in self._tasks if not t.done()} | |
| 465 | + if len(self._tasks) < self.concurrency: | |
| 466 | + jid = await self.store.next_queued() | |
| 467 | + if jid: | |
| 468 | + t = asyncio.create_task(self._run_one(jid)) | |
| 469 | + self._tasks.add(t) | |
| 470 | + continue | |
| 471 | + if time.monotonic() - last_purge > 3600: | |
| 472 | + try: | |
| 473 | + n = await self.store.purge_expired(get_settings().result_ttl_days) | |
| 474 | + if n: | |
| 475 | + log.info("jobs.purged", n=n) | |
| 476 | + except Exception as e: | |
| 477 | + log.info("jobs.purge_failed", error=str(e)) | |
| 478 | + last_purge = time.monotonic() | |
| 479 | + try: | |
| 480 | + await asyncio.wait_for(self._stop.wait(), timeout=0.5) | |
| 481 | + except TimeoutError: | |
| 482 | + pass | |
| 483 | + | |
| 484 | + def stop(self) -> None: | |
| 485 | + self._stop.set() | |
| 486 | + | |
| 487 | + @property | |
| 488 | + def active(self) -> int: | |
| 489 | + return len([t for t in self._tasks if not t.done()]) | |
added
trawls/worker/store.py
+389 −0
@@ -0,0 +1,389 @@ | ||
| 1 | +"""Persistance des jobs et pages (SQLite via aiosqlite). Chaque page est écrite dès qu'elle est prête. | |
| 2 | + | |
| 3 | +Self-host first : aucune dépendance externe. `DATABASE_URL` PostgreSQL est réservé pour plus tard. | |
| 4 | +""" | |
| 5 | + | |
| 6 | +from __future__ import annotations | |
| 7 | + | |
| 8 | +import json | |
| 9 | +import secrets | |
| 10 | +import time | |
| 11 | +from datetime import UTC, datetime, timedelta | |
| 12 | +from typing import Any | |
| 13 | + | |
| 14 | +import aiosqlite | |
| 15 | + | |
| 16 | +from trawls.config import get_settings | |
| 17 | +from trawls.models import ErrorInfo, JobKind, JobStatus, JobSummary, PageResult | |
| 18 | + | |
| 19 | +_SCHEMA = """ | |
| 20 | +CREATE TABLE IF NOT EXISTS jobs ( | |
| 21 | + id TEXT PRIMARY KEY, kind TEXT NOT NULL, status TEXT NOT NULL, | |
| 22 | + created_at TEXT NOT NULL, started_at TEXT, finished_at TEXT, | |
| 23 | + total INTEGER DEFAULT 0, completed INTEGER DEFAULT 0, failed INTEGER DEFAULT 0, skipped INTEGER DEFAULT 0, | |
| 24 | + root_url TEXT, request TEXT NOT NULL, error TEXT, credits INTEGER DEFAULT 0, meta TEXT DEFAULT '{}', | |
| 25 | + api_key_id TEXT, idempotency_key TEXT, webhook TEXT, frontier TEXT | |
| 26 | +); | |
| 27 | +CREATE INDEX IF NOT EXISTS jobs_created ON jobs(created_at DESC); | |
| 28 | +CREATE UNIQUE INDEX IF NOT EXISTS jobs_idem ON jobs(api_key_id, idempotency_key) WHERE idempotency_key IS NOT NULL; | |
| 29 | +CREATE TABLE IF NOT EXISTS pages ( | |
| 30 | + seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT NOT NULL, url TEXT NOT NULL, status TEXT NOT NULL, | |
| 31 | + depth INTEGER DEFAULT 0, created_at TEXT NOT NULL, result TEXT NOT NULL | |
| 32 | +); | |
| 33 | +CREATE INDEX IF NOT EXISTS pages_job ON pages(job_id, seq); | |
| 34 | +CREATE TABLE IF NOT EXISTS api_keys ( | |
| 35 | + id TEXT PRIMARY KEY, name TEXT, hash TEXT NOT NULL, prefix TEXT NOT NULL, created_at TEXT NOT NULL, | |
| 36 | + quota_per_day INTEGER, disabled INTEGER DEFAULT 0, last_used TEXT | |
| 37 | +); | |
| 38 | +CREATE TABLE IF NOT EXISTS usage ( | |
| 39 | + day TEXT NOT NULL, api_key_id TEXT NOT NULL, endpoint TEXT NOT NULL, count INTEGER DEFAULT 0, credits INTEGER DEFAULT 0, | |
| 40 | + PRIMARY KEY (day, api_key_id, endpoint) | |
| 41 | +); | |
| 42 | +CREATE TABLE IF NOT EXISTS request_log ( | |
| 43 | + seq INTEGER PRIMARY KEY AUTOINCREMENT, at TEXT NOT NULL, api_key_id TEXT, method TEXT, path TEXT, status INTEGER, ms REAL, ip TEXT | |
| 44 | +); | |
| 45 | +""" | |
| 46 | + | |
| 47 | + | |
| 48 | +def _now() -> str: | |
| 49 | + return datetime.now(UTC).isoformat() | |
| 50 | + | |
| 51 | + | |
| 52 | +def new_id(prefix: str = "job") -> str: | |
| 53 | + return f"{prefix}_{secrets.token_urlsafe(12)}" | |
| 54 | + | |
| 55 | + | |
| 56 | +class Store: | |
| 57 | + def __init__(self, path: str | None = None) -> None: | |
| 58 | + self.path = path or str(get_settings().db_path) | |
| 59 | + self._db: aiosqlite.Connection | None = None | |
| 60 | + | |
| 61 | + async def open(self) -> None: | |
| 62 | + self._db = await aiosqlite.connect(self.path) | |
| 63 | + self._db.row_factory = aiosqlite.Row | |
| 64 | + await self._db.execute("PRAGMA journal_mode=WAL") | |
| 65 | + await self._db.execute("PRAGMA synchronous=NORMAL") | |
| 66 | + await self._db.execute("PRAGMA busy_timeout=5000") | |
| 67 | + await self._db.executescript(_SCHEMA) | |
| 68 | + await self._db.commit() | |
| 69 | + # jobs laissés "running" par un crash → requeued (reprise) | |
| 70 | + await self._db.execute("UPDATE jobs SET status='queued' WHERE status='running'") | |
| 71 | + await self._db.commit() | |
| 72 | + | |
| 73 | + async def close(self) -> None: | |
| 74 | + if self._db: | |
| 75 | + await self._db.close() | |
| 76 | + self._db = None | |
| 77 | + | |
| 78 | + @property | |
| 79 | + def db(self) -> aiosqlite.Connection: | |
| 80 | + assert self._db is not None, "store non ouvert" | |
| 81 | + return self._db | |
| 82 | + | |
| 83 | + # ---- jobs ----------------------------------------------------------------- | |
| 84 | + | |
| 85 | + async def create_job( | |
| 86 | + self, | |
| 87 | + kind: JobKind, | |
| 88 | + request: dict[str, Any], | |
| 89 | + root_url: str | None, | |
| 90 | + api_key_id: str | None = None, | |
| 91 | + idempotency_key: str | None = None, | |
| 92 | + webhook: str | None = None, | |
| 93 | + total: int = 0, | |
| 94 | + ) -> JobSummary: | |
| 95 | + if idempotency_key: | |
| 96 | + cur = await self.db.execute( | |
| 97 | + "SELECT id FROM jobs WHERE api_key_id IS ? AND idempotency_key=?", | |
| 98 | + (api_key_id, idempotency_key), | |
| 99 | + ) | |
| 100 | + row = await cur.fetchone() | |
| 101 | + if row: | |
| 102 | + j = await self.get_job(row["id"]) | |
| 103 | + assert j is not None | |
| 104 | + return j | |
| 105 | + jid = new_id() | |
| 106 | + now = _now() | |
| 107 | + await self.db.execute( | |
| 108 | + "INSERT INTO jobs(id,kind,status,created_at,total,root_url,request,api_key_id,idempotency_key,webhook) VALUES(?,?,?,?,?,?,?,?,?,?)", | |
| 109 | + ( | |
| 110 | + jid, | |
| 111 | + kind, | |
| 112 | + "queued", | |
| 113 | + now, | |
| 114 | + total, | |
| 115 | + root_url, | |
| 116 | + json.dumps(request, default=str), | |
| 117 | + api_key_id, | |
| 118 | + idempotency_key, | |
| 119 | + webhook, | |
| 120 | + ), | |
| 121 | + ) | |
| 122 | + await self.db.commit() | |
| 123 | + return JobSummary( | |
| 124 | + id=jid, | |
| 125 | + kind=kind, | |
| 126 | + status="queued", | |
| 127 | + created_at=datetime.fromisoformat(now), | |
| 128 | + root_url=root_url, | |
| 129 | + total=total, | |
| 130 | + ) | |
| 131 | + | |
| 132 | + async def get_job(self, jid: str) -> JobSummary | None: | |
| 133 | + cur = await self.db.execute("SELECT * FROM jobs WHERE id=?", (jid,)) | |
| 134 | + row = await cur.fetchone() | |
| 135 | + return self._row_to_job(row) if row else None | |
| 136 | + | |
| 137 | + async def get_job_request(self, jid: str) -> dict[str, Any] | None: | |
| 138 | + cur = await self.db.execute("SELECT request, webhook, frontier FROM jobs WHERE id=?", (jid,)) | |
| 139 | + row = await cur.fetchone() | |
| 140 | + if not row: | |
| 141 | + return None | |
| 142 | + d = json.loads(row["request"]) | |
| 143 | + d["_webhook"] = row["webhook"] | |
| 144 | + d["_frontier"] = json.loads(row["frontier"]) if row["frontier"] else None | |
| 145 | + return d | |
| 146 | + | |
| 147 | + def _row_to_job(self, row: aiosqlite.Row) -> JobSummary: | |
| 148 | + return JobSummary( | |
| 149 | + id=row["id"], | |
| 150 | + kind=row["kind"], | |
| 151 | + status=row["status"], | |
| 152 | + created_at=datetime.fromisoformat(row["created_at"]), | |
| 153 | + started_at=datetime.fromisoformat(row["started_at"]) if row["started_at"] else None, | |
| 154 | + finished_at=datetime.fromisoformat(row["finished_at"]) if row["finished_at"] else None, | |
| 155 | + total=row["total"] or 0, | |
| 156 | + completed=row["completed"] or 0, | |
| 157 | + failed=row["failed"] or 0, | |
| 158 | + skipped=row["skipped"] or 0, | |
| 159 | + root_url=row["root_url"], | |
| 160 | + error=ErrorInfo.model_validate_json(row["error"]) if row["error"] else None, | |
| 161 | + credits_used=row["credits"] or 0, | |
| 162 | + meta=json.loads(row["meta"] or "{}"), | |
| 163 | + ) | |
| 164 | + | |
| 165 | + async def list_jobs( | |
| 166 | + self, | |
| 167 | + limit: int = 50, | |
| 168 | + kind: str | None = None, | |
| 169 | + api_key_id: str | None = None, | |
| 170 | + status: str | None = None, | |
| 171 | + ) -> list[JobSummary]: | |
| 172 | + q = "SELECT * FROM jobs WHERE 1=1" | |
| 173 | + args: list[Any] = [] | |
| 174 | + if kind: | |
| 175 | + q += " AND kind=?" | |
| 176 | + args.append(kind) | |
| 177 | + if status: | |
| 178 | + q += " AND status=?" | |
| 179 | + args.append(status) | |
| 180 | + if api_key_id: | |
| 181 | + q += " AND api_key_id=?" | |
| 182 | + args.append(api_key_id) | |
| 183 | + q += " ORDER BY created_at DESC LIMIT ?" | |
| 184 | + args.append(limit) | |
| 185 | + cur = await self.db.execute(q, args) | |
| 186 | + return [self._row_to_job(r) for r in await cur.fetchall()] | |
| 187 | + | |
| 188 | + async def next_queued(self) -> str | None: | |
| 189 | + cur = await self.db.execute("SELECT id FROM jobs WHERE status='queued' ORDER BY created_at LIMIT 1") | |
| 190 | + row = await cur.fetchone() | |
| 191 | + if not row: | |
| 192 | + return None | |
| 193 | + res = await self.db.execute( | |
| 194 | + "UPDATE jobs SET status='running', started_at=COALESCE(started_at, ?) WHERE id=? AND status='queued'", | |
| 195 | + (_now(), row["id"]), | |
| 196 | + ) | |
| 197 | + await self.db.commit() | |
| 198 | + return row["id"] if res.rowcount else None | |
| 199 | + | |
| 200 | + async def set_status( | |
| 201 | + self, jid: str, status: JobStatus, error: ErrorInfo | None = None, meta: dict[str, Any] | None = None | |
| 202 | + ) -> None: | |
| 203 | + sets = ["status=?"] | |
| 204 | + args: list[Any] = [status] | |
| 205 | + if status in ("completed", "failed", "cancelled"): | |
| 206 | + sets.append("finished_at=?") | |
| 207 | + args.append(_now()) | |
| 208 | + if error is not None: | |
| 209 | + sets.append("error=?") | |
| 210 | + args.append(error.model_dump_json()) | |
| 211 | + if meta is not None: | |
| 212 | + sets.append("meta=?") | |
| 213 | + args.append(json.dumps(meta, default=str)) | |
| 214 | + args.append(jid) | |
| 215 | + await self.db.execute(f"UPDATE jobs SET {', '.join(sets)} WHERE id=?", args) | |
| 216 | + await self.db.commit() | |
| 217 | + | |
| 218 | + async def status_of(self, jid: str) -> str | None: | |
| 219 | + cur = await self.db.execute("SELECT status FROM jobs WHERE id=?", (jid,)) | |
| 220 | + row = await cur.fetchone() | |
| 221 | + return row["status"] if row else None | |
| 222 | + | |
| 223 | + async def set_total(self, jid: str, total: int) -> None: | |
| 224 | + await self.db.execute("UPDATE jobs SET total=? WHERE id=?", (total, jid)) | |
| 225 | + await self.db.commit() | |
| 226 | + | |
| 227 | + async def save_frontier(self, jid: str, frontier: dict[str, Any]) -> None: | |
| 228 | + await self.db.execute("UPDATE jobs SET frontier=? WHERE id=?", (json.dumps(frontier), jid)) | |
| 229 | + await self.db.commit() | |
| 230 | + | |
| 231 | + async def add_page(self, jid: str, page: PageResult, credits: int = 1) -> int: | |
| 232 | + col = {"ok": "completed", "failed": "failed", "skipped": "skipped"}[page.status] | |
| 233 | + await self.db.execute( | |
| 234 | + "INSERT INTO pages(job_id,url,status,depth,created_at,result) VALUES(?,?,?,?,?,?)", | |
| 235 | + (jid, page.url, page.status, page.depth, _now(), page.model_dump_json()), | |
| 236 | + ) | |
| 237 | + await self.db.execute( | |
| 238 | + f"UPDATE jobs SET {col}={col}+1, credits=credits+? WHERE id=?", | |
| 239 | + (credits if page.status == "ok" else 0, jid), | |
| 240 | + ) | |
| 241 | + await self.db.commit() | |
| 242 | + cur = await self.db.execute("SELECT last_insert_rowid() AS s") | |
| 243 | + row = await cur.fetchone() | |
| 244 | + return int(row["s"]) if row else 0 | |
| 245 | + | |
| 246 | + async def pages( | |
| 247 | + self, jid: str, after: int = 0, limit: int = 100, status: str | None = None | |
| 248 | + ) -> tuple[list[PageResult], int | None]: | |
| 249 | + q = "SELECT seq, result FROM pages WHERE job_id=? AND seq>?" | |
| 250 | + args: list[Any] = [jid, after] | |
| 251 | + if status: | |
| 252 | + q += " AND status=?" | |
| 253 | + args.append(status) | |
| 254 | + q += " ORDER BY seq LIMIT ?" | |
| 255 | + args.append(limit + 1) | |
| 256 | + cur = await self.db.execute(q, args) | |
| 257 | + rows = await cur.fetchall() | |
| 258 | + more = len(rows) > limit | |
| 259 | + rows = rows[:limit] | |
| 260 | + out = [PageResult.model_validate_json(r["result"]) for r in rows] | |
| 261 | + return out, (int(rows[-1]["seq"]) if more and rows else None) | |
| 262 | + | |
| 263 | + async def page_urls(self, jid: str) -> list[tuple[str, str, int]]: | |
| 264 | + cur = await self.db.execute( | |
| 265 | + "SELECT url, status, depth FROM pages WHERE job_id=? ORDER BY seq", (jid,) | |
| 266 | + ) | |
| 267 | + return [(r["url"], r["status"], r["depth"]) for r in await cur.fetchall()] | |
| 268 | + | |
| 269 | + async def delete_job(self, jid: str) -> None: | |
| 270 | + await self.db.execute("DELETE FROM pages WHERE job_id=?", (jid,)) | |
| 271 | + await self.db.execute("DELETE FROM jobs WHERE id=?", (jid,)) | |
| 272 | + await self.db.commit() | |
| 273 | + | |
| 274 | + async def purge_expired(self, ttl_days: int) -> int: | |
| 275 | + cutoff = (datetime.now(UTC) - timedelta(days=ttl_days)).isoformat() | |
| 276 | + cur = await self.db.execute( | |
| 277 | + "SELECT id FROM jobs WHERE created_at<? AND status IN ('completed','failed','cancelled')", | |
| 278 | + (cutoff,), | |
| 279 | + ) | |
| 280 | + ids = [r["id"] for r in await cur.fetchall()] | |
| 281 | + for jid in ids: | |
| 282 | + await self.delete_job(jid) | |
| 283 | + return len(ids) | |
| 284 | + | |
| 285 | + # ---- clés API / usage ---------------------------------------------------- | |
| 286 | + | |
| 287 | + async def create_key(self, name: str, quota_per_day: int | None = None) -> tuple[str, str]: | |
| 288 | + from argon2 import PasswordHasher | |
| 289 | + | |
| 290 | + raw = "trw_" + secrets.token_urlsafe(32) | |
| 291 | + kid = new_id("key") | |
| 292 | + ph = PasswordHasher() | |
| 293 | + await self.db.execute( | |
| 294 | + "INSERT INTO api_keys(id,name,hash,prefix,created_at,quota_per_day) VALUES(?,?,?,?,?,?)", | |
| 295 | + (kid, name, ph.hash(raw), raw[:12], _now(), quota_per_day), | |
| 296 | + ) | |
| 297 | + await self.db.commit() | |
| 298 | + return kid, raw | |
| 299 | + | |
| 300 | + async def verify_key(self, raw: str) -> dict[str, Any] | None: | |
| 301 | + from argon2 import PasswordHasher | |
| 302 | + from argon2.exceptions import VerifyMismatchError | |
| 303 | + | |
| 304 | + cur = await self.db.execute("SELECT * FROM api_keys WHERE prefix=? AND disabled=0", (raw[:12],)) | |
| 305 | + ph = PasswordHasher() | |
| 306 | + for row in await cur.fetchall(): | |
| 307 | + try: | |
| 308 | + ph.verify(row["hash"], raw) | |
| 309 | + await self.db.execute("UPDATE api_keys SET last_used=? WHERE id=?", (_now(), row["id"])) | |
| 310 | + await self.db.commit() | |
| 311 | + return dict(row) | |
| 312 | + except VerifyMismatchError: | |
| 313 | + continue | |
| 314 | + return None | |
| 315 | + | |
| 316 | + async def list_keys(self) -> list[dict[str, Any]]: | |
| 317 | + cur = await self.db.execute( | |
| 318 | + "SELECT id,name,prefix,created_at,quota_per_day,disabled,last_used FROM api_keys ORDER BY created_at DESC" | |
| 319 | + ) | |
| 320 | + return [dict(r) for r in await cur.fetchall()] | |
| 321 | + | |
| 322 | + async def revoke_key(self, kid: str) -> None: | |
| 323 | + await self.db.execute("UPDATE api_keys SET disabled=1 WHERE id=?", (kid,)) | |
| 324 | + await self.db.commit() | |
| 325 | + | |
| 326 | + async def record_usage(self, api_key_id: str | None, endpoint: str, credits: int = 1) -> None: | |
| 327 | + day = datetime.now(UTC).strftime("%Y-%m-%d") | |
| 328 | + await self.db.execute( | |
| 329 | + "INSERT INTO usage(day,api_key_id,endpoint,count,credits) VALUES(?,?,?,1,?) ON CONFLICT(day,api_key_id,endpoint) DO UPDATE SET count=count+1, credits=credits+excluded.credits", | |
| 330 | + (day, api_key_id or "anon", endpoint, credits), | |
| 331 | + ) | |
| 332 | + await self.db.commit() | |
| 333 | + | |
| 334 | + async def usage(self, api_key_id: str | None, days: int = 30) -> list[dict[str, Any]]: | |
| 335 | + since = (datetime.now(UTC) - timedelta(days=days)).strftime("%Y-%m-%d") | |
| 336 | + if api_key_id: | |
| 337 | + cur = await self.db.execute( | |
| 338 | + "SELECT day, endpoint, count, credits FROM usage WHERE api_key_id=? AND day>=? ORDER BY day", | |
| 339 | + (api_key_id, since), | |
| 340 | + ) | |
| 341 | + else: | |
| 342 | + cur = await self.db.execute( | |
| 343 | + "SELECT day, endpoint, SUM(count) AS count, SUM(credits) AS credits FROM usage WHERE day>=? GROUP BY day, endpoint ORDER BY day", | |
| 344 | + (since,), | |
| 345 | + ) | |
| 346 | + return [dict(r) for r in await cur.fetchall()] | |
| 347 | + | |
| 348 | + async def usage_today(self, api_key_id: str) -> int: | |
| 349 | + day = datetime.now(UTC).strftime("%Y-%m-%d") | |
| 350 | + cur = await self.db.execute( | |
| 351 | + "SELECT COALESCE(SUM(credits),0) AS c FROM usage WHERE api_key_id=? AND day=?", (api_key_id, day) | |
| 352 | + ) | |
| 353 | + row = await cur.fetchone() | |
| 354 | + return int(row["c"]) if row else 0 | |
| 355 | + | |
| 356 | + async def log_request( | |
| 357 | + self, api_key_id: str | None, method: str, path: str, status: int, ms: float, ip: str | None | |
| 358 | + ) -> None: | |
| 359 | + await self.db.execute( | |
| 360 | + "INSERT INTO request_log(at,api_key_id,method,path,status,ms,ip) VALUES(?,?,?,?,?,?,?)", | |
| 361 | + (_now(), api_key_id, method, path, status, round(ms, 1), ip), | |
| 362 | + ) | |
| 363 | + if int(time.time()) % 50 == 0: | |
| 364 | + await self.db.execute( | |
| 365 | + "DELETE FROM request_log WHERE seq < (SELECT MAX(seq) FROM request_log) - 5000" | |
| 366 | + ) | |
| 367 | + await self.db.commit() | |
| 368 | + | |
| 369 | + async def recent_requests(self, limit: int = 100) -> list[dict[str, Any]]: | |
| 370 | + cur = await self.db.execute("SELECT * FROM request_log ORDER BY seq DESC LIMIT ?", (limit,)) | |
| 371 | + return [dict(r) for r in await cur.fetchall()] | |
| 372 | + | |
| 373 | + async def stats(self) -> dict[str, Any]: | |
| 374 | + out: dict[str, Any] = {} | |
| 375 | + cur = await self.db.execute("SELECT status, COUNT(*) AS n FROM jobs GROUP BY status") | |
| 376 | + out["jobs"] = {r["status"]: r["n"] for r in await cur.fetchall()} | |
| 377 | + cur = await self.db.execute("SELECT status, COUNT(*) AS n FROM pages GROUP BY status") | |
| 378 | + out["pages"] = {r["status"]: r["n"] for r in await cur.fetchall()} | |
| 379 | + return out | |
| 380 | + | |
| 381 | + | |
| 382 | +_store: Store | None = None | |
| 383 | + | |
| 384 | + | |
| 385 | +def get_store() -> Store: | |
| 386 | + global _store | |
| 387 | + if _store is None: | |
| 388 | + _store = Store() | |
| 389 | + return _store | |
| 390 | ||