link checker: plain UA token and simpler Accept header (Apple CDN compatibility)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +4 −4
modified
.env.example
+1 −1
@@ -24,4 +24,4 @@ BACKUP_PASSPHRASE=CHANGE_ME | |||||
| 24 | BACKUP_REMOTE= # optional rsync target, e.g. user@host:~/backups/tools-spboucher-ai | 24 | BACKUP_REMOTE= # optional rsync target, e.g. user@host:~/backups/tools-spboucher-ai | ||
| 25 | # Worker | 25 | # Worker | ||
| 26 | WORKER_POLL_MS=5000 | 26 | WORKER_POLL_MS=5000 | ||
| 27 | −OUTBOUND_USER_AGENT=tools.spboucher.ai link-checker (+https://tools.spboucher.ai/about) | 27 | +OUTBOUND_USER_AGENT=tools-spboucher-ai-linkcheck/1.0 | ||
modified
docs/source-policy.md
+1 −1
@@ -20,7 +20,7 @@ | |||||
| 20 | Everything else is `uncertainty_note` + `confidence: medium|low`. Low-confidence records are not published. | 20 | Everything else is `uncertainty_note` + `confidence: medium|low`. Low-confidence records are not published. | ||
| 21 | 21 | ||||
| 22 | ## Crawling and copyright | 22 | ## Crawling and copyright | ||
| 23 | −- Fetches use a declared User-Agent (`OUTBOUND_USER_AGENT`), GET with 15 s timeout, 1 MiB cap, ≤ 5 redirects, 250 ms pacing, daily/weekly schedules — a handful of requests per site per week. | 23 | +- Fetches use a declared User-Agent (`OUTBOUND_USER_AGENT`), GET with 20 s timeout, 1 MiB cap, ≤ 5 redirects, 250 ms pacing, daily/weekly schedules — a handful of requests per site per week. | ||
| 24 | - `robots.txt`: the checker only fetches URLs that editors recorded as official pages (homepage, docs, pricing, license, policy). It does not crawl, follow links or index content; it stores a title, a canonical URL and a content hash, never the page body. | 24 | - `robots.txt`: the checker only fetches URLs that editors recorded as official pages (homepage, docs, pricing, license, policy). It does not crawl, follow links or index content; it stores a title, a canonical URL and a content hash, never the page body. | ||
| 25 | - Sites that block automated clients (401/403/405/429) are marked *unverifiable*, not broken, and are re-checked manually. | 25 | - Sites that block automated clients (401/403/405/429) are marked *unverifiable*, not broken, and are re-checked manually. | ||
| 26 | - Descriptions are original editorial prose; vendor text is never copied. Logos are not redistributed (monograms are generated). Product names are used nominatively. | 26 | - Descriptions are original editorial prose; vendor text is never copied. Logos are not redistributed (monograms are generated). Product names are used nominatively. | ||
modified
src/lib/url.ts
+2 −2
@@ -48,7 +48,7 @@ export async function assertPublicHost(hostname: string) { | |||||
| 48 | } | 48 | } | ||
| 49 | 49 | ||||
| 50 | export type SafeFetchResult = { ok: boolean; status: number; finalUrl: string; redirected: boolean; hops: string[]; contentType: string | null; body: string | null; error?: string; durationMs: number }; | 50 | export type SafeFetchResult = { ok: boolean; status: number; finalUrl: string; redirected: boolean; hops: string[]; contentType: string | null; body: string | null; error?: string; durationMs: number }; | ||
| 51 | −const UA = () => process.env.OUTBOUND_USER_AGENT ?? "tools.spboucher.ai link-checker"; | 51 | +const UA = () => process.env.OUTBOUND_USER_AGENT ?? "tools-spboucher-ai-linkcheck/1.0"; // plain token: some CDNs (Apple) 404 on UAs containing URLs | ||
| 52 | 52 | ||||
| 53 | /** SSRF-safe fetch: validates every hop, caps redirects (5), body size (1 MiB) and time (15 s). Never sends credentials. */ | 53 | /** SSRF-safe fetch: validates every hop, caps redirects (5), body size (1 MiB) and time (15 s). Never sends credentials. */ | ||
| 54 | export async function safeFetch(url: string, opts: { method?: "GET" | "HEAD"; maxBytes?: number; timeoutMs?: number } = {}): Promise<SafeFetchResult> { | 54 | export async function safeFetch(url: string, opts: { method?: "GET" | "HEAD"; maxBytes?: number; timeoutMs?: number } = {}): Promise<SafeFetchResult> { | ||
@@ -63,7 +63,7 @@ export async function safeFetch(url: string, opts: { method?: "GET" | "HEAD"; ma | |||||
| 63 | const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); | 63 | const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); | ||
| 64 | let res: Response; | 64 | let res: Response; | ||
| 65 | try { | 65 | try { | ||
| 66 | − res = await fetch(current, { method: opts.method ?? "GET", redirect: "manual", signal: ctrl.signal, dispatcher, headers: { "user-agent": UA(), accept: "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.5" }, credentials: "omit" as RequestCredentials } as RequestInit); | 66 | + res = await fetch(current, { method: opts.method ?? "GET", redirect: "manual", signal: ctrl.signal, dispatcher, headers: { "user-agent": UA(), accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8" }, credentials: "omit" as RequestCredentials } as RequestInit); | ||
| 67 | } finally { clearTimeout(t); } | 67 | } finally { clearTimeout(t); } | ||
| 68 | if (res.status >= 300 && res.status < 400) { | 68 | if (res.status >= 300 && res.status < 400) { | ||
| 69 | const loc = res.headers.get("location"); if (!loc) return { ok: false, status: res.status, finalUrl: current, redirected, hops, contentType: null, body: null, error: "redirect without location", durationMs: Date.now() - started }; | 69 | const loc = res.headers.get("location"); if (!loc) return { ok: false, status: res.status, finalUrl: current, redirected, hops, contentType: null, body: null, error: "redirect without location", durationMs: Date.now() - started }; | ||
| 70 | 70 | ||||