import "server-only"; import { env } from "../env"; export interface SearchResult { title: string; url: string; snippet: string; publishedAt?: string; score?: number } export async function webSearch(query: string, opts: { maxResults?: number; recencyDays?: number } = {}): Promise<{ query: string; results: SearchResult[]; provider: string } | { error: string }> { if (!env.TAVILY_API_KEY) return { error: "Web search is not configured (TAVILY_API_KEY missing)." }; const body: Record = { query, max_results: Math.min(opts.maxResults ?? 6, 10), search_depth: "advanced", include_answer: false }; if (opts.recencyDays) body.days = opts.recencyDays; const r = await fetch("https://api.tavily.com/search", { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${env.TAVILY_API_KEY}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(25_000) }); if (!r.ok) return { error: `Tavily ${r.status}: ${(await r.text()).slice(0, 200)}` }; const j = await r.json() as { results: { title: string; url: string; content: string; published_date?: string; score?: number }[] }; return { query, provider: "tavily", results: (j.results ?? []).map(x => ({ title: x.title, url: x.url, snippet: x.content?.slice(0, 600) ?? "", publishedAt: x.published_date, score: x.score })) }; } const BLOCKED_HOSTS = /^(localhost|127\.|10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|0\.|\[?::1|169\.254\.|100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.)/; export async function fetchPage(url: string): Promise<{ url: string; title?: string; text: string; truncated: boolean } | { error: string }> { let u: URL; try { u = new URL(url); } catch { return { error: "Invalid URL" }; } if (!/^https?:$/.test(u.protocol) || BLOCKED_HOSTS.test(u.hostname)) return { error: "URL not allowed (private network or non-HTTP)." }; // Prefer Tavily extract (handles JS pages, respects robots) when configured if (env.TAVILY_API_KEY) { try { const r = await fetch("https://api.tavily.com/extract", { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${env.TAVILY_API_KEY}` }, body: JSON.stringify({ urls: [url] }), signal: AbortSignal.timeout(30_000) }); if (r.ok) { const j = await r.json() as { results?: { url: string; raw_content: string }[] }; const t = j.results?.[0]?.raw_content; if (t) return { url, text: t.slice(0, 20_000), truncated: t.length > 20_000 }; } } catch { /* fall through */ } } const r = await fetch(url, { headers: { "user-agent": "Mozilla/5.0 (compatible; ChatSPBoucherAI/1.0; +https://chat.spboucher.ai)" }, redirect: "follow", signal: AbortSignal.timeout(20_000) }); if (!r.ok) return { error: `HTTP ${r.status}` }; const ct = r.headers.get("content-type") ?? ""; if (!/text\/html|text\/plain|application\/xhtml|application\/json/.test(ct)) return { error: `Unsupported content-type ${ct}` }; const html = await r.text(); const title = html.match(/]*>([^<]*)<\/title>/i)?.[1]?.trim(); const text = html.replace(/|||/gi, " ").replace(/<[^>]+>/g, " ").replace(/ /g, " ").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/\s+/g, " ").trim(); return { url, title, text: text.slice(0, 20_000), truncated: text.length > 20_000 }; }