spb/chat-spboucher-ai

1import "server-only";2import { env } from "../env";34export interface SearchResult { title: string; url: string; snippet: string; publishedAt?: string; score?: number }56export async function webSearch(query: string, opts: { maxResults?: number; recencyDays?: number } = {}): Promise<{ query: string; results: SearchResult[]; provider: string } | { error: string }> {7  if (!env.TAVILY_API_KEY) return { error: "Web search is not configured (TAVILY_API_KEY missing)." };8  const body: Record<string, unknown> = { query, max_results: Math.min(opts.maxResults ?? 6, 10), search_depth: "advanced", include_answer: false };9  if (opts.recencyDays) body.days = opts.recencyDays;10  const r = await fetch("https://api.tavily.com/search", { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${env.TAVILY_API_KEY}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(25_000) });11  if (!r.ok) return { error: `Tavily ${r.status}: ${(await r.text()).slice(0, 200)}` };12  const j = await r.json() as { results: { title: string; url: string; content: string; published_date?: string; score?: number }[] };13  return { query, provider: "tavily", results: (j.results ?? []).map(x => ({ title: x.title, url: x.url, snippet: x.content?.slice(0, 600) ?? "", publishedAt: x.published_date, score: x.score })) };14}1516const BLOCKED_HOSTS = /^(localhost|127\.|10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|0\.|\[?::1|169\.254\.|100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.)/;1718export async function fetchPage(url: string): Promise<{ url: string; title?: string; text: string; truncated: boolean } | { error: string }> {19  let u: URL;20  try { u = new URL(url); } catch { return { error: "Invalid URL" }; }21  if (!/^https?:$/.test(u.protocol) || BLOCKED_HOSTS.test(u.hostname)) return { error: "URL not allowed (private network or non-HTTP)." };22  // Prefer Tavily extract (handles JS pages, respects robots) when configured23  if (env.TAVILY_API_KEY) {24    try {25      const r = await fetch("https://api.tavily.com/extract", { method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${env.TAVILY_API_KEY}` }, body: JSON.stringify({ urls: [url] }), signal: AbortSignal.timeout(30_000) });26      if (r.ok) { const j = await r.json() as { results?: { url: string; raw_content: string }[] }; const t = j.results?.[0]?.raw_content; if (t) return { url, text: t.slice(0, 20_000), truncated: t.length > 20_000 }; }27    } catch { /* fall through */ }28  }29  const r = await fetch(url, { headers: { "user-agent": "Mozilla/5.0 (compatible; ChatSPBoucherAI/1.0; +https://chat.spboucher.ai)" }, redirect: "follow", signal: AbortSignal.timeout(20_000) });30  if (!r.ok) return { error: `HTTP ${r.status}` };31  const ct = r.headers.get("content-type") ?? "";32  if (!/text\/html|text\/plain|application\/xhtml|application\/json/.test(ct)) return { error: `Unsupported content-type ${ct}` };33  const html = await r.text();34  const title = html.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.trim();35  const text = html.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>|<nav[\s\S]*?<\/nav>|<footer[\s\S]*?<\/footer>/gi, " ").replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/\s+/g, " ").trim();36  return { url, title, text: text.slice(0, 20_000), truncated: text.length > 20_000 };37}38