Repository Public
1// Tavily Search API client (POST /search, Bearer auth). Parameters follow the documented OpenAPI schema:2// there is no pagination, no image endpoint and no video endpoint, so those are built on top in service.ts / video.ts.3import type { Config } from "../config.ts";4import { fetchJSON } from "../util.ts";5import { COUNTRY_NAMES, canonicalURL, dedupeKey, domainOf, resultId } from "./normalize.ts";6import { ProviderError, type Batch, type ImageResult, type SearchRequest, type WebResult } from "./types.ts";78export interface TavilyRaw {9 results: any[];10 images: any[];11 answer: string | null;12 credits: number;13}1415export async function tavilySearch(config: Config, body: Record<string, unknown>): Promise<TavilyRaw> {16 if (!config.tavilyKey) throw new ProviderError(503, "provider_not_configured", "TAVILY_API_KEY is not configured");17 let response;18 try {19 response = await fetchJSON(`${config.tavilyBase}/search`, {20 method: "POST",21 headers: { "content-type": "application/json", authorization: `Bearer ${config.tavilyKey}` },22 body: JSON.stringify({ ...body, include_usage: true }),23 timeoutMs: 15_000,24 });25 } catch (error: any) {26 if (error?.name === "AbortError") throw new ProviderError(504, "provider_timeout", "The search provider did not answer in time");27 throw new ProviderError(502, "provider_unreachable", "The search provider could not be reached");28 }29 const { status, body: json, headers } = response;30 if (status === 200 && json && Array.isArray(json.results)) {31 return {32 results: json.results,33 images: Array.isArray(json.images) ? json.images : [],34 answer: typeof json.answer === "string" && json.answer.trim() ? json.answer.trim() : null,35 credits: typeof json.usage?.credits === "number" ? json.usage.credits : 1,36 };37 }38 // Provider error text is never forwarded verbatim (it can echo the key state); it is mapped to stable codes.39 if (status === 401) throw new ProviderError(503, "provider_auth", "The search provider rejected the server credential");40 if (status === 429) throw new ProviderError(429, "provider_rate_limited", "The search provider is rate limiting requests", Number(headers.get("retry-after")) || 10);41 if (status === 432 || status === 433) throw new ProviderError(503, "provider_quota", "The search provider plan limit is reached");42 if (status === 400 || status === 422) throw new ProviderError(400, "provider_rejected", "The search provider rejected this query");43 throw new ProviderError(502, "provider_error", `The search provider failed (HTTP ${status})`);44}4546function baseBody(request: SearchRequest, excludeDomains: string[]): Record<string, unknown> {47 const body: Record<string, unknown> = {48 query: request.query,49 search_depth: "basic",50 topic: "general",51 max_results: 20,52 safe_search: request.safeSearch,53 };54 if (request.language) body.language = request.language;55 if (request.region && COUNTRY_NAMES[request.region]) body.country = COUNTRY_NAMES[request.region];56 if (request.timeRange) body.time_range = request.timeRange;57 if (excludeDomains.length) body.exclude_domains = excludeDomains.slice(0, 150);58 return body;59}6061export async function tavilyWebBatch(config: Config, request: SearchRequest, excludeDomains: string[], wantAnswer: boolean): Promise<Batch<WebResult>> {62 const raw = await tavilySearch(config, {63 ...baseBody(request, excludeDomains),64 include_favicon: true,65 include_published_date: true,66 include_answer: wantAnswer ? "basic" : false,67 });68 const seen = new Set<string>();69 const items: WebResult[] = [];70 for (const r of raw.results) {71 if (typeof r?.url !== "string" || typeof r?.title !== "string") continue;72 const url = canonicalURL(r.url);73 if (!url) continue;74 const key = dedupeKey(url);75 if (seen.has(key)) continue;76 seen.add(key);77 items.push({78 id: resultId(key),79 title: r.title.trim() || domainOf(url),80 url,81 domain: domainOf(url),82 favicon: typeof r.favicon === "string" && /^https?:\/\//.test(r.favicon) ? r.favicon : null,83 snippet: typeof r.content === "string" ? cleanSnippet(r.content) : "",84 publishedDate: typeof r.published_date === "string" && r.published_date ? r.published_date : null,85 score: typeof r.score === "number" ? r.score : null,86 });87 }88 return {89 items, answer: raw.answer, credits: raw.credits, provider: "tavily", notices: [],90 domains: [...new Set(items.map((i) => i.domain))], exhausted: raw.results.length < 10,91 };92}9394/** Page extracts arrive as Markdown fragments joined by " [...] ": keep the words, drop the markup. */95export function cleanSnippet(raw: string): string {96 return raw97 .replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images98 .replace(/\[([^\]]+)\]\((?:[^)]*)\)/g, "$1") // links → their text99 .replace(/^\s{0,3}(#{1,6}|>|[-*+]|\d+\.)\s+/gm, " ") // headings, quotes, list markers at line start100 .replace(/\s#{1,6}\s/g, " ") // headings inlined by the provider101 .replace(/(\*\*|__|~~|\`{1,3})/g, "") // emphasis and code marks102 .replace(/\|/g, " ").replace(/-{3,}/g, " ") // table pipes and rules103 .replace(/\s*\[\.\.\.\]\s*/g, " … ")104 .replace(/\s+/g, " ").trim().slice(0, 420);105}106107const IMAGE_EXT_BLOCK = /\.(svg|ico)(\?|$)/i;108/** Page furniture that sites expose next to their real pictures: logos, icons, avatars, tracking pixels, tiny renditions. */109const IMAGE_JUNK = /(logo|icon|sprite|avatar|favicon|placeholder|spacer|pixel|blank|badge|emoji|banner-ad|loading|spinner)/i;110const TINY_RENDITION = /[?&](w|width|h|height|size)=([1-9]?\d|1[0-4]\d)(&|$)/i;111112export async function tavilyImageBatch(config: Config, request: SearchRequest, excludeDomains: string[]): Promise<Batch<ImageResult>> {113 const raw = await tavilySearch(config, {114 ...baseBody(request, excludeDomains),115 include_images: true,116 include_image_descriptions: true,117 });118 const seen = new Set<string>();119 const items: ImageResult[] = [];120 const usable = (entry: any): string | null => {121 const imageURL = typeof entry === "string" ? entry : entry?.url;122 if (typeof imageURL !== "string" || !/^https?:\/\//i.test(imageURL) || IMAGE_EXT_BLOCK.test(imageURL)) return null;123 if (IMAGE_JUNK.test(imageURL.split("?")[0]) || TINY_RENDITION.test(imageURL)) return null;124 return imageURL;125 };126 const push = (entry: any, source: { url: string; title: string } | null) => {127 const imageURL = usable(entry);128 if (!imageURL) return;129 const key = dedupeKey(imageURL);130 if (seen.has(key)) return;131 seen.add(key);132 const description = typeof entry === "object" && typeof entry?.description === "string" && entry.description.trim() ? entry.description.trim().slice(0, 300) : null;133 items.push({134 id: resultId(key), imageURL, imageDomain: domainOf(imageURL), description,135 sourceURL: source?.url ?? null, sourceTitle: source?.title ?? null, sourceDomain: source ? domainOf(source.url) : null,136 thumbnailURL: null, width: null, height: null, author: null, license: null, provider: "tavily",137 });138 };139 // Images attached to a result carry their source page; the provider's top-level list does not.140 // Interleave so the grid does not open with a long run from a single site.141 const perResult: { entry: any; source: { url: string; title: string } }[][] = [];142 for (const r of raw.results) {143 if (!Array.isArray(r?.images) || typeof r?.url !== "string") continue;144 const url = canonicalURL(r.url);145 if (!url) continue;146 // The provider scores each image of a page (its og:image highest); keep the best few per page.147 const ranked = r.images.filter((e: any) => usable(e) !== null).sort((a: any, b: any) => (Number(b?.score) || 0) - (Number(a?.score) || 0));148 perResult.push(ranked.slice(0, 5).map((entry: any) => ({ entry, source: { url, title: typeof r.title === "string" ? r.title.trim() : "" } })));149 }150 for (const entry of raw.images) push(entry, null);151 for (let i = 0; perResult.some((list) => i < list.length); i++) {152 for (const list of perResult) if (i < list.length) push(list[i].entry, list[i].source);153 }154 const domains = [...new Set(raw.results.map((r: any) => (typeof r?.url === "string" ? domainOf(r.url) : "")).filter(Boolean))] as string[];155 return {156 items, answer: null, credits: raw.credits, provider: "tavily",157 notices: ["image-dimensions-measured-on-device"], domains, exhausted: raw.results.length < 10 || items.length === 0,158 };159}160