Repository Public
1// Video search. Tavily has no video endpoint, so two authorised strategies sit behind one interface:2// 1. YouTube Data API v3 (when YOUTUBE_API_KEY is configured): real video search with duration and publication date.3// 2. Otherwise: a Tavily web search restricted to video platforms, where every candidate URL must (a) match the4// platform's watch-page pattern and (b) be confirmed by the platform's own oEmbed endpoint. Pages that are not5// confirmed are dropped — an ordinary web page is never presented as a video.6import type { Config } from "../config.ts";7import { fetchJSON } from "../util.ts";8import { resultId } from "./normalize.ts";9import { cleanSnippet, tavilySearch } from "./tavily.ts";10import { COUNTRY_NAMES } from "./normalize.ts";11import { ProviderError, type Batch, type SearchRequest, type VideoResult } from "./types.ts";1213type Platform = VideoResult["platform"];14const PLATFORM_DOMAINS: Record<Platform, string[]> = {15 youtube: ["youtube.com", "youtu.be"],16 vimeo: ["vimeo.com"],17 dailymotion: ["dailymotion.com"],18};1920export interface VideoRef { platform: Platform; id: string; url: string; embedURL: string }2122/** Recognises a watch page and returns its canonical form; anything else (channels, playlists, searches) is null. */23export function parseVideoURL(raw: string): VideoRef | null {24 let url: URL;25 try { url = new URL(raw); } catch { return null; }26 const host = url.hostname.toLowerCase().replace(/^(www|m|music)\./, "");27 const path = url.pathname;28 let m: RegExpExecArray | null;29 if (host === "youtube.com") {30 const v = url.searchParams.get("v");31 const id = path === "/watch" && v ? v : (m = /^\/(shorts|embed|live)\/([\w-]{11})/.exec(path)) ? m[2] : null;32 if (id && /^[\w-]{11}$/.test(id)) return { platform: "youtube", id, url: `https://www.youtube.com/watch?v=${id}`, embedURL: `https://www.youtube-nocookie.com/embed/${id}` };33 return null;34 }35 if (host === "youtu.be" && (m = /^\/([\w-]{11})/.exec(path))) {36 return { platform: "youtube", id: m[1], url: `https://www.youtube.com/watch?v=${m[1]}`, embedURL: `https://www.youtube-nocookie.com/embed/${m[1]}` };37 }38 if ((host === "vimeo.com" || host === "player.vimeo.com") && (m = /^\/(?:video\/)?(\d{5,12})(?:\/|$)/.exec(path))) {39 return { platform: "vimeo", id: m[1], url: `https://vimeo.com/${m[1]}`, embedURL: `https://player.vimeo.com/video/${m[1]}` };40 }41 if (host === "dailymotion.com" && (m = /^\/video\/([a-z0-9]+)/i.exec(path))) {42 return { platform: "dailymotion", id: m[1], url: `https://www.dailymotion.com/video/${m[1]}`, embedURL: `https://www.dailymotion.com/embed/video/${m[1]}` };43 }44 return null;45}4647const oembedCache = new Map<string, { at: number; value: VideoResult | null }>();48const OEMBED_TTL = 24 * 3600_000;4950async function confirm(config: Config, ref: VideoRef, fallbackDescription: string | null): Promise<VideoResult | null> {51 const cached = oembedCache.get(ref.url);52 if (cached && Date.now() - cached.at < OEMBED_TTL) return cached.value ? { ...cached.value, description: cached.value.description ?? fallbackDescription } : null;53 const base = config.oembedBases[ref.platform];54 let value: VideoResult | null = null;55 try {56 const { status, body } = await fetchJSON(`${base}?format=json&url=${encodeURIComponent(ref.url)}`, { timeoutMs: 6000, headers: { accept: "application/json" } });57 if (status === 200 && body && typeof body.title === "string" && (body.type === undefined || body.type === "video")) {58 value = {59 id: resultId(ref.url), url: ref.url, title: body.title.trim(), platform: ref.platform,60 channel: typeof body.author_name === "string" ? body.author_name : null,61 channelURL: typeof body.author_url === "string" ? body.author_url : null,62 // YouTube oEmbed points at the 4:3 "hqdefault" still (letterboxed); "mqdefault" is the same frame in 16:9.63 thumbnail: typeof body.thumbnail_url === "string" ? body.thumbnail_url.replace(/(i\.ytimg\.com\/vi\/[\w-]{11}\/)hqdefault\.jpg$/, "$1mqdefault.jpg") : null,64 // Vimeo's oEmbed reports duration and upload date; YouTube's and Dailymotion's do not.65 durationSeconds: typeof body.duration === "number" && body.duration > 0 ? Math.round(body.duration) : null,66 publishedDate: typeof body.upload_date === "string" ? body.upload_date.replace(" ", "T") : null,67 description: typeof body.description === "string" && body.description.trim() ? body.description.trim().slice(0, 300) : null,68 embedURL: ref.embedURL, verifiedBy: "oembed",69 };70 } else if (status >= 500 || status === 429) {71 return null; // transient: do not cache the failure72 }73 } catch {74 return null;75 }76 oembedCache.set(ref.url, { at: Date.now(), value });77 if (oembedCache.size > 5000) oembedCache.delete(oembedCache.keys().next().value!);78 return value ? { ...value, description: value.description ?? fallbackDescription } : null;79}8081async function mapLimit<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]> {82 const out: R[] = new Array(items.length);83 let next = 0;84 await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {85 while (next < items.length) { const i = next++; out[i] = await fn(items[i]); }86 }));87 return out;88}8990async function tavilyVideoBatch(config: Config, request: SearchRequest, excludeIds: string[]): Promise<Batch<VideoResult>> {91 const platforms: Platform[] = request.source ? [request.source] : ["youtube", "vimeo", "dailymotion"];92 const body: Record<string, unknown> = {93 query: request.query, search_depth: "basic", topic: "general", max_results: 20, safe_search: request.safeSearch,94 include_domains: platforms.flatMap((p) => PLATFORM_DOMAINS[p]),95 };96 if (request.language) body.language = request.language;97 if (request.region && COUNTRY_NAMES[request.region]) body.country = COUNTRY_NAMES[request.region];98 if (request.timeRange) body.time_range = request.timeRange;99 // The provider returns the same video under several URL variants, so one call yields few distinct videos.100 // A second phrasing of the same query widens the set; both run in parallel and the second is optional.101 const [first, second] = await Promise.allSettled([102 tavilySearch(config, body),103 /\bvideos?\b/i.test(request.query) ? Promise.reject(new Error("skip")) : tavilySearch(config, { ...body, query: `${request.query} video` }),104 ]);105 if (first.status === "rejected") throw first.reason;106 const raw = { results: [...first.value.results], credits: first.value.credits };107 if (second.status === "fulfilled") {108 // Interleave so the second phrasing does not push the best matches down.109 const merged: any[] = [];110 for (let i = 0; i < Math.max(first.value.results.length, second.value.results.length); i++) {111 if (i < first.value.results.length) merged.push(first.value.results[i]);112 if (i < second.value.results.length) merged.push(second.value.results[i]);113 }114 raw.results = merged;115 raw.credits += second.value.credits;116 }117 const seen = new Set(excludeIds);118 const candidates: { ref: VideoRef; description: string | null }[] = [];119 for (const r of raw.results) {120 const ref = typeof r?.url === "string" ? parseVideoURL(r.url) : null;121 if (!ref || seen.has(ref.url)) continue;122 seen.add(ref.url);123 candidates.push({ ref, description: typeof r.content === "string" ? cleanSnippet(r.content.replace(/^\s*#*\s*Transcript\b:?/i, "")).slice(0, 300) || null : null });124 }125 // A platform that does not answer in time leaves a candidate unconfirmed (never cached as "not a video"):126 // those are tried once more, and a batch that still has unanswered candidates is not kept in the search cache.127 let outcomes = await mapLimit(candidates, 6, (c) => confirm(config, c.ref, c.description));128 const unanswered = () => candidates.filter((c, i) => outcomes[i] === null && !oembedCache.has(c.ref.url));129 if (unanswered().length) {130 await new Promise((r) => setTimeout(r, 400));131 outcomes = await mapLimit(candidates, 4, (c, ) => confirm(config, c.ref, c.description));132 }133 const incomplete = unanswered().length > 0;134 const confirmed = outcomes.filter((v): v is VideoResult => v !== null);135 const notices: string[] = [];136 if (incomplete) notices.push("video-confirmation-incomplete");137 if (confirmed.some((v) => v.durationSeconds === null)) notices.push("video-duration-unavailable-without-youtube-data-api");138 if (confirmed.some((v) => v.publishedDate === null)) notices.push("video-date-unavailable-without-youtube-data-api");139 // The provider has no offset: a second batch would return the same pages, so this strategy is single-batch.140 return { items: confirmed, answer: null, credits: raw.credits, provider: "tavily+oembed", notices, domains: [], exhausted: true, volatile: incomplete };141}142143export function parseISODuration(value: string): number | null {144 const m = /^P(?:(\d+)D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$/.exec(value);145 if (!m) return null;146 return (Number(m[1] ?? 0) * 86400) + (Number(m[2] ?? 0) * 3600) + (Number(m[3] ?? 0) * 60) + Number(m[4] ?? 0);147}148149async function youtubeBatch(config: Config, request: SearchRequest, pageToken: string | null): Promise<Batch<VideoResult>> {150 const base = process.env.YOUTUBE_BASE ?? "https://www.googleapis.com/youtube/v3";151 const params = new URLSearchParams({ part: "snippet", type: "video", maxResults: "20", q: request.query, key: config.youtubeKey, safeSearch: request.safeSearch ? "moderate" : "none" });152 if (pageToken) params.set("pageToken", pageToken);153 if (request.language) params.set("relevanceLanguage", request.language.slice(0, 2));154 if (request.region) params.set("regionCode", request.region.toUpperCase());155 if (request.timeRange) {156 const days = { day: 1, week: 7, month: 31, year: 366 }[request.timeRange];157 params.set("publishedAfter", new Date(Date.now() - days * 86400_000).toISOString());158 }159 const search = await fetchJSON(`${base}/search?${params}`, { timeoutMs: 10_000 });160 if (search.status === 403) throw new ProviderError(503, "provider_quota", "The video provider quota is exhausted");161 if (search.status !== 200 || !Array.isArray(search.body?.items)) throw new ProviderError(502, "provider_error", `The video provider failed (HTTP ${search.status})`);162 const ids: string[] = search.body.items.map((i: any) => i?.id?.videoId).filter((x: any) => typeof x === "string");163 const details = new Map<string, any>();164 if (ids.length) {165 const d = await fetchJSON(`${base}/videos?${new URLSearchParams({ part: "contentDetails,snippet", id: ids.join(","), key: config.youtubeKey })}`, { timeoutMs: 10_000 });166 if (d.status === 200 && Array.isArray(d.body?.items)) for (const item of d.body.items) details.set(item.id, item);167 }168 const items: VideoResult[] = [];169 for (const id of ids) {170 const item = details.get(id);171 if (!item) continue; // only videos the API confirms172 const s = item.snippet ?? {};173 const url = `https://www.youtube.com/watch?v=${id}`;174 items.push({175 id: resultId(url), url, title: String(s.title ?? "").trim(), platform: "youtube",176 channel: s.channelTitle ?? null, channelURL: s.channelId ? `https://www.youtube.com/channel/${s.channelId}` : null,177 thumbnail: s.thumbnails?.high?.url ?? s.thumbnails?.medium?.url ?? s.thumbnails?.default?.url ?? null,178 durationSeconds: typeof item.contentDetails?.duration === "string" ? parseISODuration(item.contentDetails.duration) : null,179 publishedDate: s.publishedAt ?? null,180 description: typeof s.description === "string" && s.description.trim() ? s.description.trim().slice(0, 300) : null,181 embedURL: `https://www.youtube-nocookie.com/embed/${id}`, verifiedBy: "youtube-data-api",182 });183 }184 const nextToken = typeof search.body.nextPageToken === "string" ? search.body.nextPageToken : null;185 // search.list costs 100 quota units and videos.list 1; reported as "credits" of the youtube provider.186 return { items, answer: null, credits: 101, provider: "youtube", notices: [], domains: [], exhausted: !nextToken, nextToken };187}188189export function videoProviderName(config: Config, request: SearchRequest): string {190 return config.youtubeKey && (!request.source || request.source === "youtube") ? "youtube" : "tavily";191}192193export async function videoBatch(config: Config, request: SearchRequest, previous: Batch<VideoResult>[]): Promise<Batch<VideoResult>> {194 if (videoProviderName(config, request) === "youtube") {195 return youtubeBatch(config, request, previous.length ? previous[previous.length - 1].nextToken ?? null : null);196 }197 return tavilyVideoBatch(config, request, previous.flatMap((b) => b.items.map((i) => i.url)));198}199