Repository Public
1import test from "node:test";2import assert from "node:assert/strict";3import { canonicalURL, dedupeKey, normalizeQuery, parseSearchRequest } from "../src/search/normalize.ts";4import { parseISODuration, parseVideoURL } from "../src/search/video.ts";5import { Client, fakeProvider, makeApp } from "./helpers.ts";67const result = (n: number, extra: any = {}) => ({ title: `Title ${n}`, url: `https://site${n}.example/page`, content: `Snippet ${n}\n text`, score: 1 - n / 100, favicon: `https://site${n}.example/f.ico`, ...extra });89test("query normalization and request validation", () => {10 assert.equal(normalizeQuery(" apple\u0000 silicon\n ai "), "apple silicon ai");11 assert.throws(() => normalizeQuery(" "), /empty/);12 assert.throws(() => normalizeQuery("x".repeat(401)), /longer/);13 assert.throws(() => parseSearchRequest({ query: "a", type: "news" }), /type/);14 assert.throws(() => parseSearchRequest({ query: "a", page: 0 }), /page/);15 assert.throws(() => parseSearchRequest({ query: "a", language: "english" }), /language/);16 const parsed = parseSearchRequest({ query: " Swift ", type: "images", language: "FR", region: "CA", timeRange: "week" });17 assert.deepEqual({ ...parsed }, { query: "Swift", type: "images", page: 1, language: "fr", region: "ca", timeRange: "week", source: undefined, safeSearch: true, summary: false });18});1920test("URL canonicalization removes tracking and fragments", () => {21 assert.equal(canonicalURL("https://WWW.Example.com/a/?utm_source=x&id=2&fbclid=9#top"), "https://www.example.com/a/?id=2");22 assert.equal(canonicalURL("javascript:alert(1)"), null);23 assert.equal(dedupeKey("http://www.example.com/a/"), dedupeKey("https://example.com/a"));24});2526test("video URL recognition accepts watch pages only", () => {27 assert.equal(parseVideoURL("https://m.youtube.com/watch?v=dQw4w9WgXcQ&xstg=abc")?.url, "https://www.youtube.com/watch?v=dQw4w9WgXcQ");28 assert.equal(parseVideoURL("https://youtu.be/dQw4w9WgXcQ?t=3")?.platform, "youtube");29 assert.equal(parseVideoURL("https://www.youtube.com/shorts/dQw4w9WgXcQ")?.id, "dQw4w9WgXcQ");30 assert.equal(parseVideoURL("https://vimeo.com/76979871")?.embedURL, "https://player.vimeo.com/video/76979871");31 assert.equal(parseVideoURL("https://www.dailymotion.com/video/x8abc12")?.platform, "dailymotion");32 for (const url of ["https://www.youtube.com/playlist?list=PL1", "https://www.youtube.com/@channel", "https://vimeo.com/channels/staffpicks", "https://example.com/watch?v=dQw4w9WgXcQ", "nope"]) {33 assert.equal(parseVideoURL(url), null, url);34 }35 assert.equal(parseISODuration("PT1H2M3S"), 3723);36 assert.equal(parseISODuration("PT45S"), 45);37});3839test("web search: provider request, normalization, dedup, cache, paging, usage accounting", async () => {40 const provider = await fakeProvider();41 const app = makeApp({ tavilyBase: provider.url });42 const client = new Client(app);43 provider.handler = (_path, body) => body.exclude_domains44 ? { body: { results: [result(30), result(31)], images: [], usage: { credits: 1 } } }45 : { body: { answer: " Summary text. ", usage: { credits: 1 }, images: [], results: [46 ...Array.from({ length: 19 }, (_, i) => result(i)),47 result(0, { url: "http://site0.example/page/?utm_campaign=x#frag" }), // duplicate of the first48 { title: 5, url: "https://bad.example" }, { title: "No URL" }, { title: "Script", url: "javascript:alert(1)" },49 ] } };50 const first = await client.call("POST", "/api/search", { query: " Apple Silicon ", type: "web", summary: true, language: "en", region: "ca", timeRange: "month" });51 assert.equal(first.status, 200);52 const sent = provider.calls[0];53 assert.equal(sent.auth, "Bearer tvly-test");54 assert.equal(sent.body.query, "Apple Silicon");55 assert.equal(sent.body.max_results, 20);56 assert.equal(sent.body.country, "canada");57 assert.equal(sent.body.language, "en");58 assert.equal(sent.body.time_range, "month");59 assert.equal(sent.body.include_answer, "basic");60 assert.equal(first.json.results.length, 10);61 assert.equal(first.json.answer.text, "Summary text.");62 assert.equal(first.json.results[0].domain, "site0.example");63 assert.equal(first.json.results[0].snippet, "Snippet 0 text");64 assert.equal(first.json.cached, false);65 assert.equal(first.json.hasMore, true);66 assert.ok(!JSON.stringify(first.json).includes("tvly-test"));6768 // Same query, different spacing/case → cache hit, no provider call.69 const again = await client.call("POST", "/api/search", { query: "apple silicon", type: "web", summary: true, language: "en", region: "ca", timeRange: "month" });70 assert.equal(again.json.cached, true);71 assert.equal(provider.calls.length, 1);7273 // Page 2: the first batch has 19 unique results, so a second batch (other domains) completes the page.74 const second = await client.call("POST", "/api/search", { query: "apple silicon", type: "web", summary: true, language: "en", region: "ca", timeRange: "month", page: 2 });75 assert.equal(second.json.results.length, 10);76 assert.equal(second.json.answer, null);77 assert.equal(provider.calls.length, 2);78 assert.ok(provider.calls[1].body.exclude_domains.includes("site0.example"));79 assert.equal(provider.calls[1].body.include_answer, false);80 const ids = new Set([...first.json.results, ...second.json.results].map((r: any) => r.url));81 assert.equal(ids.size, 20);82 assert.equal(second.json.hasMore, true);8384 // Page 3: the provider had only two more results; the last one is served and nothing further is fetched.85 const third = await client.call("POST", "/api/search", { query: "apple silicon", type: "web", summary: true, language: "en", region: "ca", timeRange: "month", page: 3 });86 assert.equal(provider.calls.length, 2);87 assert.deepEqual(third.json.results.map((r: any) => r.domain), ["site31.example"]);88 assert.equal(third.json.hasMore, false);8990 const usage = app.db.prepare("SELECT requests, credits, cache_hits FROM usage_daily WHERE kind = 'web'").get() as any;91 assert.equal(usage.requests, 2);92 assert.equal(usage.credits, 2);93 assert.ok(usage.cache_hits >= 3);94 await app.server.close(); await provider.close();95});9697test("provider failures map to stable errors and never leak provider text or the key", async () => {98 const provider = await fakeProvider();99 const app = makeApp({ tavilyBase: provider.url });100 const client = new Client(app);101 const cases: [number, number, string][] = [[401, 503, "provider_auth"], [429, 429, "provider_rate_limited"], [432, 503, "provider_quota"], [500, 502, "provider_error"]];102 for (const [upstream, status, code] of cases) {103 provider.handler = () => ({ status: upstream, body: { detail: { error: "secret upstream text tvly-test" } }, headers: upstream === 429 ? { "retry-after": "7" } : {} });104 const r = await client.call("POST", "/api/search", { query: "failure " + upstream });105 assert.equal(r.status, status);106 assert.equal(r.json.error.code, code);107 assert.ok(!JSON.stringify(r.json).includes("upstream text"));108 if (upstream === 429) assert.equal(r.headers["retry-after"], "7");109 }110 const errors = (app.db.prepare("SELECT SUM(errors) AS n FROM usage_daily").get() as any).n;111 assert.equal(errors, 4);112 await app.server.close(); await provider.close();113114 const unreachable = makeApp({ tavilyBase: "http://127.0.0.1:1" });115 const r = await new Client(unreachable).call("POST", "/api/search", { query: "x" });116 assert.equal(r.status, 502);117 assert.equal(r.json.error.code, "provider_unreachable");118 await unreachable.server.close();119120 const noKey = makeApp({ tavilyKey: "" });121 assert.equal((await new Client(noKey).call("POST", "/api/search", { query: "x" })).json.error.code, "provider_not_configured");122 await noKey.server.close();123});124125test("daily budget and rate limits fail closed", async () => {126 const provider = await fakeProvider();127 provider.handler = () => ({ body: { results: [result(1)], images: [], usage: { credits: 1 } } });128 const app = makeApp({ tavilyBase: provider.url, dailyCreditBudget: 2, anonSearchPerHour: 4 });129 const client = new Client(app);130 assert.equal((await client.call("POST", "/api/search", { query: "one" })).status, 200);131 assert.equal((await client.call("POST", "/api/search", { query: "two" })).status, 200);132 const blocked = await client.call("POST", "/api/search", { query: "three" });133 assert.equal(blocked.status, 503);134 assert.equal(blocked.json.error.code, "budget_exhausted");135 assert.equal((await client.call("POST", "/api/search", { query: "one" })).status, 200); // cached answers still served136 const limited = await client.call("POST", "/api/search", { query: "one" });137 assert.equal(limited.status, 429);138 assert.equal(limited.json.error.code, "rate_limited");139 await app.server.close(); await provider.close();140141 const closed = makeApp({ anonSearch: false });142 assert.equal((await new Client(closed).call("POST", "/api/search", { query: "x" })).status, 401);143 await closed.server.close();144});145146test("image search: page attribution, junk filtering, Commons merge with dimensions", async () => {147 const provider = await fakeProvider();148 process.env.COMMONS_API = provider.url + "/commons";149 provider.handler = (path) => path === "/commons"150 ? { body: { query: { pages: {151 "1": { index: 1, title: "File:Aurora over lake.jpg", imageinfo: [{ url: "https://upload.example/aurora.jpg", descriptionurl: "https://commons.example/wiki/File:Aurora", mime: "image/jpeg", width: 4000, height: 3000, thumburl: "https://upload.example/thumb/640px-aurora.jpg?utm_source=x", extmetadata: { Artist: { value: "<a href='x'>Jane & Co</a>" }, LicenseShortName: { value: "CC BY 4.0" }, ImageDescription: { value: "<p>Aurora</p>" } } }] },152 "2": { index: 2, title: "File:Tiny.png", imageinfo: [{ url: "https://upload.example/tiny.png", mime: "image/png", width: 50, height: 50 }] },153 "3": { index: 3, title: "File:Doc.pdf", imageinfo: [{ url: "https://upload.example/doc.pdf", mime: "application/pdf", width: 900, height: 900 }] },154 } } } }155 : { body: { usage: { credits: 1 }, images: [{ url: "https://cdn.example/top.jpg", description: "Top image" }, "https://cdn.example/plain.jpg"],156 results: [{ title: "Gallery", url: "https://photos.example/gallery?utm_source=a", images: [157 { url: "https://photos.example/a.jpg", description: "A", score: 5 }, { url: "https://photos.example/hero.jpg", description: null, score: 10 },158 { url: "https://photos.example/site-logo.png", score: 9 }, { url: "https://photos.example/small.jpg?width=24", score: 8 }, { url: "https://photos.example/vector.svg", score: 7 },159 { url: "data:image/png;base64,AAAA", score: 6 }, { url: "https://cdn.example/top.jpg", score: 1 },160 ] }] } };161 const app = makeApp({ tavilyBase: provider.url });162 const r = await new Client(app).call("POST", "/api/search", { query: "aurora", type: "images" });163 assert.equal(r.status, 200);164 assert.equal(provider.calls.find((c) => c.path === "/search").body.include_images, true);165 const urls = r.json.results.map((i: any) => i.imageURL);166 assert.deepEqual(urls, ["https://cdn.example/top.jpg", "https://cdn.example/plain.jpg", "https://photos.example/hero.jpg", "https://photos.example/a.jpg", "https://upload.example/aurora.jpg"]);167 const hero = r.json.results[2];168 assert.equal(hero.sourceURL, "https://photos.example/gallery");169 assert.equal(hero.sourceTitle, "Gallery");170 assert.equal(hero.width, null);171 assert.equal(r.json.results[0].sourceURL, null);172 const commons = r.json.results[4];173 assert.deepEqual([commons.width, commons.height, commons.author, commons.license, commons.provider, commons.thumbnailURL],174 [4000, 3000, "Jane & Co", "CC BY 4.0", "wikimedia-commons", "https://upload.example/thumb/640px-aurora.jpg"]);175 assert.equal(r.json.provider, "tavily+wikimedia-commons");176 delete process.env.COMMONS_API;177 await app.server.close(); await provider.close();178});179180test("video search: only platform-confirmed watch pages are returned", async () => {181 const provider = await fakeProvider();182 provider.handler = (path, _body, query) => {183 if (path === "/search") return { body: { usage: { credits: 1 }, images: [], results: [184 { title: "Video A", url: "https://www.youtube.com/watch?v=AAAAAAAAAAA&xstg=1", content: "Transcript A" },185 { title: "Video A again", url: "https://youtu.be/AAAAAAAAAAA" },186 { title: "A playlist", url: "https://www.youtube.com/playlist?list=PL1" },187 { title: "Removed video", url: "https://www.youtube.com/watch?v=BBBBBBBBBBB" },188 { title: "Vimeo", url: "https://vimeo.com/123456789" },189 { title: "Blog about video", url: "https://vimeo.com/blog/post/hello" },190 ] } };191 const url = query.get("url")!;192 if (url.includes("BBBBBBBBBBB")) return { status: 404, body: null };193 if (path === "/yt") return { body: { type: "video", title: "Real title A", author_name: "Channel A", author_url: "https://www.youtube.com/@a", thumbnail_url: "https://i.ytimg.com/vi/AAAAAAAAAAA/hqdefault.jpg" } };194 if (path === "/vimeo") return { body: { type: "video", title: "Vimeo film", author_name: "Studio", duration: 125, upload_date: "2024-05-01 10:00:00", thumbnail_url: "https://i.vimeocdn.com/x.jpg", description: "A film" } };195 return { status: 404, body: null };196 };197 const app = makeApp({ tavilyBase: provider.url, oembedBases: { youtube: provider.url + "/yt", vimeo: provider.url + "/vimeo", dailymotion: provider.url + "/dm" } });198 const r = await new Client(app).call("POST", "/api/search", { query: "apple silicon video", type: "videos" });199 assert.equal(r.status, 200);200 const search = provider.calls.filter((c) => c.path === "/search");201 assert.equal(search.length, 1); // the query already says "video": no second phrasing202 assert.deepEqual(search[0].body.include_domains, ["youtube.com", "youtu.be", "vimeo.com", "dailymotion.com"]);203 assert.equal(r.json.results.length, 2);204 const [a, v] = r.json.results;205 assert.deepEqual([a.title, a.channel, a.platform, a.durationSeconds, a.verifiedBy, a.description], ["Real title A", "Channel A", "youtube", null, "oembed", "A"]);206 assert.equal(a.thumbnail, "https://i.ytimg.com/vi/AAAAAAAAAAA/mqdefault.jpg");207 assert.deepEqual([v.title, v.durationSeconds, v.publishedDate, v.embedURL], ["Vimeo film", 125, "2024-05-01T10:00:00", "https://player.vimeo.com/video/123456789"]);208 assert.ok(r.json.notices.includes("video-duration-unavailable-without-youtube-data-api"));209 assert.equal(r.json.provider, "tavily+oembed");210211 const filtered = await new Client(app).call("POST", "/api/search", { query: "apple silicon video", type: "videos", source: "vimeo" });212 assert.deepEqual(provider.calls.filter((c) => c.path === "/search").at(-1).body.include_domains, ["vimeo.com"]);213 assert.equal(filtered.status, 200);214 await app.server.close(); await provider.close();215});216217test("video search through the YouTube Data API when a key is configured", async () => {218 const provider = await fakeProvider();219 process.env.YOUTUBE_BASE = provider.url;220 provider.handler = (path) => path === "/search"221 ? { body: { nextPageToken: "NEXT", items: [{ id: { videoId: "AAAAAAAAAAA" } }, { id: { videoId: "GONE0000000" } }] } }222 : { body: { items: [{ id: "AAAAAAAAAAA", contentDetails: { duration: "PT10M5S" }, snippet: { title: "Talk", channelTitle: "Chan", channelId: "UC1", publishedAt: "2025-01-02T00:00:00Z", description: "Desc", thumbnails: { high: { url: "https://i.ytimg.com/h.jpg" } } } }] } };223 const app = makeApp({ youtubeKey: "yt-key", tavilyBase: "http://127.0.0.1:1" });224 const r = await new Client(app).call("POST", "/api/search", { query: "talk", type: "videos", region: "ca" });225 assert.equal(r.status, 200);226 assert.equal(provider.calls[0].query.get("regionCode"), "CA");227 assert.equal(r.json.results.length, 1);228 assert.deepEqual([r.json.results[0].durationSeconds, r.json.results[0].publishedDate, r.json.results[0].verifiedBy], [605, "2025-01-02T00:00:00Z", "youtube-data-api"]);229 assert.equal(r.json.provider, "youtube");230 assert.ok(!JSON.stringify(r.json).includes("yt-key"));231 delete process.env.YOUTUBE_BASE;232 await app.server.close(); await provider.close();233});234