Repository Public
1import { all, bumpTitles, get, log, now, run, tx } from "./db";2import { env } from "./env";3import { fetchText } from "./ssrf";45// Films and series that may legally be watched, with the reason recorded for every title.6//7// A title is imported only when an independent source documents its rights:8// - Wikidata says « public domain » (P6216) or gives a free licence (P275), AND9// - the video is hosted by Internet Archive (P724) or Wikimedia Commons (P10), which serve it directly to the browser.10// Whole Internet Archive collections are NOT imported: they contain many uploads whose rights are wrong or unknown.11// Series: Wikidata titles with the same rule, plus a short reviewed list (docs/SOURCES.md), plus what the administrator adds.1213const UA = "SP-TV/1.0 (personal media catalogue; https://tv.spboucher.ai)";14const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));1516async function getJson<T>(url: string, limitMb = 30): Promise<T> { return JSON.parse(await fetchText(url, limitMb * 1024 * 1024, 120_000)) as T; }1718type Binding = Record<string, { value: string } | undefined>;19async function sparql(query: string): Promise<Binding[]> {20 const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`;21 for (let attempt = 0; ; attempt++) {22 try { return (await getJson<{ results: { bindings: Binding[] } }>(url, 60)).results.bindings; }23 catch (e) { if (attempt >= 2) throw e; await sleep(5000 * (attempt + 1)); }24 }25}26const qid = (uri: string | undefined) => uri?.slice(uri.lastIndexOf("/") + 1) ?? "";2728// --- genres ---------------------------------------------------------------------------------------------------------29const GENRES: [RegExp, string][] = [30 [/documentaire/i, "Documentaire"], [/science-fiction/i, "Science-fiction"], [/anim|dessin animé/i, "Animation"], [/horreur|épouvante|monstre|gothique|vampire|zombie/i, "Horreur"],31 [/western/i, "Western"], [/noir|criminel|policier|énigme|gangster|détective/i, "Policier"], [/thriller|suspense|espionnage/i, "Thriller"], [/guerre|propagande/i, "Guerre"],32 [/musical|musique|opéra/i, "Musical"], [/amour|romant/i, "Romance"], [/fantasy|fantast|merveilleux|conte/i, "Fantastique"], [/aventure|cape et d'épée|pirate/i, "Aventure"],33 [/action|arts martiaux/i, "Action"], [/comédie|comique|burlesque|slapstick|parodie|satir/i, "Comédie"], [/drame|dramatique|mélodrame|tragédie|procès/i, "Drame"],34 [/histor|biograph|péplum/i, "Histoire"], [/familial|enfants|jeunesse/i, "Famille"], [/muet/i, "Muet"], [/expérimental|avant-garde|d'art/i, "Expérimental"], [/sport/i, "Sport"],35];36const ADULT = /pornograph|érotique|hentai|sexploitation/i;37export function mapGenres(labels: string[]): string[] {38 const out = new Set<string>();39 for (const l of labels) for (const [re, g] of GENRES) if (re.test(l)) out.add(g);40 return [...out];41}4243// --- Internet Archive -----------------------------------------------------------------------------------------------44interface IaFile { name: string; format?: string; source?: string; length?: string; height?: string; width?: string; size?: string }45interface IaItem { metadata?: { title?: string; licenseurl?: string; mediatype?: string; description?: string | string[]; year?: string; date?: string; "access-restricted-item"?: string }; files?: IaFile[]; is_dark?: boolean }4647const STOP = new Set(["the", "a", "an", "of", "and", "le", "la", "les", "de", "du", "des", "un", "une", "et", "el", "los", "der", "die", "das", "in", "to", "film", "movie", "full", "hd"]);48const words = (s: string) => new Set(s.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").split(" ").filter((w) => w.length > 1 && !STOP.has(w) && !/^(19|20)\d\d$/.test(w)));49/**50 * Guards against a wrong Internet Archive identifier on Wikidata (it happens: a namesake, a remake, a trailer).51 * 0 = refuse, higher = better. The item must be about the same work: its year must not contradict the film's, and its52 * title must share its significant words with one of the film's titles.53 */54export function iaMatch(item: IaItem, titles: (string | undefined)[], year: number | undefined): number {55 const t = item.metadata?.title ?? ""; if (!t) return 1;56 const y = Number(/(?:^|[^\d])((?:18|19|20)\d\d)(?:[^\d]|$)/.exec(`${item.metadata?.year ?? ""} ${item.metadata?.date ?? ""} ${t}`)?.[1]);57 if (year && y && Math.abs(y - year) > 2) return 0;58 const iw = words(t); if (!iw.size) return 1;59 let best = 0;60 for (const cand of titles) {61 if (!cand) continue; const cw = words(cand); if (!cw.size) continue;62 const common = [...cw].filter((w) => iw.has(w)).length;63 best = Math.max(best, common / cw.size + (common === cw.size && iw.size === cw.size ? 1 : 0));64 }65 return best >= 0.6 ? 1 + best : 0;66}67export interface MediaPick { url: string; name: string; height: number | null; duration: number | null; size: number | null; subtitles: { label: string; lang: string; url: string }[] }6869const iaCache = new Map<string, IaItem | null>();70let iaLast = 0;71async function iaItem(id: string): Promise<IaItem | null> {72 if (iaCache.has(id)) return iaCache.get(id)!;73 // One request at a time, 4 per second at most.74 const wait = iaLast + 250 - Date.now(); if (wait > 0) await sleep(wait); iaLast = Date.now();75 let item: IaItem | null = null;76 try { item = await getJson<IaItem>(`https://archive.org/metadata/${encodeURIComponent(id)}`, 20); } catch { item = null; }77 if (item && (!item.files || item.is_dark || item.metadata?.["access-restricted-item"] === "true")) item = null;78 iaCache.set(id, item); if (iaCache.size > 300) iaCache.delete(iaCache.keys().next().value!);79 return item;80}81const secs = (s: string | undefined): number | null => {82 if (!s) return null;83 if (/^\d+(\.\d+)?$/.test(s)) return Math.round(Number(s));84 const p = s.split(":").map(Number); if (p.some(Number.isNaN)) return null;85 return Math.round(p.reduce((a, b) => a * 60 + b, 0));86};87const iaUrl = (id: string, name: string) => `https://archive.org/download/${encodeURIComponent(id)}/${name.split("/").map(encodeURIComponent).join("/")}`;88const LANGS: Record<string, string> = { en: "English", fr: "Français", es: "Español", de: "Deutsch", it: "Italiano", pt: "Português", nl: "Nederlands", ru: "Русский", ja: "日本語", zh: "中文", ar: "العربية", pl: "Polski", sv: "Svenska", tr: "Türkçe" };89function subLang(file: string): { lang: string; label: string } {90 const base = file.toLowerCase().replace(/\.(srt|vtt)$/, "");91 if (base.endsWith(".asr")) return { lang: "en", label: "English (automatique)" };92 const m = /[._ -]([a-z]{2,3}|english|french|fran[cç]ais|spanish|german|italian|portuguese)$/.exec(base);93 const raw = m?.[1] ?? "";94 const code = ({ english: "en", eng: "en", french: "fr", fre: "fr", fra: "fr", français: "fr", francais: "fr", spanish: "es", spa: "es", german: "de", ger: "de", deu: "de", italian: "it", ita: "it", portuguese: "pt", por: "pt" } as Record<string, string>)[raw] ?? (raw.length === 2 ? raw : "");95 return code && LANGS[code] ? { lang: code, label: LANGS[code] } : { lang: "und", label: raw ? raw.toUpperCase() : "Sous-titres" };96}97const isMp4 = (f: IaFile) => /\.(mp4|m4v)$/i.test(f.name);98const rank = (f: IaFile) => (/^h\.264/i.test(f.format ?? "") ? 0 : /mpeg4/i.test(f.format ?? "") ? 1 : 2) * 10_000 - Math.min(1080, Number(f.height) || 0);99100/** Browser-playable videos of an item (H.264/MPEG-4 in MP4), one per distinct work, best rendition first. */101export function pickVideos(id: string, item: IaItem): MediaPick[] {102 const files = item.files ?? [];103 const groups = new Map<string, IaFile[]>();104 for (const f of files.filter(isMp4)) {105 const key = f.name.replace(/\.(mp4|m4v)$/i, "").replace(/(_512kb|\.ia|_h264|_hd)$/i, "").toLowerCase();106 const l = groups.get(key); if (l) l.push(f); else groups.set(key, [f]);107 }108 const subs = files.filter((f) => /\.(srt|vtt)$/i.test(f.name));109 return [...groups.entries()].map(([key, list]) => {110 const f = list.sort((a, b) => rank(a) - rank(b))[0];111 const mine = subs.filter((s) => s.name.toLowerCase().startsWith(key) || (groups.size === 1 && !s.name.includes("/")) || s.name.toLowerCase().startsWith(key.split("/")[0] + "/"));112 const seen = new Set<string>();113 return {114 url: iaUrl(id, f.name), name: f.name, height: Number(f.height) || null, duration: secs(f.length), size: Number(f.size) || null,115 subtitles: mine.map((s) => ({ ...subLang(s.name), url: iaUrl(id, s.name) })).filter((s) => (seen.has(s.label) ? false : (seen.add(s.label), true))).slice(0, 40),116 };117 });118}119120function rightsFromLicence(url: string | undefined): { status: string; label: string } | null {121 if (!url) return null;122 const u = url.toLowerCase();123 if (u.includes("publicdomain")) return { status: "pd", label: u.includes("zero") ? "CC0 (domaine public)" : "Domaine public" };124 const m = /creativecommons\.org\/licenses\/([a-z-]+)\/(\d\.\d)/.exec(u);125 return m ? { status: "cc", label: `Creative Commons ${m[1].toUpperCase()} ${m[2]}` } : null;126}127128// --- Wikipedia synopses ---------------------------------------------------------------------------------------------129async function extracts(lang: "fr" | "en", titles: string[]): Promise<Map<string, string>> {130 const out = new Map<string, string>(); let failures = 0;131 for (let i = 0; i < titles.length; i += 20) {132 const batch = titles.slice(i, i + 20);133 try {134 const d = await getJson<{ query?: { normalized?: { from: string; to: string }[]; pages?: Record<string, { title: string; extract?: string }> } }>(135 `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&maxlag=5&prop=extracts&exintro=1&explaintext=1&exlimit=20&redirects=1&titles=${encodeURIComponent(batch.join("|"))}`);136 const back = new Map((d.query?.normalized ?? []).map((n) => [n.to, n.from]));137 for (const p of Object.values(d.query?.pages ?? {})) if (p.extract && p.extract.length > 40) { out.set(p.title, p.extract.trim()); const f = back.get(p.title); if (f) out.set(f, p.extract.trim()); }138 failures = 0;139 } catch (e) {140 // Rate limited: wait, retry this batch once more later, and give up for this run after three refusals in a row141 // (titles without a synopsis are asked again at the next import).142 if (++failures >= 3) { log("warn", "vod", `Synopsis Wikipédia (${lang}) : interrompu après limitation de débit, ${out.size} reçus ; le reste sera demandé au prochain import`, { error: (e as Error).message }); break; }143 await sleep(30_000 * failures); i -= 20; continue;144 }145 await sleep(1100);146 }147 return out;148}149const wikiTitle = (url: string | undefined) => (url ? decodeURIComponent(url.slice(url.lastIndexOf("/wiki/") + 6)).replace(/_/g, " ") : null);150151const tmdb = async (_imdb: string): Promise<{ poster?: string; backdrop?: string; overview?: string } | null> => null; // artwork and French synopses come from the dedicated TMDB pass (tmdb.ts)152void ((_: string) => _);153154// --- import ---------------------------------------------------------------------------------------------------------155interface Candidate { qid: string; ia: Set<string>; commons: Set<string>; jurisdictions: Set<string>; licence: string | null }156interface Details {157 fr?: string; en?: string; orig?: string; year?: number; duration?: number; image?: string; directors: string[]; cast: string[]; genres: string[]; langs: string[]; countries: string[];158 frw?: string; enw?: string; types: string[]; imdb?: string; juris: string[]; licence?: string;159}160const DETAILS = (ids: string[]) => `SELECT ?f ?fr ?en ?orig (MIN(YEAR(?d)) AS ?year) (SAMPLE(?dur) AS ?duration) (SAMPLE(?img) AS ?image) (SAMPLE(?imdbId) AS ?imdb)161 (GROUP_CONCAT(DISTINCT ?dirL;separator="|") AS ?directors) (GROUP_CONCAT(DISTINCT ?castL;separator="|") AS ?cast) (GROUP_CONCAT(DISTINCT ?genreL;separator="|") AS ?genres)162 (GROUP_CONCAT(DISTINCT ?langC;separator="|") AS ?langs) (GROUP_CONCAT(DISTINCT ?ctryC;separator="|") AS ?countries) (SAMPLE(?frwiki) AS ?frw) (SAMPLE(?enwiki) AS ?enw)163 (GROUP_CONCAT(DISTINCT ?typeId;separator="|") AS ?types) WHERE {164 VALUES ?f { ${ids.map((i) => "wd:" + i).join(" ")} }165 OPTIONAL { ?f rdfs:label ?fr FILTER(LANG(?fr)="fr") } OPTIONAL { ?f rdfs:label ?en FILTER(LANG(?en)="en") } OPTIONAL { ?f wdt:P1476 ?orig }166 OPTIONAL { ?f wdt:P577 ?d } OPTIONAL { ?f wdt:P2047 ?dur } OPTIONAL { ?f wdt:P18 ?img } OPTIONAL { ?f wdt:P345 ?imdbId }167 OPTIONAL { ?f wdt:P57 ?dir . ?dir rdfs:label ?dirL FILTER(LANG(?dirL)="en") }168 OPTIONAL { ?f wdt:P161 ?c . ?c rdfs:label ?castL FILTER(LANG(?castL)="en") }169 OPTIONAL { ?f wdt:P136 ?g . ?g rdfs:label ?genreL FILTER(LANG(?genreL)="fr") }170 OPTIONAL { ?f wdt:P364 ?l . ?l wdt:P218 ?langC } OPTIONAL { ?f wdt:P495 ?ct . ?ct wdt:P297 ?ctryC }171 OPTIONAL { ?frwiki schema:about ?f ; schema:isPartOf <https://fr.wikipedia.org/> } OPTIONAL { ?enwiki schema:about ?f ; schema:isPartOf <https://en.wikipedia.org/> }172 OPTIONAL { ?f wdt:P31 ?type BIND(STRAFTER(STR(?type),"entity/") AS ?typeId) }173 } GROUP BY ?f ?fr ?en ?orig`;174175// Kept light on purpose: the public SPARQL endpoint cuts a response off after 60 s. Direct « instance of » values instead176// of a subclass walk, one query per (host, rights) pair; jurisdiction and licence names come with the details.177const FILM_TYPES = "wd:Q11424 wd:Q24862 wd:Q202866 wd:Q17517379 wd:Q226730 wd:Q24869 wd:Q93204 wd:Q506240 wd:Q20650540 wd:Q29168811 wd:Q336144 wd:Q18011172";178const SERIES_TYPES = "wd:Q5398426 wd:Q1259759 wd:Q526877 wd:Q581714 wd:Q15416";179180async function candidates(types: string, withCommons: boolean): Promise<Map<string, Candidate>> {181 const out = new Map<string, Candidate>();182 const add = (rows: Binding[]) => {183 for (const b of rows) {184 const id = qid(b.f?.value); if (!id) continue;185 let c = out.get(id); if (!c) out.set(id, (c = { qid: id, ia: new Set(), commons: new Set(), jurisdictions: new Set(), licence: null }));186 if (b.ia) c.ia.add(b.ia.value); if (b.video) c.commons.add(b.video.value);187 }188 };189 const cls = `VALUES ?type { ${types} } ?f wdt:P31 ?type .`;190 for (const [prop, v] of [["P724", "ia"], ...(withCommons ? [["P10", "video"]] : [])] as const) {191 add(await sparql(`SELECT DISTINCT ?f ?${v} WHERE { ?f wdt:P6216 wd:Q19652 ; wdt:${prop} ?${v} . ${cls} }`));192 add(await sparql(`SELECT DISTINCT ?f ?${v} WHERE { ?f wdt:P275 ?lic ; wdt:${prop} ?${v} . ?lic wdt:P31/wdt:P279* wd:Q196294 . ${cls} }`));193 }194 return out;195}196197export async function details(ids: string[]): Promise<Map<string, Details>> {198 const out = new Map<string, Details>();199 const list = (b: Binding, k: string) => (b[k]?.value ? b[k]!.value.split("|").filter(Boolean) : []);200 for (let i = 0; i < ids.length; i += 60) {201 for (const b of await sparql(DETAILS(ids.slice(i, i + 60)))) {202 const id = qid(b.f?.value); const prev = out.get(id);203 const d: Details = {204 fr: b.fr?.value, en: b.en?.value, orig: b.orig?.value, year: b.year ? Number(b.year.value) : undefined, duration: b.duration ? Number(b.duration.value) : undefined,205 image: b.image?.value, directors: list(b, "directors"), cast: list(b, "cast"), genres: list(b, "genres"), langs: list(b, "langs"), countries: list(b, "countries"),206 frw: b.frw?.value, enw: b.enw?.value, types: list(b, "types"), imdb: b.imdb?.value, juris: list(b, "juris"), licence: b.licence?.value,207 };208 out.set(id, prev ? { ...d, orig: prev.orig ?? d.orig } : d); // several original titles produce several rows: keep the first209 }210 await sleep(400);211 }212 // Rights wording (jurisdiction of the public-domain statement, licence name): a separate light query, otherwise the213 // optional joins above multiply and the endpoint times out.214 for (let i = 0; i < ids.length; i += 200) {215 const v = ids.slice(i, i + 200).map((x) => "wd:" + x).join(" ");216 for (const b of await sparql(`SELECT ?f ?jL ?licL WHERE { VALUES ?f { ${v} } { ?f p:P6216 ?st . ?st ps:P6216 wd:Q19652 ; pq:P1001 ?j . ?j rdfs:label ?jL FILTER(LANG(?jL)="fr") } UNION { ?f wdt:P275 ?lic . ?lic rdfs:label ?licL FILTER(LANG(?licL)="en") } }`)) {217 const d = out.get(qid(b.f?.value)); if (!d) continue;218 if (b.jL && !d.juris.includes(b.jL.value)) d.juris.push(b.jL.value); if (b.licL) d.licence = b.licL.value;219 }220 await sleep(300);221 }222 return out;223}224225const commonsImage = (u: string | undefined, width: number) => (u ? `${u.replace(/^http:/, "https:")}?width=${width}` : null);226const ANIMATED = new Set(["Q202866", "Q17517379", "Q29168811", "Q581714"]);227const DOCU = new Set(["Q93204", "Q431289", "Q18089587"]);228229function rightsOf(c: Candidate, iaLicence?: string): { status: string; basis: string } {230 if (c.licence) return { status: "cc", basis: `Licence libre selon Wikidata : ${c.licence}` };231 const where = [...c.jurisdictions].join(", ");232 const ia = rightsFromLicence(iaLicence);233 return { status: "pd", basis: `Domaine public selon Wikidata${where ? ` (${where})` : ""}${ia ? ` ; mention « ${ia.label} » sur Internet Archive` : ""}` };234}235236export interface VodResult { tmdb?: { examined: number; matched: number; posters: number; synopses: number }; onf?: { listed: number; checked: number; added: number; notFree: number; errors: number; remaining: number }; films: number; series: number; episodes: number; added: number; updated: number; skippedNoVideo: number; skippedMismatch: number; skippedAdult: number; deactivated: number; errors: string[]; durationMs: number }237238interface Upsert {239 kind: "movie" | "series"; origin: string; originId: string; title: string; original?: string | null; year?: number | null; synopsis?: string | null; synopsisSource?: string | null; runtime?: number | null;240 genres: string[]; directors: string[]; cast: string[]; languages: string[]; countries: string[]; poster?: string | null; backdrop?: string | null; documentary: boolean; animation: boolean; short: boolean;241 rightsStatus: string; rightsBasis: string; rightsUrl?: string | null; sourceName: string; sourceUrl?: string | null; imdb?: string | null; wikidata?: string | null; libraryDir?: number | null;242}243export function upsertTitle(t: Upsert): { id: number; created: boolean } {244 const prev = get<{ id: number }>("SELECT id FROM titles WHERE origin = ? AND origin_id = ?", t.origin, t.originId);245 const vals = [t.kind, t.title, t.original ?? null, t.year ?? null, t.synopsis ?? null, t.synopsisSource ?? null, t.runtime ?? null, t.genres.join(";"), t.directors.slice(0, 6).join(";"), t.cast.slice(0, 12).join(";"),246 t.languages.join(";"), t.countries.join(";"), t.poster ?? null, t.backdrop ?? null, Number(t.documentary), Number(t.animation), Number(t.short), t.rightsStatus, t.rightsBasis, t.rightsUrl ?? null,247 t.sourceName, t.sourceUrl ?? null, t.imdb ?? null, t.wikidata ?? null, t.libraryDir ?? null, now()];248 if (prev) {249 run(`UPDATE titles SET kind=?, title=?, original_title=?, year=?, synopsis=COALESCE(?, synopsis), synopsis_source=COALESCE(?, synopsis_source), runtime_sec=?, genres=?, directors=?, cast_names=?, languages=?, countries=?,250 poster=COALESCE(?, poster), backdrop=COALESCE(?, backdrop), is_documentary=?, is_animation=?, is_short=?, rights_status=?, rights_basis=?, rights_url=?, source_name=?, source_url=?, imdb_id=?, wikidata_id=?, library_dir=?,251 active=1, updated_at=?, checked_at=? WHERE id=?`, ...vals, now(), prev.id);252 return { id: prev.id, created: false };253 }254 const r = run(`INSERT INTO titles(kind, title, original_title, year, synopsis, synopsis_source, runtime_sec, genres, directors, cast_names, languages, countries, poster, backdrop, is_documentary, is_animation, is_short,255 rights_status, rights_basis, rights_url, source_name, source_url, imdb_id, wikidata_id, library_dir, updated_at, checked_at, added_at, origin, origin_id) VALUES(${"?,".repeat(29)}?)`, ...vals, now(), now(), t.origin, t.originId);256 return { id: Number(r.lastInsertRowid), created: true };257}258export function setMedia(titleId: number, episodeId: number | null, m: { kind: "remote" | "file" | "embed"; url?: string; path?: string; container?: string | null; vcodec?: string | null; acodec?: string | null; width?: number | null; height?: number | null; duration?: number | null; size?: number | null; mtime?: number | null; playback?: string; subtitles?: unknown[]; audio?: unknown[] }): number {259 const prev = m.path ? get<{ id: number }>("SELECT id FROM media WHERE path = ?", m.path)260 : get<{ id: number }>(`SELECT id FROM media WHERE title_id = ? AND episode_id ${episodeId === null ? "IS NULL" : "= ?"}`, ...(episodeId === null ? [titleId] : [titleId, episodeId]));261 const vals = [titleId, episodeId, m.kind, m.url ?? null, m.path ?? null, m.container ?? null, m.vcodec ?? null, m.acodec ?? null, m.width ?? null, m.height ?? null, m.duration ?? null, m.size ?? null, m.mtime ?? null,262 m.playback ?? "direct", JSON.stringify(m.subtitles ?? []), JSON.stringify(m.audio ?? [])];263 // Updating in place keeps the media id, hence the playback position, across re-imports.264 if (prev) { run("UPDATE media SET title_id=?, episode_id=?, kind=?, url=?, path=?, container=?, vcodec=?, acodec=?, width=?, height=?, duration_sec=?, size_bytes=?, mtime=?, playback=?, subtitles=?, audio=?, error=NULL WHERE id=?", ...vals, prev.id); return prev.id; }265 return Number(run("INSERT INTO media(title_id, episode_id, kind, url, path, container, vcodec, acodec, width, height, duration_sec, size_bytes, mtime, playback, subtitles, audio, added_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ...vals, now()).lastInsertRowid);266}267export function upsertEpisode(titleId: number, season: number, episode: number, name: string | null, synopsis: string | null = null, airDate: string | null = null): number {268 run("INSERT INTO episodes(title_id, season, episode, name, synopsis, air_date) VALUES(?,?,?,?,?,?) ON CONFLICT(title_id, season, episode) DO UPDATE SET name = excluded.name, synopsis = COALESCE(excluded.synopsis, synopsis), air_date = COALESCE(excluded.air_date, air_date)", titleId, season, episode, name, synopsis, airDate);269 return get<{ id: number }>("SELECT id FROM episodes WHERE title_id = ? AND season = ? AND episode = ?", titleId, season, episode)!.id;270}271272function base(c: Candidate, d: Details | undefined, fallbackTitle: string) {273 const d0 = d ?? { directors: [], cast: [], genres: [], langs: [], countries: [], types: [], juris: [] } as Details;274 for (const j of d0.juris) c.jurisdictions.add(j);275 if (d0.licence) c.licence = d0.licence;276 const genres = mapGenres(d0.genres);277 const animation = genres.includes("Animation") || d0.types.some((t) => ANIMATED.has(t));278 const documentary = genres.includes("Documentaire") || d0.types.some((t) => DOCU.has(t));279 if (animation && !genres.includes("Animation")) genres.push("Animation");280 if (documentary && !genres.includes("Documentaire")) genres.push("Documentaire");281 return { d0, genres, animation, documentary, title: d0.fr || d0.en || d0.orig || fallbackTitle, original: d0.orig && d0.orig !== (d0.fr || d0.en) ? d0.orig : d0.en && d0.en !== d0.fr ? d0.en : null };282}283284/** Parse « S01E02 », « 1x02 », « Episode 3 », « Chapter 4 », trailing numbers. */285export function episodeNumber(name: string): { season: number; episode: number } | null {286 const n = name.replace(/\.[a-z0-9]{2,4}$/i, "");287 let m = /s(\d{1,2})[ ._-]?e(\d{1,3})/i.exec(n); if (m) return { season: +m[1], episode: +m[2] };288 m = /(?:^|[^\d])(\d{1,2})x(\d{1,3})(?:[^\d]|$)/i.exec(n); if (m) return { season: +m[1], episode: +m[2] };289 m = /(?:ep(?:isode)?|chapter|chapitre|part|partie|ch)[ ._-]*(\d{1,3})/i.exec(n); if (m) return { season: 1, episode: +m[1] };290 m = /(\d{1,3})(?:_512kb)?$/.exec(n.trim()); if (m) return { season: 1, episode: +m[1] };291 return null;292}293294// Reviewed series that Wikidata does not flag with a copyright status (see docs/SOURCES.md for the evidence).295export const CURATED_SERIES = [296 { ia: "pioneer.one.season1.720p.x264-vodo", qid: "Q639740", title: "Pioneer One", status: "cc", basis: "Creative Commons BY-NC-SA : série financée par ses spectateurs et diffusée gratuitement par ses auteurs (VODO) ; licence indiquée sur Internet Archive" },297];298299export async function importVod(shouldStop: () => boolean = () => false): Promise<VodResult> {300 const t0 = now();301 const res: VodResult = { films: 0, series: 0, episodes: 0, added: 0, updated: 0, skippedNoVideo: 0, skippedMismatch: 0, skippedAdult: 0, deactivated: 0, errors: [], durationMs: 0 };302 const started = now();303 const builtin = (kind: string, name: string, ref: string, as: string, note: string) =>304 run("INSERT INTO vod_sources(kind, name, ref, as_kind, rights_note, builtin, created_at) VALUES(?,?,?,?,?,1,?) ON CONFLICT(kind, ref) DO NOTHING", kind, name, ref, as, note, now());305 builtin("wikidata", "Films libres — Wikidata × Internet Archive / Wikimedia Commons", "films", "movie", "Statut « domaine public » ou licence libre documentés sur Wikidata ; vidéo servie par Internet Archive ou Wikimedia Commons");306 builtin("wikidata", "Séries et feuilletons libres — Wikidata × Internet Archive", "series", "series", "Même règle que les films, plus une courte liste vérifiée à la main");307 const enabled = (ref: string) => !!get("SELECT 1 FROM vod_sources WHERE kind = 'wikidata' AND ref = ? AND enabled = 1", ref);308 const done = (kind: string, ref: string, n: number, err?: string) => run("UPDATE vod_sources SET last_sync_at = ?, last_status = ?, last_error = ?, title_count = ? WHERE kind = ? AND ref = ?", now(), err ? "error" : "ok", err ?? null, n, kind, ref);309310 // ---- films311 if (enabled("films")) try {312 const cands = await candidates(FILM_TYPES, true);313 log("info", "vod", `Films : ${cands.size} candidats aux droits documentés`);314 const det = await details([...cands.keys()]);315 log("info", "vod", `Films : métadonnées Wikidata reçues (${det.size})`);316 // Synopses: French Wikipedia first, English otherwise.317 const frT = new Map<string, string>(), enT = new Map<string, string>();318 for (const [id, d] of det) { const f = wikiTitle(d.frw), e = wikiTitle(d.enw); if (f) frT.set(id, f); else if (e) enT.set(id, e); }319 const known = new Set(all<{ origin_id: string }>("SELECT origin_id FROM titles WHERE origin = 'wikidata' AND synopsis IS NOT NULL").map((r) => r.origin_id));320 const frX = await extracts("fr", [...frT].filter(([id]) => !known.has(id)).map(([, t]) => t));321 const enX = await extracts("en", [...enT].filter(([id]) => !known.has(id)).map(([, t]) => t));322 log("info", "vod", `Films : synopsis Wikipédia reçus (${frX.size} fr, ${enX.size} en) ; vérification des sources vidéo`);323 let seen = 0;324 for (const c of cands.values()) {325 if (++seen % 250 === 0) log("info", "vod", `Films : ${seen}/${cands.size} vérifiés, ${res.films} retenus`);326 const b = base(c, det.get(c.qid), c.qid);327 if (b.d0.genres.some((g) => ADULT.test(g))) { res.skippedAdult++; continue; }328 const existing = get<{ id: number; checked_at: number | null }>("SELECT id, checked_at FROM titles WHERE origin = 'wikidata' AND origin_id = ?", c.qid);329 // A title already verified in the last 30 days keeps its media; only its metadata is refreshed.330 let pick: MediaPick | null = null, iaId: string | null = null, iaLicence: string | undefined, commons: string | null = null;331 const fresh = existing?.checked_at && now() - existing.checked_at < 30 * 86400_000 && get("SELECT 1 FROM media WHERE title_id = ?", existing.id);332 if (!fresh) {333 let bestScore = 0;334 for (const id of c.ia) {335 const item = await iaItem(id); if (!item || item.metadata?.mediatype !== "movies") continue;336 const score = iaMatch(item, [b.d0.fr, b.d0.en, b.d0.orig], b.d0.year); if (score <= bestScore) { if (!score) res.skippedMismatch++; continue; }337 const vids = pickVideos(id, item); if (!vids.length) continue;338 pick = vids.sort((x, y) => (y.duration ?? 0) - (x.duration ?? 0))[0]; iaId = id; iaLicence = item.metadata?.licenseurl; bestScore = score;339 }340 if (!pick) commons = [...c.commons].find((u) => /\.webm$/i.test(decodeURIComponent(u))) ?? null;341 if (!pick && !commons) { res.skippedNoVideo++; if (existing) run("UPDATE titles SET active = 0, checked_at = ? WHERE id = ?", now(), existing.id); continue; }342 }343 const d = b.d0; const rights = rightsOf(c, iaLicence);344 const runtime = pick?.duration ?? (d.duration ? Math.round(d.duration * 60) : null);345 const extra = d.imdb ? await tmdb(d.imdb) : null;346 const synopsis = extra?.overview ?? frX.get(frT.get(c.qid) ?? "") ?? enX.get(enT.get(c.qid) ?? "") ?? null;347 const u = upsertTitle({348 kind: "movie", origin: "wikidata", originId: c.qid, title: b.title, original: b.original, year: d.year ?? null, synopsis,349 synopsisSource: synopsis ? (extra?.overview ? "TMDB" : frX.has(frT.get(c.qid) ?? "") ? "Wikipédia (fr)" : "Wikipedia (en)") : null, runtime,350 genres: b.genres, directors: d.directors, cast: d.cast, languages: d.langs, countries: d.countries,351 poster: extra?.poster ?? commonsImage(d.image, 500) ?? (iaId ? `https://archive.org/services/img/${encodeURIComponent(iaId)}` : null), backdrop: extra?.backdrop ?? commonsImage(d.image, 1280),352 documentary: b.documentary, animation: b.animation, short: !!runtime && runtime < 40 * 60,353 rightsStatus: rights.status, rightsBasis: rights.basis, rightsUrl: `https://www.wikidata.org/wiki/${c.qid}`,354 sourceName: fresh ? get<{ source_name: string }>("SELECT source_name FROM titles WHERE id = ?", existing!.id)!.source_name : pick ? "Internet Archive" : "Wikimedia Commons",355 sourceUrl: fresh ? get<{ source_url: string | null }>("SELECT source_url FROM titles WHERE id = ?", existing!.id)!.source_url : iaId ? `https://archive.org/details/${encodeURIComponent(iaId)}` : commons ? commons.replace(/^http:/, "https:").replace("Special:FilePath/", "File:") : null,356 imdb: d.imdb, wikidata: c.qid,357 });358 if (pick) setMedia(u.id, null, { kind: "remote", url: pick.url, container: "mp4", vcodec: "h264", height: pick.height, duration: pick.duration, size: pick.size, subtitles: pick.subtitles });359 else if (commons) setMedia(u.id, null, { kind: "remote", url: commons.replace(/^http:/, "https:"), container: "webm", duration: runtime });360 if (u.created) res.added++; else res.updated++;361 res.films++;362 }363 // Second pass for the synopses a rate limit made us skip: the verification above took long enough for it to lift.364 const missing = all<{ origin_id: string }>("SELECT origin_id FROM titles WHERE origin = 'wikidata' AND kind = 'movie' AND active = 1 AND synopsis IS NULL").map((r) => r.origin_id);365 for (const [lang, map, src] of [["fr", frT, "Wikipédia (fr)"], ["en", enT, "Wikipedia (en)"]] as const) {366 const want = missing.filter((id) => map.has(id)); if (!want.length) continue;367 const x = await extracts(lang, want.map((id) => map.get(id)!));368 for (const id of want) { const text = x.get(map.get(id)!); if (text) run("UPDATE titles SET synopsis = ?, synopsis_source = ? WHERE origin = 'wikidata' AND origin_id = ?", text, src, id); }369 }370 res.deactivated += Number(run("UPDATE titles SET active = 0 WHERE origin = 'wikidata' AND kind = 'movie' AND active = 1 AND updated_at < ?", started).changes);371 done("wikidata", "films", res.films);372 } catch (e) { res.errors.push(`films : ${(e as Error).message}`); done("wikidata", "films", res.films, (e as Error).message); log("error", "vod", "Import des films en échec", { error: (e as Error).message }); }373374 // ---- series375 if (enabled("series")) try {376 const cands = await candidates(SERIES_TYPES, false);377 for (const s of CURATED_SERIES) if (!cands.has(s.qid)) cands.set(s.qid, { qid: s.qid, ia: new Set([s.ia]), commons: new Set(), jurisdictions: new Set(), licence: null });378 const det = await details([...cands.keys()]);379 const frX = await extracts("fr", [...det.values()].map((d) => wikiTitle(d.frw)).filter((t): t is string => !!t));380 const enX = await extracts("en", [...det.values()].map((d) => wikiTitle(d.enw)).filter((t): t is string => !!t));381 for (const c of cands.values()) {382 const curated = CURATED_SERIES.find((s) => s.qid === c.qid);383 const b = base(c, det.get(c.qid), curated?.title ?? c.qid); const d = b.d0;384 if (d.genres.some((g) => ADULT.test(g))) { res.skippedAdult++; continue; }385 // Episodes: every video of a multi-file item, or the numbered sibling items of a serial (« name1 », « name2 », …).386 const eps: { season: number; episode: number; name: string; pick: MediaPick }[] = [];387 let iaLicence: string | undefined, firstId: string | null = null;388 for (const id of c.ia) {389 const item = await iaItem(id); if (!item) continue;390 firstId ??= id; iaLicence ??= item.metadata?.licenseurl;391 const vids = pickVideos(id, item);392 if (vids.length > 1) vids.forEach((v, i) => { const n = episodeNumber(v.name.split("/").pop()!) ?? { season: 1, episode: i + 1 }; eps.push({ ...n, name: v.name.split("/").pop()!.replace(/\.(mp4|m4v)$/i, "").replace(/[._]+/g, " ").trim(), pick: v }); });393 else if (vids.length === 1) {394 const m = /^(.*?)(\d+)$/.exec(id);395 const siblings: string[] = [];396 if (m) {397 try {398 const r = await getJson<{ response: { docs: { identifier: string }[] } }>(`https://archive.org/advancedsearch.php?q=${encodeURIComponent(`identifier:${m[1]}*`)}&fl[]=identifier&rows=200&output=json`);399 siblings.push(...r.response.docs.map((x) => x.identifier).filter((x) => new RegExp(`^${m[1].replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\d+$`).test(x)));400 } catch { /* keep the single item */ }401 }402 if (siblings.length > 1) {403 for (const sid of siblings.sort((x, y) => Number(/\d+$/.exec(x)![0]) - Number(/\d+$/.exec(y)![0]))) {404 const it = await iaItem(sid); const v = it ? pickVideos(sid, it)[0] : null; if (!v || !it) continue;405 eps.push({ season: 1, episode: Number(/\d+$/.exec(sid)![0]), name: (it.metadata?.title ?? sid).replace(/^.*?:\s*/, "").trim(), pick: v });406 }407 } else eps.push({ season: 1, episode: eps.length + 1, name: item.metadata?.title ?? id, pick: vids[0] });408 }409 }410 const uniq = new Map(eps.map((e) => [`${e.season}x${e.episode}`, e]));411 if (uniq.size < 2) { res.skippedNoVideo++; continue; } // a « series » with a single video is not presented as one412 const rights = curated ? { status: curated.status, basis: curated.basis } : rightsOf(c, iaLicence);413 const extra = d.imdb ? await tmdb(d.imdb) : null;414 const synopsis = extra?.overview ?? frX.get(wikiTitle(d.frw) ?? "") ?? enX.get(wikiTitle(d.enw) ?? "") ?? null;415 const u = upsertTitle({416 kind: "series", origin: "wikidata", originId: c.qid, title: b.title, original: b.original, year: d.year ?? null, synopsis, synopsisSource: synopsis ? (extra?.overview ? "TMDB" : frX.has(wikiTitle(d.frw) ?? "") ? "Wikipédia (fr)" : "Wikipedia (en)") : null,417 runtime: null, genres: b.genres, directors: d.directors, cast: d.cast, languages: d.langs, countries: d.countries,418 poster: extra?.poster ?? commonsImage(d.image, 500) ?? (firstId ? `https://archive.org/services/img/${encodeURIComponent(firstId)}` : null), backdrop: extra?.backdrop ?? commonsImage(d.image, 1280),419 documentary: b.documentary, animation: b.animation, short: false, rightsStatus: rights.status, rightsBasis: rights.basis, rightsUrl: `https://www.wikidata.org/wiki/${c.qid}`,420 sourceName: "Internet Archive", sourceUrl: firstId ? `https://archive.org/details/${encodeURIComponent(firstId)}` : null, imdb: d.imdb, wikidata: c.qid,421 });422 tx(() => {423 const keep: number[] = [];424 for (const e of uniq.values()) {425 const eid = upsertEpisode(u.id, e.season, e.episode, e.name); keep.push(eid);426 setMedia(u.id, eid, { kind: "remote", url: e.pick.url, container: "mp4", vcodec: "h264", height: e.pick.height, duration: e.pick.duration, size: e.pick.size, subtitles: e.pick.subtitles });427 }428 run(`DELETE FROM episodes WHERE title_id = ? AND id NOT IN (${keep.map(() => "?").join(",")})`, u.id, ...keep);429 });430 if (u.created) res.added++; else res.updated++;431 res.series++; res.episodes += uniq.size;432 }433 res.deactivated += Number(run("UPDATE titles SET active = 0 WHERE origin = 'wikidata' AND kind = 'series' AND active = 1 AND updated_at < ?", started).changes);434 done("wikidata", "series", res.series);435 } catch (e) { res.errors.push(`séries : ${(e as Error).message}`); done("wikidata", "series", res.series, (e as Error).message); log("error", "vod", "Import des séries en échec", { error: (e as Error).message }); }436437 bumpTitles();438 // ---- Office national du film (official embedded player, free films only)439 builtin("onf", "ONF — Office national du film du Canada (lecteur officiel intégré)", "films", "movie", "Intégration autorisée par l'ONF pour un usage personnel et non commercial, avec lien vers onf.ca ; seuls les films offerts gratuitement et en entier par le lecteur intégré sont listés");440 if (get("SELECT 1 FROM vod_sources WHERE kind = 'onf' AND enabled = 1")) try {441 const { importOnf } = await import("./onf");442 const o = await importOnf(4000, shouldStop);443 res.onf = o; done("onf", "films", get<{ n: number }>("SELECT COUNT(*) n FROM titles t WHERE origin = 'onf' AND active = 1 AND EXISTS(SELECT 1 FROM media m WHERE m.title_id = t.id)")!.n);444 } catch (e) { res.errors.push(`ONF : ${(e as Error).message}`); done("onf", "films", 0, (e as Error).message); log("error", "vod", "Import ONF en échec", { error: (e as Error).message }); }445 if (env.tmdbKey) try { const { enrichTmdb } = await import("./tmdb"); res.tmdb = await enrichTmdb(); } catch (e) { res.errors.push(`TMDB : ${(e as Error).message}`); }446 res.durationMs = now() - t0;447 bumpTitles();448 log(res.errors.length ? "warn" : "info", "vod", `Catalogue films et séries : ${res.films} films, ${res.series} séries (${res.episodes} épisodes)`, res);449 return res;450}451452/** Looks a personal file up on Wikidata by title (and year) to borrow its poster, synopsis and credits. Returns null when nothing matches confidently. */453export async function enrichment(title: string, year: number | null, kind: "movie" | "series") {454 const cls = kind === "movie" ? "wd:Q11424" : "wd:Q5398426";455 const rows = await sparql(`SELECT ?f (MIN(YEAR(?d)) AS ?y) WHERE { SERVICE wikibase:mwapi { bd:serviceParam wikibase:endpoint "www.wikidata.org" ; wikibase:api "EntitySearch" ; mwapi:search ${JSON.stringify(title)} ; mwapi:language "fr" . ?f wikibase:apiOutputItem mwapi:item . }456 ?f wdt:P31/wdt:P279* ${cls} . OPTIONAL { ?f wdt:P577 ?d } OPTIONAL { ?f wdt:P580 ?d } } GROUP BY ?f LIMIT 8`);457 const hits = rows.map((b) => ({ id: qid(b.f?.value), y: b.y ? Number(b.y.value) : null }));458 const hit = year ? hits.find((h) => h.y !== null && Math.abs(h.y - year) <= 1) : hits.length === 1 ? hits[0] : null;459 if (!hit) return null;460 const d = (await details([hit.id])).get(hit.id); if (!d) return null;461 const c: Candidate = { qid: hit.id, ia: new Set(), commons: new Set(), jurisdictions: new Set(), licence: null };462 const b = base(c, d, title);463 const fr = wikiTitle(d.frw), en = wikiTitle(d.enw);464 const x = fr ? await extracts("fr", [fr]) : en ? await extracts("en", [en]) : new Map<string, string>();465 const extra = d.imdb ? await tmdb(d.imdb) : null;466 const synopsis = extra?.overview ?? x.get(fr ?? en ?? "") ?? null;467 return {468 title: b.title, original: b.original, year: d.year ?? year, synopsis, synopsisSource: synopsis ? (extra?.overview ? "TMDB" : fr ? "Wikipédia (fr)" : "Wikipedia (en)") : null, genres: b.genres, directors: d.directors, cast: d.cast,469 languages: d.langs, countries: d.countries, poster: extra?.poster ?? commonsImage(d.image, 500), backdrop: extra?.backdrop ?? commonsImage(d.image, 1280), documentary: b.documentary, animation: b.animation, imdb: d.imdb ?? null, wikidata: hit.id,470 };471}472