SPB Git

Repository Public

vod.ts 39.3 KB 472 lines typescript

1import { all, bumpTitles, get, log, now, run, tx } from "./db";2import { env } from "./env";3import { fetchText } from "./ssrf";45// Films and series that may legally be watched, with the reason recorded for every title.6//7// A title is imported only when an independent source documents its rights:8//   - Wikidata says « public domain » (P6216) or gives a free licence (P275), AND9//   - the video is hosted by Internet Archive (P724) or Wikimedia Commons (P10), which serve it directly to the browser.10// Whole Internet Archive collections are NOT imported: they contain many uploads whose rights are wrong or unknown.11// Series: Wikidata titles with the same rule, plus a short reviewed list (docs/SOURCES.md), plus what the administrator adds.1213const UA = "SP-TV/1.0 (personal media catalogue; https://tv.spboucher.ai)";14const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));1516async function getJson<T>(url: string, limitMb = 30): Promise<T> { return JSON.parse(await fetchText(url, limitMb * 1024 * 1024, 120_000)) as T; }1718type Binding = Record<string, { value: string } | undefined>;19async function sparql(query: string): Promise<Binding[]> {20  const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`;21  for (let attempt = 0; ; attempt++) {22    try { return (await getJson<{ results: { bindings: Binding[] } }>(url, 60)).results.bindings; }23    catch (e) { if (attempt >= 2) throw e; await sleep(5000 * (attempt + 1)); }24  }25}26const qid = (uri: string | undefined) => uri?.slice(uri.lastIndexOf("/") + 1) ?? "";2728// --- genres ---------------------------------------------------------------------------------------------------------29const GENRES: [RegExp, string][] = [30  [/documentaire/i, "Documentaire"], [/science-fiction/i, "Science-fiction"], [/anim|dessin animé/i, "Animation"], [/horreur|épouvante|monstre|gothique|vampire|zombie/i, "Horreur"],31  [/western/i, "Western"], [/noir|criminel|policier|énigme|gangster|détective/i, "Policier"], [/thriller|suspense|espionnage/i, "Thriller"], [/guerre|propagande/i, "Guerre"],32  [/musical|musique|opéra/i, "Musical"], [/amour|romant/i, "Romance"], [/fantasy|fantast|merveilleux|conte/i, "Fantastique"], [/aventure|cape et d'épée|pirate/i, "Aventure"],33  [/action|arts martiaux/i, "Action"], [/comédie|comique|burlesque|slapstick|parodie|satir/i, "Comédie"], [/drame|dramatique|mélodrame|tragédie|procès/i, "Drame"],34  [/histor|biograph|péplum/i, "Histoire"], [/familial|enfants|jeunesse/i, "Famille"], [/muet/i, "Muet"], [/expérimental|avant-garde|d'art/i, "Expérimental"], [/sport/i, "Sport"],35];36const ADULT = /pornograph|érotique|hentai|sexploitation/i;37export function mapGenres(labels: string[]): string[] {38  const out = new Set<string>();39  for (const l of labels) for (const [re, g] of GENRES) if (re.test(l)) out.add(g);40  return [...out];41}4243// --- Internet Archive -----------------------------------------------------------------------------------------------44interface IaFile { name: string; format?: string; source?: string; length?: string; height?: string; width?: string; size?: string }45interface IaItem { metadata?: { title?: string; licenseurl?: string; mediatype?: string; description?: string | string[]; year?: string; date?: string; "access-restricted-item"?: string }; files?: IaFile[]; is_dark?: boolean }4647const STOP = new Set(["the", "a", "an", "of", "and", "le", "la", "les", "de", "du", "des", "un", "une", "et", "el", "los", "der", "die", "das", "in", "to", "film", "movie", "full", "hd"]);48const words = (s: string) => new Set(s.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").split(" ").filter((w) => w.length > 1 && !STOP.has(w) && !/^(19|20)\d\d$/.test(w)));49/**50 * Guards against a wrong Internet Archive identifier on Wikidata (it happens: a namesake, a remake, a trailer).51 * 0 = refuse, higher = better. The item must be about the same work: its year must not contradict the film's, and its52 * title must share its significant words with one of the film's titles.53 */54export function iaMatch(item: IaItem, titles: (string | undefined)[], year: number | undefined): number {55  const t = item.metadata?.title ?? ""; if (!t) return 1;56  const y = Number(/(?:^|[^\d])((?:18|19|20)\d\d)(?:[^\d]|$)/.exec(`${item.metadata?.year ?? ""} ${item.metadata?.date ?? ""} ${t}`)?.[1]);57  if (year && y && Math.abs(y - year) > 2) return 0;58  const iw = words(t); if (!iw.size) return 1;59  let best = 0;60  for (const cand of titles) {61    if (!cand) continue; const cw = words(cand); if (!cw.size) continue;62    const common = [...cw].filter((w) => iw.has(w)).length;63    best = Math.max(best, common / cw.size + (common === cw.size && iw.size === cw.size ? 1 : 0));64  }65  return best >= 0.6 ? 1 + best : 0;66}67export interface MediaPick { url: string; name: string; height: number | null; duration: number | null; size: number | null; subtitles: { label: string; lang: string; url: string }[] }6869const iaCache = new Map<string, IaItem | null>();70let iaLast = 0;71async function iaItem(id: string): Promise<IaItem | null> {72  if (iaCache.has(id)) return iaCache.get(id)!;73  // One request at a time, 4 per second at most.74  const wait = iaLast + 250 - Date.now(); if (wait > 0) await sleep(wait); iaLast = Date.now();75  let item: IaItem | null = null;76  try { item = await getJson<IaItem>(`https://archive.org/metadata/${encodeURIComponent(id)}`, 20); } catch { item = null; }77  if (item && (!item.files || item.is_dark || item.metadata?.["access-restricted-item"] === "true")) item = null;78  iaCache.set(id, item); if (iaCache.size > 300) iaCache.delete(iaCache.keys().next().value!);79  return item;80}81const secs = (s: string | undefined): number | null => {82  if (!s) return null;83  if (/^\d+(\.\d+)?$/.test(s)) return Math.round(Number(s));84  const p = s.split(":").map(Number); if (p.some(Number.isNaN)) return null;85  return Math.round(p.reduce((a, b) => a * 60 + b, 0));86};87const iaUrl = (id: string, name: string) => `https://archive.org/download/${encodeURIComponent(id)}/${name.split("/").map(encodeURIComponent).join("/")}`;88const LANGS: Record<string, string> = { en: "English", fr: "Français", es: "Español", de: "Deutsch", it: "Italiano", pt: "Português", nl: "Nederlands", ru: "Русский", ja: "日本語", zh: "中文", ar: "العربية", pl: "Polski", sv: "Svenska", tr: "Türkçe" };89function subLang(file: string): { lang: string; label: string } {90  const base = file.toLowerCase().replace(/\.(srt|vtt)$/, "");91  if (base.endsWith(".asr")) return { lang: "en", label: "English (automatique)" };92  const m = /[._ -]([a-z]{2,3}|english|french|fran[cç]ais|spanish|german|italian|portuguese)$/.exec(base);93  const raw = m?.[1] ?? "";94  const code = ({ english: "en", eng: "en", french: "fr", fre: "fr", fra: "fr", français: "fr", francais: "fr", spanish: "es", spa: "es", german: "de", ger: "de", deu: "de", italian: "it", ita: "it", portuguese: "pt", por: "pt" } as Record<string, string>)[raw] ?? (raw.length === 2 ? raw : "");95  return code && LANGS[code] ? { lang: code, label: LANGS[code] } : { lang: "und", label: raw ? raw.toUpperCase() : "Sous-titres" };96}97const isMp4 = (f: IaFile) => /\.(mp4|m4v)$/i.test(f.name);98const rank = (f: IaFile) => (/^h\.264/i.test(f.format ?? "") ? 0 : /mpeg4/i.test(f.format ?? "") ? 1 : 2) * 10_000 - Math.min(1080, Number(f.height) || 0);99100/** Browser-playable videos of an item (H.264/MPEG-4 in MP4), one per distinct work, best rendition first. */101export function pickVideos(id: string, item: IaItem): MediaPick[] {102  const files = item.files ?? [];103  const groups = new Map<string, IaFile[]>();104  for (const f of files.filter(isMp4)) {105    const key = f.name.replace(/\.(mp4|m4v)$/i, "").replace(/(_512kb|\.ia|_h264|_hd)$/i, "").toLowerCase();106    const l = groups.get(key); if (l) l.push(f); else groups.set(key, [f]);107  }108  const subs = files.filter((f) => /\.(srt|vtt)$/i.test(f.name));109  return [...groups.entries()].map(([key, list]) => {110    const f = list.sort((a, b) => rank(a) - rank(b))[0];111    const mine = subs.filter((s) => s.name.toLowerCase().startsWith(key) || (groups.size === 1 && !s.name.includes("/")) || s.name.toLowerCase().startsWith(key.split("/")[0] + "/"));112    const seen = new Set<string>();113    return {114      url: iaUrl(id, f.name), name: f.name, height: Number(f.height) || null, duration: secs(f.length), size: Number(f.size) || null,115      subtitles: mine.map((s) => ({ ...subLang(s.name), url: iaUrl(id, s.name) })).filter((s) => (seen.has(s.label) ? false : (seen.add(s.label), true))).slice(0, 40),116    };117  });118}119120function rightsFromLicence(url: string | undefined): { status: string; label: string } | null {121  if (!url) return null;122  const u = url.toLowerCase();123  if (u.includes("publicdomain")) return { status: "pd", label: u.includes("zero") ? "CC0 (domaine public)" : "Domaine public" };124  const m = /creativecommons\.org\/licenses\/([a-z-]+)\/(\d\.\d)/.exec(u);125  return m ? { status: "cc", label: `Creative Commons ${m[1].toUpperCase()} ${m[2]}` } : null;126}127128// --- Wikipedia synopses ---------------------------------------------------------------------------------------------129async function extracts(lang: "fr" | "en", titles: string[]): Promise<Map<string, string>> {130  const out = new Map<string, string>(); let failures = 0;131  for (let i = 0; i < titles.length; i += 20) {132    const batch = titles.slice(i, i + 20);133    try {134      const d = await getJson<{ query?: { normalized?: { from: string; to: string }[]; pages?: Record<string, { title: string; extract?: string }> } }>(135        `https://${lang}.wikipedia.org/w/api.php?action=query&format=json&maxlag=5&prop=extracts&exintro=1&explaintext=1&exlimit=20&redirects=1&titles=${encodeURIComponent(batch.join("|"))}`);136      const back = new Map((d.query?.normalized ?? []).map((n) => [n.to, n.from]));137      for (const p of Object.values(d.query?.pages ?? {})) if (p.extract && p.extract.length > 40) { out.set(p.title, p.extract.trim()); const f = back.get(p.title); if (f) out.set(f, p.extract.trim()); }138      failures = 0;139    } catch (e) {140      // Rate limited: wait, retry this batch once more later, and give up for this run after three refusals in a row141      // (titles without a synopsis are asked again at the next import).142      if (++failures >= 3) { log("warn", "vod", `Synopsis Wikipédia (${lang}) : interrompu après limitation de débit, ${out.size} reçus ; le reste sera demandé au prochain import`, { error: (e as Error).message }); break; }143      await sleep(30_000 * failures); i -= 20; continue;144    }145    await sleep(1100);146  }147  return out;148}149const wikiTitle = (url: string | undefined) => (url ? decodeURIComponent(url.slice(url.lastIndexOf("/wiki/") + 6)).replace(/_/g, " ") : null);150151const tmdb = async (_imdb: string): Promise<{ poster?: string; backdrop?: string; overview?: string } | null> => null; // artwork and French synopses come from the dedicated TMDB pass (tmdb.ts)152void ((_: string) => _);153154// --- import ---------------------------------------------------------------------------------------------------------155interface Candidate { qid: string; ia: Set<string>; commons: Set<string>; jurisdictions: Set<string>; licence: string | null }156interface Details {157  fr?: string; en?: string; orig?: string; year?: number; duration?: number; image?: string; directors: string[]; cast: string[]; genres: string[]; langs: string[]; countries: string[];158  frw?: string; enw?: string; types: string[]; imdb?: string; juris: string[]; licence?: string;159}160const DETAILS = (ids: string[]) => `SELECT ?f ?fr ?en ?orig (MIN(YEAR(?d)) AS ?year) (SAMPLE(?dur) AS ?duration) (SAMPLE(?img) AS ?image) (SAMPLE(?imdbId) AS ?imdb)161 (GROUP_CONCAT(DISTINCT ?dirL;separator="|") AS ?directors) (GROUP_CONCAT(DISTINCT ?castL;separator="|") AS ?cast) (GROUP_CONCAT(DISTINCT ?genreL;separator="|") AS ?genres)162 (GROUP_CONCAT(DISTINCT ?langC;separator="|") AS ?langs) (GROUP_CONCAT(DISTINCT ?ctryC;separator="|") AS ?countries) (SAMPLE(?frwiki) AS ?frw) (SAMPLE(?enwiki) AS ?enw)163 (GROUP_CONCAT(DISTINCT ?typeId;separator="|") AS ?types) WHERE {164  VALUES ?f { ${ids.map((i) => "wd:" + i).join(" ")} }165  OPTIONAL { ?f rdfs:label ?fr FILTER(LANG(?fr)="fr") } OPTIONAL { ?f rdfs:label ?en FILTER(LANG(?en)="en") } OPTIONAL { ?f wdt:P1476 ?orig }166  OPTIONAL { ?f wdt:P577 ?d } OPTIONAL { ?f wdt:P2047 ?dur } OPTIONAL { ?f wdt:P18 ?img } OPTIONAL { ?f wdt:P345 ?imdbId }167  OPTIONAL { ?f wdt:P57 ?dir . ?dir rdfs:label ?dirL FILTER(LANG(?dirL)="en") }168  OPTIONAL { ?f wdt:P161 ?c . ?c rdfs:label ?castL FILTER(LANG(?castL)="en") }169  OPTIONAL { ?f wdt:P136 ?g . ?g rdfs:label ?genreL FILTER(LANG(?genreL)="fr") }170  OPTIONAL { ?f wdt:P364 ?l . ?l wdt:P218 ?langC } OPTIONAL { ?f wdt:P495 ?ct . ?ct wdt:P297 ?ctryC }171  OPTIONAL { ?frwiki schema:about ?f ; schema:isPartOf <https://fr.wikipedia.org/> } OPTIONAL { ?enwiki schema:about ?f ; schema:isPartOf <https://en.wikipedia.org/> }172  OPTIONAL { ?f wdt:P31 ?type BIND(STRAFTER(STR(?type),"entity/") AS ?typeId) }173 } GROUP BY ?f ?fr ?en ?orig`;174175// Kept light on purpose: the public SPARQL endpoint cuts a response off after 60 s. Direct « instance of » values instead176// of a subclass walk, one query per (host, rights) pair; jurisdiction and licence names come with the details.177const FILM_TYPES = "wd:Q11424 wd:Q24862 wd:Q202866 wd:Q17517379 wd:Q226730 wd:Q24869 wd:Q93204 wd:Q506240 wd:Q20650540 wd:Q29168811 wd:Q336144 wd:Q18011172";178const SERIES_TYPES = "wd:Q5398426 wd:Q1259759 wd:Q526877 wd:Q581714 wd:Q15416";179180async function candidates(types: string, withCommons: boolean): Promise<Map<string, Candidate>> {181  const out = new Map<string, Candidate>();182  const add = (rows: Binding[]) => {183    for (const b of rows) {184      const id = qid(b.f?.value); if (!id) continue;185      let c = out.get(id); if (!c) out.set(id, (c = { qid: id, ia: new Set(), commons: new Set(), jurisdictions: new Set(), licence: null }));186      if (b.ia) c.ia.add(b.ia.value); if (b.video) c.commons.add(b.video.value);187    }188  };189  const cls = `VALUES ?type { ${types} } ?f wdt:P31 ?type .`;190  for (const [prop, v] of [["P724", "ia"], ...(withCommons ? [["P10", "video"]] : [])] as const) {191    add(await sparql(`SELECT DISTINCT ?f ?${v} WHERE { ?f wdt:P6216 wd:Q19652 ; wdt:${prop} ?${v} . ${cls} }`));192    add(await sparql(`SELECT DISTINCT ?f ?${v} WHERE { ?f wdt:P275 ?lic ; wdt:${prop} ?${v} . ?lic wdt:P31/wdt:P279* wd:Q196294 . ${cls} }`));193  }194  return out;195}196197export async function details(ids: string[]): Promise<Map<string, Details>> {198  const out = new Map<string, Details>();199  const list = (b: Binding, k: string) => (b[k]?.value ? b[k]!.value.split("|").filter(Boolean) : []);200  for (let i = 0; i < ids.length; i += 60) {201    for (const b of await sparql(DETAILS(ids.slice(i, i + 60)))) {202      const id = qid(b.f?.value); const prev = out.get(id);203      const d: Details = {204        fr: b.fr?.value, en: b.en?.value, orig: b.orig?.value, year: b.year ? Number(b.year.value) : undefined, duration: b.duration ? Number(b.duration.value) : undefined,205        image: b.image?.value, directors: list(b, "directors"), cast: list(b, "cast"), genres: list(b, "genres"), langs: list(b, "langs"), countries: list(b, "countries"),206        frw: b.frw?.value, enw: b.enw?.value, types: list(b, "types"), imdb: b.imdb?.value, juris: list(b, "juris"), licence: b.licence?.value,207      };208      out.set(id, prev ? { ...d, orig: prev.orig ?? d.orig } : d); // several original titles produce several rows: keep the first209    }210    await sleep(400);211  }212  // Rights wording (jurisdiction of the public-domain statement, licence name): a separate light query, otherwise the213  // optional joins above multiply and the endpoint times out.214  for (let i = 0; i < ids.length; i += 200) {215    const v = ids.slice(i, i + 200).map((x) => "wd:" + x).join(" ");216    for (const b of await sparql(`SELECT ?f ?jL ?licL WHERE { VALUES ?f { ${v} } { ?f p:P6216 ?st . ?st ps:P6216 wd:Q19652 ; pq:P1001 ?j . ?j rdfs:label ?jL FILTER(LANG(?jL)="fr") } UNION { ?f wdt:P275 ?lic . ?lic rdfs:label ?licL FILTER(LANG(?licL)="en") } }`)) {217      const d = out.get(qid(b.f?.value)); if (!d) continue;218      if (b.jL && !d.juris.includes(b.jL.value)) d.juris.push(b.jL.value); if (b.licL) d.licence = b.licL.value;219    }220    await sleep(300);221  }222  return out;223}224225const commonsImage = (u: string | undefined, width: number) => (u ? `${u.replace(/^http:/, "https:")}?width=${width}` : null);226const ANIMATED = new Set(["Q202866", "Q17517379", "Q29168811", "Q581714"]);227const DOCU = new Set(["Q93204", "Q431289", "Q18089587"]);228229function rightsOf(c: Candidate, iaLicence?: string): { status: string; basis: string } {230  if (c.licence) return { status: "cc", basis: `Licence libre selon Wikidata : ${c.licence}` };231  const where = [...c.jurisdictions].join(", ");232  const ia = rightsFromLicence(iaLicence);233  return { status: "pd", basis: `Domaine public selon Wikidata${where ? ` (${where})` : ""}${ia ? ` ; mention « ${ia.label} » sur Internet Archive` : ""}` };234}235236export interface VodResult { tmdb?: { examined: number; matched: number; posters: number; synopses: number }; onf?: { listed: number; checked: number; added: number; notFree: number; errors: number; remaining: number }; films: number; series: number; episodes: number; added: number; updated: number; skippedNoVideo: number; skippedMismatch: number; skippedAdult: number; deactivated: number; errors: string[]; durationMs: number }237238interface Upsert {239  kind: "movie" | "series"; origin: string; originId: string; title: string; original?: string | null; year?: number | null; synopsis?: string | null; synopsisSource?: string | null; runtime?: number | null;240  genres: string[]; directors: string[]; cast: string[]; languages: string[]; countries: string[]; poster?: string | null; backdrop?: string | null; documentary: boolean; animation: boolean; short: boolean;241  rightsStatus: string; rightsBasis: string; rightsUrl?: string | null; sourceName: string; sourceUrl?: string | null; imdb?: string | null; wikidata?: string | null; libraryDir?: number | null;242}243export function upsertTitle(t: Upsert): { id: number; created: boolean } {244  const prev = get<{ id: number }>("SELECT id FROM titles WHERE origin = ? AND origin_id = ?", t.origin, t.originId);245  const vals = [t.kind, t.title, t.original ?? null, t.year ?? null, t.synopsis ?? null, t.synopsisSource ?? null, t.runtime ?? null, t.genres.join(";"), t.directors.slice(0, 6).join(";"), t.cast.slice(0, 12).join(";"),246    t.languages.join(";"), t.countries.join(";"), t.poster ?? null, t.backdrop ?? null, Number(t.documentary), Number(t.animation), Number(t.short), t.rightsStatus, t.rightsBasis, t.rightsUrl ?? null,247    t.sourceName, t.sourceUrl ?? null, t.imdb ?? null, t.wikidata ?? null, t.libraryDir ?? null, now()];248  if (prev) {249    run(`UPDATE titles SET kind=?, title=?, original_title=?, year=?, synopsis=COALESCE(?, synopsis), synopsis_source=COALESCE(?, synopsis_source), runtime_sec=?, genres=?, directors=?, cast_names=?, languages=?, countries=?,250      poster=COALESCE(?, poster), backdrop=COALESCE(?, backdrop), is_documentary=?, is_animation=?, is_short=?, rights_status=?, rights_basis=?, rights_url=?, source_name=?, source_url=?, imdb_id=?, wikidata_id=?, library_dir=?,251      active=1, updated_at=?, checked_at=? WHERE id=?`, ...vals, now(), prev.id);252    return { id: prev.id, created: false };253  }254  const r = run(`INSERT INTO titles(kind, title, original_title, year, synopsis, synopsis_source, runtime_sec, genres, directors, cast_names, languages, countries, poster, backdrop, is_documentary, is_animation, is_short,255    rights_status, rights_basis, rights_url, source_name, source_url, imdb_id, wikidata_id, library_dir, updated_at, checked_at, added_at, origin, origin_id) VALUES(${"?,".repeat(29)}?)`, ...vals, now(), now(), t.origin, t.originId);256  return { id: Number(r.lastInsertRowid), created: true };257}258export function setMedia(titleId: number, episodeId: number | null, m: { kind: "remote" | "file" | "embed"; url?: string; path?: string; container?: string | null; vcodec?: string | null; acodec?: string | null; width?: number | null; height?: number | null; duration?: number | null; size?: number | null; mtime?: number | null; playback?: string; subtitles?: unknown[]; audio?: unknown[] }): number {259  const prev = m.path ? get<{ id: number }>("SELECT id FROM media WHERE path = ?", m.path)260    : get<{ id: number }>(`SELECT id FROM media WHERE title_id = ? AND episode_id ${episodeId === null ? "IS NULL" : "= ?"}`, ...(episodeId === null ? [titleId] : [titleId, episodeId]));261  const vals = [titleId, episodeId, m.kind, m.url ?? null, m.path ?? null, m.container ?? null, m.vcodec ?? null, m.acodec ?? null, m.width ?? null, m.height ?? null, m.duration ?? null, m.size ?? null, m.mtime ?? null,262    m.playback ?? "direct", JSON.stringify(m.subtitles ?? []), JSON.stringify(m.audio ?? [])];263  // Updating in place keeps the media id, hence the playback position, across re-imports.264  if (prev) { run("UPDATE media SET title_id=?, episode_id=?, kind=?, url=?, path=?, container=?, vcodec=?, acodec=?, width=?, height=?, duration_sec=?, size_bytes=?, mtime=?, playback=?, subtitles=?, audio=?, error=NULL WHERE id=?", ...vals, prev.id); return prev.id; }265  return Number(run("INSERT INTO media(title_id, episode_id, kind, url, path, container, vcodec, acodec, width, height, duration_sec, size_bytes, mtime, playback, subtitles, audio, added_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ...vals, now()).lastInsertRowid);266}267export function upsertEpisode(titleId: number, season: number, episode: number, name: string | null, synopsis: string | null = null, airDate: string | null = null): number {268  run("INSERT INTO episodes(title_id, season, episode, name, synopsis, air_date) VALUES(?,?,?,?,?,?) ON CONFLICT(title_id, season, episode) DO UPDATE SET name = excluded.name, synopsis = COALESCE(excluded.synopsis, synopsis), air_date = COALESCE(excluded.air_date, air_date)", titleId, season, episode, name, synopsis, airDate);269  return get<{ id: number }>("SELECT id FROM episodes WHERE title_id = ? AND season = ? AND episode = ?", titleId, season, episode)!.id;270}271272function base(c: Candidate, d: Details | undefined, fallbackTitle: string) {273  const d0 = d ?? { directors: [], cast: [], genres: [], langs: [], countries: [], types: [], juris: [] } as Details;274  for (const j of d0.juris) c.jurisdictions.add(j);275  if (d0.licence) c.licence = d0.licence;276  const genres = mapGenres(d0.genres);277  const animation = genres.includes("Animation") || d0.types.some((t) => ANIMATED.has(t));278  const documentary = genres.includes("Documentaire") || d0.types.some((t) => DOCU.has(t));279  if (animation && !genres.includes("Animation")) genres.push("Animation");280  if (documentary && !genres.includes("Documentaire")) genres.push("Documentaire");281  return { d0, genres, animation, documentary, title: d0.fr || d0.en || d0.orig || fallbackTitle, original: d0.orig && d0.orig !== (d0.fr || d0.en) ? d0.orig : d0.en && d0.en !== d0.fr ? d0.en : null };282}283284/** Parse « S01E02 », « 1x02 », « Episode 3 », « Chapter 4 », trailing numbers. */285export function episodeNumber(name: string): { season: number; episode: number } | null {286  const n = name.replace(/\.[a-z0-9]{2,4}$/i, "");287  let m = /s(\d{1,2})[ ._-]?e(\d{1,3})/i.exec(n); if (m) return { season: +m[1], episode: +m[2] };288  m = /(?:^|[^\d])(\d{1,2})x(\d{1,3})(?:[^\d]|$)/i.exec(n); if (m) return { season: +m[1], episode: +m[2] };289  m = /(?:ep(?:isode)?|chapter|chapitre|part|partie|ch)[ ._-]*(\d{1,3})/i.exec(n); if (m) return { season: 1, episode: +m[1] };290  m = /(\d{1,3})(?:_512kb)?$/.exec(n.trim()); if (m) return { season: 1, episode: +m[1] };291  return null;292}293294// Reviewed series that Wikidata does not flag with a copyright status (see docs/SOURCES.md for the evidence).295export const CURATED_SERIES = [296  { ia: "pioneer.one.season1.720p.x264-vodo", qid: "Q639740", title: "Pioneer One", status: "cc", basis: "Creative Commons BY-NC-SA : série financée par ses spectateurs et diffusée gratuitement par ses auteurs (VODO) ; licence indiquée sur Internet Archive" },297];298299export async function importVod(shouldStop: () => boolean = () => false): Promise<VodResult> {300  const t0 = now();301  const res: VodResult = { films: 0, series: 0, episodes: 0, added: 0, updated: 0, skippedNoVideo: 0, skippedMismatch: 0, skippedAdult: 0, deactivated: 0, errors: [], durationMs: 0 };302  const started = now();303  const builtin = (kind: string, name: string, ref: string, as: string, note: string) =>304    run("INSERT INTO vod_sources(kind, name, ref, as_kind, rights_note, builtin, created_at) VALUES(?,?,?,?,?,1,?) ON CONFLICT(kind, ref) DO NOTHING", kind, name, ref, as, note, now());305  builtin("wikidata", "Films libres — Wikidata × Internet Archive / Wikimedia Commons", "films", "movie", "Statut « domaine public » ou licence libre documentés sur Wikidata ; vidéo servie par Internet Archive ou Wikimedia Commons");306  builtin("wikidata", "Séries et feuilletons libres — Wikidata × Internet Archive", "series", "series", "Même règle que les films, plus une courte liste vérifiée à la main");307  const enabled = (ref: string) => !!get("SELECT 1 FROM vod_sources WHERE kind = 'wikidata' AND ref = ? AND enabled = 1", ref);308  const done = (kind: string, ref: string, n: number, err?: string) => run("UPDATE vod_sources SET last_sync_at = ?, last_status = ?, last_error = ?, title_count = ? WHERE kind = ? AND ref = ?", now(), err ? "error" : "ok", err ?? null, n, kind, ref);309310  // ---- films311  if (enabled("films")) try {312    const cands = await candidates(FILM_TYPES, true);313    log("info", "vod", `Films : ${cands.size} candidats aux droits documentés`);314    const det = await details([...cands.keys()]);315    log("info", "vod", `Films : métadonnées Wikidata reçues (${det.size})`);316    // Synopses: French Wikipedia first, English otherwise.317    const frT = new Map<string, string>(), enT = new Map<string, string>();318    for (const [id, d] of det) { const f = wikiTitle(d.frw), e = wikiTitle(d.enw); if (f) frT.set(id, f); else if (e) enT.set(id, e); }319    const known = new Set(all<{ origin_id: string }>("SELECT origin_id FROM titles WHERE origin = 'wikidata' AND synopsis IS NOT NULL").map((r) => r.origin_id));320    const frX = await extracts("fr", [...frT].filter(([id]) => !known.has(id)).map(([, t]) => t));321    const enX = await extracts("en", [...enT].filter(([id]) => !known.has(id)).map(([, t]) => t));322    log("info", "vod", `Films : synopsis Wikipédia reçus (${frX.size} fr, ${enX.size} en) ; vérification des sources vidéo`);323    let seen = 0;324    for (const c of cands.values()) {325      if (++seen % 250 === 0) log("info", "vod", `Films : ${seen}/${cands.size} vérifiés, ${res.films} retenus`);326      const b = base(c, det.get(c.qid), c.qid);327      if (b.d0.genres.some((g) => ADULT.test(g))) { res.skippedAdult++; continue; }328      const existing = get<{ id: number; checked_at: number | null }>("SELECT id, checked_at FROM titles WHERE origin = 'wikidata' AND origin_id = ?", c.qid);329      // A title already verified in the last 30 days keeps its media; only its metadata is refreshed.330      let pick: MediaPick | null = null, iaId: string | null = null, iaLicence: string | undefined, commons: string | null = null;331      const fresh = existing?.checked_at && now() - existing.checked_at < 30 * 86400_000 && get("SELECT 1 FROM media WHERE title_id = ?", existing.id);332      if (!fresh) {333        let bestScore = 0;334        for (const id of c.ia) {335          const item = await iaItem(id); if (!item || item.metadata?.mediatype !== "movies") continue;336          const score = iaMatch(item, [b.d0.fr, b.d0.en, b.d0.orig], b.d0.year); if (score <= bestScore) { if (!score) res.skippedMismatch++; continue; }337          const vids = pickVideos(id, item); if (!vids.length) continue;338          pick = vids.sort((x, y) => (y.duration ?? 0) - (x.duration ?? 0))[0]; iaId = id; iaLicence = item.metadata?.licenseurl; bestScore = score;339        }340        if (!pick) commons = [...c.commons].find((u) => /\.webm$/i.test(decodeURIComponent(u))) ?? null;341        if (!pick && !commons) { res.skippedNoVideo++; if (existing) run("UPDATE titles SET active = 0, checked_at = ? WHERE id = ?", now(), existing.id); continue; }342      }343      const d = b.d0; const rights = rightsOf(c, iaLicence);344      const runtime = pick?.duration ?? (d.duration ? Math.round(d.duration * 60) : null);345      const extra = d.imdb ? await tmdb(d.imdb) : null;346      const synopsis = extra?.overview ?? frX.get(frT.get(c.qid) ?? "") ?? enX.get(enT.get(c.qid) ?? "") ?? null;347      const u = upsertTitle({348        kind: "movie", origin: "wikidata", originId: c.qid, title: b.title, original: b.original, year: d.year ?? null, synopsis,349        synopsisSource: synopsis ? (extra?.overview ? "TMDB" : frX.has(frT.get(c.qid) ?? "") ? "Wikipédia (fr)" : "Wikipedia (en)") : null, runtime,350        genres: b.genres, directors: d.directors, cast: d.cast, languages: d.langs, countries: d.countries,351        poster: extra?.poster ?? commonsImage(d.image, 500) ?? (iaId ? `https://archive.org/services/img/${encodeURIComponent(iaId)}` : null), backdrop: extra?.backdrop ?? commonsImage(d.image, 1280),352        documentary: b.documentary, animation: b.animation, short: !!runtime && runtime < 40 * 60,353        rightsStatus: rights.status, rightsBasis: rights.basis, rightsUrl: `https://www.wikidata.org/wiki/${c.qid}`,354        sourceName: fresh ? get<{ source_name: string }>("SELECT source_name FROM titles WHERE id = ?", existing!.id)!.source_name : pick ? "Internet Archive" : "Wikimedia Commons",355        sourceUrl: fresh ? get<{ source_url: string | null }>("SELECT source_url FROM titles WHERE id = ?", existing!.id)!.source_url : iaId ? `https://archive.org/details/${encodeURIComponent(iaId)}` : commons ? commons.replace(/^http:/, "https:").replace("Special:FilePath/", "File:") : null,356        imdb: d.imdb, wikidata: c.qid,357      });358      if (pick) setMedia(u.id, null, { kind: "remote", url: pick.url, container: "mp4", vcodec: "h264", height: pick.height, duration: pick.duration, size: pick.size, subtitles: pick.subtitles });359      else if (commons) setMedia(u.id, null, { kind: "remote", url: commons.replace(/^http:/, "https:"), container: "webm", duration: runtime });360      if (u.created) res.added++; else res.updated++;361      res.films++;362    }363    // Second pass for the synopses a rate limit made us skip: the verification above took long enough for it to lift.364    const missing = all<{ origin_id: string }>("SELECT origin_id FROM titles WHERE origin = 'wikidata' AND kind = 'movie' AND active = 1 AND synopsis IS NULL").map((r) => r.origin_id);365    for (const [lang, map, src] of [["fr", frT, "Wikipédia (fr)"], ["en", enT, "Wikipedia (en)"]] as const) {366      const want = missing.filter((id) => map.has(id)); if (!want.length) continue;367      const x = await extracts(lang, want.map((id) => map.get(id)!));368      for (const id of want) { const text = x.get(map.get(id)!); if (text) run("UPDATE titles SET synopsis = ?, synopsis_source = ? WHERE origin = 'wikidata' AND origin_id = ?", text, src, id); }369    }370    res.deactivated += Number(run("UPDATE titles SET active = 0 WHERE origin = 'wikidata' AND kind = 'movie' AND active = 1 AND updated_at < ?", started).changes);371    done("wikidata", "films", res.films);372  } catch (e) { res.errors.push(`films : ${(e as Error).message}`); done("wikidata", "films", res.films, (e as Error).message); log("error", "vod", "Import des films en échec", { error: (e as Error).message }); }373374  // ---- series375  if (enabled("series")) try {376    const cands = await candidates(SERIES_TYPES, false);377    for (const s of CURATED_SERIES) if (!cands.has(s.qid)) cands.set(s.qid, { qid: s.qid, ia: new Set([s.ia]), commons: new Set(), jurisdictions: new Set(), licence: null });378    const det = await details([...cands.keys()]);379    const frX = await extracts("fr", [...det.values()].map((d) => wikiTitle(d.frw)).filter((t): t is string => !!t));380    const enX = await extracts("en", [...det.values()].map((d) => wikiTitle(d.enw)).filter((t): t is string => !!t));381    for (const c of cands.values()) {382      const curated = CURATED_SERIES.find((s) => s.qid === c.qid);383      const b = base(c, det.get(c.qid), curated?.title ?? c.qid); const d = b.d0;384      if (d.genres.some((g) => ADULT.test(g))) { res.skippedAdult++; continue; }385      // Episodes: every video of a multi-file item, or the numbered sibling items of a serial (« name1 », « name2 », …).386      const eps: { season: number; episode: number; name: string; pick: MediaPick }[] = [];387      let iaLicence: string | undefined, firstId: string | null = null;388      for (const id of c.ia) {389        const item = await iaItem(id); if (!item) continue;390        firstId ??= id; iaLicence ??= item.metadata?.licenseurl;391        const vids = pickVideos(id, item);392        if (vids.length > 1) vids.forEach((v, i) => { const n = episodeNumber(v.name.split("/").pop()!) ?? { season: 1, episode: i + 1 }; eps.push({ ...n, name: v.name.split("/").pop()!.replace(/\.(mp4|m4v)$/i, "").replace(/[._]+/g, " ").trim(), pick: v }); });393        else if (vids.length === 1) {394          const m = /^(.*?)(\d+)$/.exec(id);395          const siblings: string[] = [];396          if (m) {397            try {398              const r = await getJson<{ response: { docs: { identifier: string }[] } }>(`https://archive.org/advancedsearch.php?q=${encodeURIComponent(`identifier:${m[1]}*`)}&fl[]=identifier&rows=200&output=json`);399              siblings.push(...r.response.docs.map((x) => x.identifier).filter((x) => new RegExp(`^${m[1].replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\d+$`).test(x)));400            } catch { /* keep the single item */ }401          }402          if (siblings.length > 1) {403            for (const sid of siblings.sort((x, y) => Number(/\d+$/.exec(x)![0]) - Number(/\d+$/.exec(y)![0]))) {404              const it = await iaItem(sid); const v = it ? pickVideos(sid, it)[0] : null; if (!v || !it) continue;405              eps.push({ season: 1, episode: Number(/\d+$/.exec(sid)![0]), name: (it.metadata?.title ?? sid).replace(/^.*?:\s*/, "").trim(), pick: v });406            }407          } else eps.push({ season: 1, episode: eps.length + 1, name: item.metadata?.title ?? id, pick: vids[0] });408        }409      }410      const uniq = new Map(eps.map((e) => [`${e.season}x${e.episode}`, e]));411      if (uniq.size < 2) { res.skippedNoVideo++; continue; } // a « series » with a single video is not presented as one412      const rights = curated ? { status: curated.status, basis: curated.basis } : rightsOf(c, iaLicence);413      const extra = d.imdb ? await tmdb(d.imdb) : null;414      const synopsis = extra?.overview ?? frX.get(wikiTitle(d.frw) ?? "") ?? enX.get(wikiTitle(d.enw) ?? "") ?? null;415      const u = upsertTitle({416        kind: "series", origin: "wikidata", originId: c.qid, title: b.title, original: b.original, year: d.year ?? null, synopsis, synopsisSource: synopsis ? (extra?.overview ? "TMDB" : frX.has(wikiTitle(d.frw) ?? "") ? "Wikipédia (fr)" : "Wikipedia (en)") : null,417        runtime: null, genres: b.genres, directors: d.directors, cast: d.cast, languages: d.langs, countries: d.countries,418        poster: extra?.poster ?? commonsImage(d.image, 500) ?? (firstId ? `https://archive.org/services/img/${encodeURIComponent(firstId)}` : null), backdrop: extra?.backdrop ?? commonsImage(d.image, 1280),419        documentary: b.documentary, animation: b.animation, short: false, rightsStatus: rights.status, rightsBasis: rights.basis, rightsUrl: `https://www.wikidata.org/wiki/${c.qid}`,420        sourceName: "Internet Archive", sourceUrl: firstId ? `https://archive.org/details/${encodeURIComponent(firstId)}` : null, imdb: d.imdb, wikidata: c.qid,421      });422      tx(() => {423        const keep: number[] = [];424        for (const e of uniq.values()) {425          const eid = upsertEpisode(u.id, e.season, e.episode, e.name); keep.push(eid);426          setMedia(u.id, eid, { kind: "remote", url: e.pick.url, container: "mp4", vcodec: "h264", height: e.pick.height, duration: e.pick.duration, size: e.pick.size, subtitles: e.pick.subtitles });427        }428        run(`DELETE FROM episodes WHERE title_id = ? AND id NOT IN (${keep.map(() => "?").join(",")})`, u.id, ...keep);429      });430      if (u.created) res.added++; else res.updated++;431      res.series++; res.episodes += uniq.size;432    }433    res.deactivated += Number(run("UPDATE titles SET active = 0 WHERE origin = 'wikidata' AND kind = 'series' AND active = 1 AND updated_at < ?", started).changes);434    done("wikidata", "series", res.series);435  } catch (e) { res.errors.push(`séries : ${(e as Error).message}`); done("wikidata", "series", res.series, (e as Error).message); log("error", "vod", "Import des séries en échec", { error: (e as Error).message }); }436437    bumpTitles();438  // ---- Office national du film (official embedded player, free films only)439  builtin("onf", "ONF — Office national du film du Canada (lecteur officiel intégré)", "films", "movie", "Intégration autorisée par l'ONF pour un usage personnel et non commercial, avec lien vers onf.ca ; seuls les films offerts gratuitement et en entier par le lecteur intégré sont listés");440  if (get("SELECT 1 FROM vod_sources WHERE kind = 'onf' AND enabled = 1")) try {441    const { importOnf } = await import("./onf");442    const o = await importOnf(4000, shouldStop);443    res.onf = o; done("onf", "films", get<{ n: number }>("SELECT COUNT(*) n FROM titles t WHERE origin = 'onf' AND active = 1 AND EXISTS(SELECT 1 FROM media m WHERE m.title_id = t.id)")!.n);444  } catch (e) { res.errors.push(`ONF : ${(e as Error).message}`); done("onf", "films", 0, (e as Error).message); log("error", "vod", "Import ONF en échec", { error: (e as Error).message }); }445  if (env.tmdbKey) try { const { enrichTmdb } = await import("./tmdb"); res.tmdb = await enrichTmdb(); } catch (e) { res.errors.push(`TMDB : ${(e as Error).message}`); }446  res.durationMs = now() - t0;447  bumpTitles();448  log(res.errors.length ? "warn" : "info", "vod", `Catalogue films et séries : ${res.films} films, ${res.series} séries (${res.episodes} épisodes)`, res);449  return res;450}451452/** Looks a personal file up on Wikidata by title (and year) to borrow its poster, synopsis and credits. Returns null when nothing matches confidently. */453export async function enrichment(title: string, year: number | null, kind: "movie" | "series") {454  const cls = kind === "movie" ? "wd:Q11424" : "wd:Q5398426";455  const rows = await sparql(`SELECT ?f (MIN(YEAR(?d)) AS ?y) WHERE { SERVICE wikibase:mwapi { bd:serviceParam wikibase:endpoint "www.wikidata.org" ; wikibase:api "EntitySearch" ; mwapi:search ${JSON.stringify(title)} ; mwapi:language "fr" . ?f wikibase:apiOutputItem mwapi:item . }456    ?f wdt:P31/wdt:P279* ${cls} . OPTIONAL { ?f wdt:P577 ?d } OPTIONAL { ?f wdt:P580 ?d } } GROUP BY ?f LIMIT 8`);457  const hits = rows.map((b) => ({ id: qid(b.f?.value), y: b.y ? Number(b.y.value) : null }));458  const hit = year ? hits.find((h) => h.y !== null && Math.abs(h.y - year) <= 1) : hits.length === 1 ? hits[0] : null;459  if (!hit) return null;460  const d = (await details([hit.id])).get(hit.id); if (!d) return null;461  const c: Candidate = { qid: hit.id, ia: new Set(), commons: new Set(), jurisdictions: new Set(), licence: null };462  const b = base(c, d, title);463  const fr = wikiTitle(d.frw), en = wikiTitle(d.enw);464  const x = fr ? await extracts("fr", [fr]) : en ? await extracts("en", [en]) : new Map<string, string>();465  const extra = d.imdb ? await tmdb(d.imdb) : null;466  const synopsis = extra?.overview ?? x.get(fr ?? en ?? "") ?? null;467  return {468    title: b.title, original: b.original, year: d.year ?? year, synopsis, synopsisSource: synopsis ? (extra?.overview ? "TMDB" : fr ? "Wikipédia (fr)" : "Wikipedia (en)") : null, genres: b.genres, directors: d.directors, cast: d.cast,469    languages: d.langs, countries: d.countries, poster: extra?.poster ?? commonsImage(d.image, 500), backdrop: extra?.backdrop ?? commonsImage(d.image, 1280), documentary: b.documentary, animation: b.animation, imdb: d.imdb ?? null, wikidata: hit.id,470  };471}472
Files main
Menu

Theme