SPB Git

Repository Public

onf.ts 8.4 KB 97 lines typescript

1import { all, get, log, now, run } from "./db";2import { fetchText } from "./ssrf";3import { setMedia, upsertTitle } from "./vod";45// Office national du film du Canada (onf.ca). The ONF allows its player to be embedded « à des fins personnelles et non6// commerciales », with a link back to onf.ca (https://aide.onf.ca/conditions/). Films are therefore played inside the7// official ONF player (an iframe); no stream address is ever taken out of it.8//9// A film is listed only after checking that the embedded player really offers the whole film to an anonymous visitor:10// the embed page declares what it plays and for how long; a rental or a trailer does not match the film's duration.1112const SITEMAP = "https://www.onf.ca/sitemap-films.xml";13const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));14let last = 0;15async function page(url: string): Promise<string> {16  const wait = last + 650 - Date.now(); if (wait > 0) await sleep(wait); last = Date.now(); // one request at a time, under two per second17  return fetchText(url, 3 * 1024 * 1024, 30_000);18}19const ENT: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };20const decode = (s: string) => s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => (e[0] === "#" ? String.fromCodePoint(e[1].toLowerCase() === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10)) : ENT[e.toLowerCase()] ?? m));21const layer = (html: string, key: string) => { const m = new RegExp(`'${key}':\\s*(?:'((?:[^'\\\\]|\\\\.)*)'|"((?:[^"\\\\]|\\\\.)*)")`).exec(html); return m ? decode((m[1] ?? m[2] ?? "").replace(/\\(['"])/g, "$1")).trim() : ""; };2223export interface OnfEmbed { playable: boolean; seconds: number; year: number | null }24/** What the official embedded player offers to an anonymous visitor. */25export function parseEmbed(html: string): OnfEmbed {26  const hasSource = /"source":\s*"https:\/\/[^"]+\.m3u8/.test(html);27  return { playable: hasSource && layer(html, "nfb_work_publishing_status") === "Public", seconds: Number(layer(html, "nfb_version_duration")) || 0, year: Number(layer(html, "nfb_version_year")) || null };28}29export interface OnfFilm { title: string; year: number | null; seconds: number; directors: string[]; subjects: string[]; synopsis: string | null; poster: string | null; backdrop: string | null; kinds: string[] }30export function parseFilm(html: string): OnfFilm | null {31  const m = /<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/.exec(html);32  let ld: { name?: string; description?: string; thumbnailUrl?: string; image?: string; director?: { name?: string }[] } = {};33  try { if (m) ld = JSON.parse(m[1]); } catch { /* fall back on the data layer */ }34  const title = decode(ld.name ?? "") || layer(html, "nfb_version_title"); if (!title) return null;35  const creator = layer(html, "nfb_version_creator");36  // Categories declared for this film (« 'nfb_version_category': {'en': [...], 'fr': ['Fiction', 'Long métrage de fiction']} »), not the site menu.37  const cat = /'nfb_version_category':\s*\{[^}]*'fr':\s*\[([^\]]*)\]/.exec(html)?.[1] ?? "";38  const kinds = [...cat.matchAll(/'((?:[^'\\]|\\.)*)'/g)].map((x) => decode(x[1].replace(/\\'/g, "'")));39  return {40    title, year: Number(layer(html, "nfb_version_year")) || null, seconds: Number(layer(html, "nfb_version_duration")) || 0,41    directors: (ld.director ?? []).map((d) => decode(d.name ?? "")).filter(Boolean).concat(creator && !(ld.director ?? []).length ? creator.split(/,\s*/) : []),42    subjects: layer(html, "nfb_version_subject").split(/,\s*/).filter(Boolean), synopsis: ld.description ? decode(ld.description).trim() : null,43    poster: ld.image ? decode(ld.image) : null, backdrop: ld.thumbnailUrl ? decode(ld.thumbnailUrl).replace(/\?w=\d+.*$/, "?w=1280") : null, kinds,44  };45}46const GENRE: [RegExp, string][] = [[/guerre|conflit/i, "Guerre"], [/histoire|archives/i, "Histoire"], [/musique|danse/i, "Musical"], [/sport/i, "Sport"], [/enfants|jeunes|famille/i, "Famille"], [/humour|comédie/i, "Comédie"], [/science|environnement|nature|animaux/i, "Science et nature"], [/art|cinéma|littérature/i, "Arts"], [/autochtone/i, "Peuples autochtones"], [/politique|société|travail|économie/i, "Société"]];4748export interface OnfResult { listed: number; checked: number; added: number; notFree: number; errors: number; remaining: number; durationMs: number }4950/** Checks up to `budget` films not yet examined (then re-checks the oldest ones), politely. Safe to interrupt: every film is committed on its own. */51export async function importOnf(budget = 4000, shouldStop: () => boolean = () => false): Promise<OnfResult> {52  const t0 = now(); const res: OnfResult = { listed: 0, checked: 0, added: 0, notFree: 0, errors: 0, remaining: 0, durationMs: 0 };53  const xml = await fetchText(SITEMAP, 10 * 1024 * 1024, 60_000);54  const slugs = [...new Set([...xml.matchAll(/<loc>https:\/\/www\.onf\.ca\/film\/([^/<]+)\/<\/loc>/g)].map((m) => m[1]))];55  res.listed = slugs.length;56  const known = new Map(all<{ origin_id: string; checked_at: number | null }>("SELECT origin_id, checked_at FROM titles WHERE origin = 'onf'").map((r) => [r.origin_id, r.checked_at ?? 0]));57  const fresh = slugs.filter((s) => !known.has(s));58  const stale = slugs.filter((s) => known.has(s) && now() - known.get(s)! > 45 * 86400_000).sort((a, b) => known.get(a)! - known.get(b)!).slice(0, 150);59  const todo = [...fresh, ...stale].slice(0, budget);60  for (const slug of todo) {61    if (shouldStop()) break;62    try {63      const url = `https://www.onf.ca/film/${slug}/`;64      const embed = parseEmbed(await page(`${url}embed/player/`));65      const film = parseFilm(await page(url));66      res.checked++;67      if (!film) { res.errors++; continue; }68      // The whole film, not a trailer or an excerpt: the player must announce (nearly) the duration of the work.69      const free = embed.playable && embed.seconds > 0 && (!film.seconds || embed.seconds >= film.seconds * 0.9);70      const has = (re: RegExp) => film.kinds.some((k) => re.test(k));71      const doc = has(/documentaire/i), anim = has(/animation/i);72      const genres = [...new Set([...(doc ? ["Documentaire"] : []), ...(anim ? ["Animation"] : []), ...(has(/fiction/i) ? ["Drame"] : []), ...(has(/expérimental/i) ? ["Expérimental"] : []), ...(has(/comédie|humour/i) ? ["Comédie"] : []),73        ...(has(/enfant|jeunesse|famille/i) ? ["Famille"] : []), ...film.subjects.flatMap((s) => GENRE.filter(([re]) => re.test(s)).map(([, g]) => g))])];74      const t = upsertTitle({75        kind: "movie", origin: "onf", originId: slug, title: film.title, year: film.year ?? embed.year, synopsis: film.synopsis, synopsisSource: film.synopsis ? "ONF" : null, runtime: embed.seconds || film.seconds || null,76        genres, directors: film.directors, cast: [], languages: ["fr"], countries: ["CA"], poster: film.poster ?? film.backdrop, backdrop: film.backdrop, documentary: doc, animation: anim,77        short: (embed.seconds || film.seconds) > 0 && (embed.seconds || film.seconds) < 40 * 60, rightsStatus: "official",78        rightsBasis: "Diffusion gratuite par l'Office national du film du Canada dans son lecteur officiel intégré, autorisé pour un usage personnel et non commercial (conditions d'utilisation de l'ONF) ; gratuité vérifiée film par film",79        rightsUrl: "https://aide.onf.ca/conditions/", sourceName: "ONF", sourceUrl: url,80      });81      if (free) { setMedia(t.id, null, { kind: "embed", url: `${url}embed/player/`, container: "embed", duration: embed.seconds, playback: "embed" }); if (t.created) res.added++; }82      else { run("DELETE FROM media WHERE title_id = ?", t.id); run("UPDATE titles SET active = 0 WHERE id = ?", t.id); res.notFree++; }83    } catch (e) {84      res.errors++;85      if (res.errors > 25 && res.errors > res.checked / 2) { log("warn", "vod", "ONF : trop d'erreurs, import interrompu", { error: (e as Error).message }); break; }86    }87    if (res.checked % 200 === 0 && res.checked) { log("info", "vod", `ONF : ${res.checked}/${todo.length} films vérifiés, ${res.added} ajoutés`); run("INSERT INTO settings(key, value) VALUES('titles_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", String(now())); }88  }89  // Films withdrawn from the ONF site.90  const live = new Set(slugs);91  for (const k of known.keys()) if (!live.has(k)) run("UPDATE titles SET active = 0 WHERE origin = 'onf' AND origin_id = ?", k);92  res.remaining = Math.max(0, fresh.length - res.checked);93  res.durationMs = now() - t0;94  void get;95  return res;96}97
Files main
Menu

Theme