SPB Git

Repository Public

sync.ts 18.1 KB 225 lines typescript

1import fs from "node:fs";2import path from "node:path";3import { all, bumpCatalog, db, get, log, now, run, setSetting, tx } from "./db";4import { parseM3u, slugKey, type M3uEntry } from "./m3u";5import { normalize } from "./catalog";6import { loadLive, type LiveSource } from "./engine";7import { getSetting } from "./db";8import { fetchText } from "./ssrf";910const API = "https://iptv-org.github.io/api";11const FAST = "https://raw.githubusercontent.com/BuddyChewChew/app-m3u-generator/main/playlists";12// Built-in sources. Each one was reviewed before being listed (docs/SOURCES.md): maintained repository, documented13// free-to-air policy, no credentials or bypass needed. `enabled: false` = listed for the administrator but not imported14// until switched on. Order matters: the first source that lists a URL owns the stream.15export const BUILTIN_SOURCES: { name: string; url: string; country: string | null; kind: string; enabled?: boolean }[] = [16  // Shipped with the application: streams read on the broadcasters' own sites and verified one by one (sources/official-verified.m3u).17  { name: "Sélection vérifiée — diffuseurs officiels (Radio-Canada, Assemblée nationale, CPAC, Global, France 24…)", url: "builtin:official-verified.m3u", country: null, kind: "curated" },18  { name: "IPTV-org — playlist principale", url: "https://iptv-org.github.io/iptv/index.m3u", country: null, kind: "iptv-org" },19  { name: "IPTV-org — Canada", url: "https://iptv-org.github.io/iptv/countries/ca.m3u", country: "CA", kind: "iptv-org" },20  { name: "IPTV-org — États-Unis", url: "https://iptv-org.github.io/iptv/countries/us.m3u", country: "US", kind: "iptv-org" },21  { name: "IPTV-org — France", url: "https://iptv-org.github.io/iptv/countries/fr.m3u", country: "FR", kind: "iptv-org" },22  { name: "Free-TV — chaînes gratuites du monde", url: "https://raw.githubusercontent.com/Free-TV/IPTV/master/playlist.m3u8", country: null, kind: "m3u" },23  { name: "Nexonic IPTV — chaînes officiellement gratuites", url: "https://raw.githubusercontent.com/Shahrukhalid/nexonic-iptv/main/playlists/all.m3u", country: null, kind: "m3u" },24  // Official free ad-supported services (FAST), regional line-ups generated daily from the services' own public APIs.25  { name: "Pluto TV — Canada", url: `${FAST}/plutotv_ca.m3u`, country: "CA", kind: "fast" },26  { name: "Samsung TV Plus — Canada", url: `${FAST}/samsungtvplus_ca.m3u`, country: "CA", kind: "fast" },27  { name: "Pluto TV — France", url: `${FAST}/plutotv_fr.m3u`, country: "FR", kind: "fast" },28  { name: "Samsung TV Plus — France", url: `${FAST}/samsungtvplus_fr.m3u`, country: "FR", kind: "fast" },29  { name: "Plex — Canada (jetons anonymes renouvelés chaque jour)", url: `${FAST}/plex_ca.m3u`, country: "CA", kind: "fast", enabled: false },30  // United States line-ups, measured playable from Canada on 2026-11-17 (Pluto 14/14, Roku 14/14, Tubi 13/14 sampled streams).31  // Samsung TV Plus US answers 403 from Canada (geo-restricted) and Plex US uses short-lived tokens: neither is added.32  { name: "Pluto TV — États-Unis", url: `${FAST}/plutotv_us.m3u`, country: "US", kind: "fast" },33  { name: "The Roku Channel", url: `${FAST}/roku_all.m3u`, country: "US", kind: "fast" },34  { name: "Tubi", url: `${FAST}/tubi_all.m3u`, country: "US", kind: "fast" },35  { name: "FreeCastHub — diffuseurs publics", url: "https://raw.githubusercontent.com/freecasthub/public-iptv/main/playlist.m3u", country: null, kind: "m3u", enabled: false },36];37// Programme guides published for the FAST line-ups above (same channel identifiers as the playlists).38export const BUILTIN_EPG = [39  { name: "Guide Pluto TV — Canada", url: "https://i.mjh.nz/PlutoTV/ca.xml.gz" },40  { name: "Guide Samsung TV Plus — Canada", url: "https://i.mjh.nz/SamsungTVPlus/ca.xml.gz" },41  { name: "Guide Pluto TV — France", url: "https://i.mjh.nz/PlutoTV/fr.xml.gz" },42  { name: "Guide Samsung TV Plus — France", url: "https://i.mjh.nz/SamsungTVPlus/fr.xml.gz" },43  { name: "Guide Pluto TV — États-Unis", url: "https://i.mjh.nz/PlutoTV/us.xml.gz" },44  { name: "Guide The Roku Channel", url: "https://i.mjh.nz/Roku/all.xml.gz" },45  { name: "Guide Tubi", url: `${FAST}/tubi_epg.xml` },46];4748// Page links (not streams) that some playlists list: they cannot be played by a video element and are not extracted.49const PAGE_HOSTS = /(^|\.)(youtube\.com|youtu\.be|twitch\.tv|dailymotion\.com|facebook\.com|vimeo\.com)$/i;50// Subscription brands: a public playlist that lists them is almost certainly relaying a pay channel. The free51// ad-supported variants that share the brand are let through explicitly.52const PREMIUM = /\b(hbo|cinemax|showtime|starz|dazn|sky\s+(sports?|cinema|atlantic|max)|canal\s?\+|bein\s+sports?|disney\s+(channel|junior|jr\.?|xd)|eurosport|nba\s+tv|nfl\s+network|tnt\s+sports|bt\s+sport|premier\s+sports|supersport|movistar|super\s?[ée]cran|crave|sportsnet|tsn\s?\d|rds\s?\d|rds$|fox\s+sports\s+\d|espn|lcn)\b/i;53const PREMIUM_FREE = /bein\s+sports?\s+xtra|the\s+ocho|rds\s+social/i;54export const looksPremium = (name: string) => PREMIUM.test(name) && !PREMIUM_FREE.test(name);55const cleanName = (n: string) => normalize(n.replace(/\b(uhd|fhd|hd|sd|4k)\b/gi, " ").replace(/\((en|fr|eng|fra)\)/gi, " "));5657export function ensureBuiltinSources() {58  for (const s of BUILTIN_SOURCES)59    run("INSERT INTO sources(kind, name, url, country, builtin, enabled, created_at) VALUES(?, ?, ?, ?, 1, ?, ?) ON CONFLICT(url) DO NOTHING", s.kind, s.name, s.url, s.country, s.enabled === false ? 0 : 1, now());60  for (const e of BUILTIN_EPG) run("INSERT INTO epg_sources(name, url, builtin, created_at) VALUES(?, ?, 1, ?) ON CONFLICT(url) DO NOTHING", e.name, e.url, now());61  if (!get("SELECT 1 FROM lists WHERE id = 1")) run("INSERT INTO lists(id, name, position, created_at) VALUES(1, 'Favoris', 0, ?)", now());62}6364interface ApiChannel { id: string; name: string; alt_names: string[]; network: string | null; country: string; categories: string[]; is_nsfw: boolean; closed: string | null; website: string | null }65interface ApiFeed { channel: string; id: string; is_main: boolean; languages: string[] }66interface ApiLogo { channel: string; feed: string | null; in_use: boolean; url: string }67interface ApiGuide { channel: string | null; feed: string | null; site: string; site_id: string; site_name: string; lang: string }6869async function api<T>(name: string, limitMb: number): Promise<T> {70  return JSON.parse(await fetchText(`${API}/${name}.json`, limitMb * 1024 * 1024, 120_000)) as T;71}7273export interface SyncResult {74  /** `entries` = lines in the playlist, `kept` = streams this source contributed after filters and de-duplication. */75  sources: { id: number; name: string; ok: boolean; entries: number; kept?: number; error?: string }[];76  channels: number; streams: number; newChannels: number; newStreams: number; removedStreams: number;77  duplicates: number; skippedBlocked: number; skippedProtocol: number; skippedPages: number; skippedPremium: number; mergedByName: number;78  parseErrors: number; apiErrors: string[]; durationMs: number;79}8081/** Full catalogue synchronisation: IPTV-org metadata API + every enabled M3U source. User state is never touched. */82export async function syncCatalog(): Promise<SyncResult> {83  const t0 = now();84  ensureBuiltinSources();85  const res: SyncResult = { sources: [], channels: 0, streams: 0, newChannels: 0, newStreams: 0, removedStreams: 0, duplicates: 0, skippedBlocked: 0, skippedProtocol: 0, skippedPages: 0, skippedPremium: 0, mergedByName: 0, parseErrors: 0, apiErrors: [], durationMs: 0 };8687  // 1. Metadata. A failure here is reported but does not stop the playlist import: existing metadata is kept.88  const meta: { channels?: ApiChannel[]; feeds?: ApiFeed[]; logos?: ApiLogo[]; guides?: ApiGuide[]; blocklist?: { channel: string }[]; categories?: { id: string; name: string }[]; countries?: { code: string; name: string; flag: string }[]; languages?: { code: string; name: string }[] } = {};89  await Promise.all(([["channels", 40], ["feeds", 40], ["logos", 40], ["guides", 80], ["blocklist", 5], ["categories", 1], ["countries", 1], ["languages", 5]] as const).map(async ([n, mb]) => {90    try { (meta as Record<string, unknown>)[n] = await api(n, mb); } catch (e) { res.apiErrors.push(`${n}.json : ${(e as Error).message}`); }91  }));92  const chanById = new Map((meta.channels ?? []).map((c) => [c.id, c]));93  const blocked = new Set((meta.blocklist ?? []).map((b) => b.channel));94  const catByName = new Map((meta.categories ?? all<{ id: string; name: string }>("SELECT id, name FROM categories")).map((c) => [c.name.toLowerCase(), c.id]));9596  // 2. Playlists, in source order (the main IPTV-org playlist first, so it owns the streams it lists).97  // Order = ownership of a stream listed twice: the playback priority set in the administration first, then the verified selection, then the built-in lists.98  const sources = all<LiveSource>("SELECT id, name, url, country, kind, config, secret, content FROM sources WHERE enabled = 1 ORDER BY priority, (kind = 'curated') DESC, builtin DESC, id");99  const premiumFilter = getSetting("premium_filter") !== "0";100  const sourceKind = new Map(sources.map((s) => [s.id, s.kind]));101  const perSource = new Map<number, number>();102  const okSources: number[] = [];103  const byUrl = new Map<string, { e: M3uEntry; sourceId: number; country: string | null }>();104  const regionOf = new Map<string, string>(); // stream url → country of the regional playlist that lists it105  for (const s of sources) {106    try {107      // Whatever the connector (playlist address, imported file, single stream, provider API), a live source yields playlist entries.108      const parsed = await loadLive(s, (name) => fs.readFileSync(path.join(/*turbopackIgnore: true*/ process.cwd(), "sources", path.basename(name)), "utf8"));109      res.parseErrors += parsed.errors.length;110      for (const e of parsed.entries) {111        if (!/^(https?|provider):\/\//i.test(e.url)) { res.skippedProtocol++; continue; }112        let host = ""; try { host = new URL(e.url).hostname; } catch { res.skippedProtocol++; continue; }113        if (PAGE_HOSTS.test(host)) { res.skippedPages++; continue; }114        if (e.channelId && (blocked.has(e.channelId) || chanById.get(e.channelId)?.is_nsfw)) { res.skippedBlocked++; continue; }115        if (premiumFilter && s.kind !== "fast" && s.kind !== "curated" && s.kind !== "xtream" && looksPremium(chanById.get(e.channelId ?? "")?.name ?? e.name)) { res.skippedPremium++; continue; }116        if (s.country) regionOf.set(e.url, s.country);117        if (byUrl.has(e.url)) { res.duplicates++; continue; }118        byUrl.set(e.url, { e, sourceId: s.id, country: s.country ?? e.country });119        perSource.set(s.id, (perSource.get(s.id) ?? 0) + 1);120      }121      okSources.push(s.id);122      res.sources.push({ id: s.id, name: s.name, ok: true, entries: parsed.entries.length, kept: perSource.get(s.id) ?? 0 });123      run("UPDATE sources SET last_sync_at = ?, last_status = 'ok', last_error = NULL, stream_count = ? WHERE id = ?", now(), parsed.entries.length, s.id);124    } catch (e) {125      const msg = (e as Error).message;126      res.sources.push({ id: s.id, name: s.name, ok: false, entries: 0, error: msg });127      run("UPDATE sources SET last_sync_at = ?, last_status = 'error', last_error = ? WHERE id = ?", now(), msg, s.id);128      log("error", "sync", `Source en échec : ${s.name}`, { url: /^https?:/.test(s.url) ? s.url : s.kind, error: msg });129    }130  }131132  // 3. Group streams into channels.133  const feedsByChannel = new Map<string, ApiFeed[]>();134  for (const f of meta.feeds ?? []) { const l = feedsByChannel.get(f.channel); if (l) l.push(f); else feedsByChannel.set(f.channel, [f]); }135  const logoByChannel = new Map<string, string>();136  for (const l of meta.logos ?? []) if (l.in_use && (!logoByChannel.has(l.channel) || l.feed === null)) logoByChannel.set(l.channel, l.url);137138  interface Chan { id: string; name: string; alt: string; country: string | null; categories: string; languages: Set<string>; logo: string | null; website: string | null; network: string | null; known: number }139  const chans = new Map<string, Chan>();140  const streamChannel = new Map<string, string>();141  // Name index of the IPTV-org database: lets a stream without a usable tvg-id join the channel it is a variant of.142  const byName = new Map<string, ApiChannel[]>();143  for (const c of meta.channels ?? []) {144    if (c.is_nsfw || blocked.has(c.id)) continue;145    for (const n of new Set([c.name, ...c.alt_names].map(cleanName))) { if (!n) continue; const l = byName.get(n); if (l) l.push(c); else byName.set(n, [c]); }146  }147  for (const [url, { e, country, sourceId }] of byUrl) {148    let known = e.channelId ? chanById.get(e.channelId) : undefined;149    if (!known && sourceKind.get(sourceId) !== "iptv-org") {150      // Same name and same country, or a name that designates a single channel worldwide: same channel.151      const cands = byName.get(cleanName(e.name)) ?? [];152      const hit = country ? cands.filter((c) => c.country === country) : cands;153      if (hit.length === 1) { known = hit[0]; res.mergedByName++; }154    }155    const id = known ? known.id : sourceKind.get(sourceId) === "iptv-org" || !country ? `x-${slugKey(e.name) || "sans-nom"}` : `x-${country.toLowerCase()}-${slugKey(e.name) || "sans-nom"}`;156    let c = chans.get(id);157    if (!c) {158      c = known159        ? { id, name: known.name, alt: known.alt_names.join("|"), country: known.country, categories: known.categories.join(";"), languages: new Set(), logo: e.logo || logoByChannel.get(id) || null, website: known.website, network: known.network, known: 1 }160        : { id, name: e.name, alt: "", country: country ?? regionOf.get(url) ?? null, categories: e.group.map((g) => catByName.get(g.toLowerCase())).filter(Boolean).join(";"), languages: new Set(), logo: e.logo, website: null, network: null, known: 0 };161      chans.set(id, c);162    }163    if (!c.logo && e.logo) c.logo = e.logo;164    if (!c.country) c.country = regionOf.get(url) ?? null;165    if (known) {166      const feeds = feedsByChannel.get(id) ?? [];167      const f = feeds.find((x) => x.id === e.feed) ?? feeds.find((x) => x.is_main);168      for (const l of f?.languages ?? []) c.languages.add(l);169    }170    streamChannel.set(url, id);171  }172173  // 4. Write. One transaction: readers keep seeing the previous catalogue until it commits.174  const firstSync = !get("SELECT 1 FROM settings WHERE key = 'initial_sync_at'");175  const d = db();176  const t = now();177  tx(() => {178    if (meta.categories) { run("DELETE FROM categories"); const st = d.prepare("INSERT INTO categories(id, name) VALUES(?, ?)"); for (const c of meta.categories) st.run(c.id, c.name); }179    if (meta.countries) { run("DELETE FROM countries"); const st = d.prepare("INSERT INTO countries(code, name, flag) VALUES(?, ?, ?)"); for (const c of meta.countries) st.run(c.code, c.name, c.flag); }180    if (meta.languages) { run("DELETE FROM languages"); const st = d.prepare("INSERT OR IGNORE INTO languages(code, name) VALUES(?, ?)"); for (const c of meta.languages) st.run(c.code, c.name); }181182    const existing = new Set(all<{ id: string }>("SELECT id FROM channels").map((r) => r.id));183    const upChan = d.prepare(`INSERT INTO channels(id, name, alt_names, country, categories, languages, logo, website, network, known, active, first_seen, updated_at)184      VALUES(?,?,?,?,?,?,?,?,?,?,1,?,?)185      ON CONFLICT(id) DO UPDATE SET name=excluded.name, alt_names=excluded.alt_names, country=COALESCE(excluded.country, country), categories=excluded.categories,186        languages=excluded.languages, logo=COALESCE(excluded.logo, logo), website=excluded.website, network=excluded.network, known=excluded.known, active=1, updated_at=excluded.updated_at`);187    for (const c of chans.values()) {188      if (!existing.has(c.id)) res.newChannels++;189      upChan.run(c.id, c.name, c.alt, c.country, c.categories, [...c.languages].join(";"), c.logo, c.website, c.network, c.known, t, t);190    }191    const existingUrls = new Set(all<{ url: string }>("SELECT url FROM streams").map((r) => r.url));192    const upStream = d.prepare(`INSERT INTO streams(channel_id, source_id, url, title, feed, quality, labels, referrer, user_agent, first_seen, last_seen, tvg_id)193      VALUES(?,?,?,?,?,?,?,?,?,?,?,?)194      ON CONFLICT(url) DO UPDATE SET tvg_id=excluded.tvg_id, channel_id=excluded.channel_id, title=excluded.title, feed=excluded.feed, quality=excluded.quality, labels=excluded.labels,195        referrer=excluded.referrer, user_agent=excluded.user_agent, last_seen=excluded.last_seen`);196    for (const [url, { e, sourceId }] of byUrl) {197      if (!existingUrls.has(url)) res.newStreams++;198      upStream.run(streamChannel.get(url)!, sourceId, url, e.rawTitle, e.feed, e.quality, e.labels.join(";"), e.referrer, e.userAgent, t, t, e.tvgId);199    }200    // Streams that vanished from a source that synced correctly are removed; a failed source keeps its streams.201    if (okSources.length) {202      const marks = okSources.map(() => "?").join(",");203      res.removedStreams = Number(run(`DELETE FROM streams WHERE last_seen < ? AND source_id IN (${marks})`, t, ...okSources).changes);204    }205    run("UPDATE channels SET active = EXISTS(SELECT 1 FROM streams s WHERE s.channel_id = channels.id)");206    // A channel without streams survives only if the user kept something about it (favourite, hidden, history).207    run(`DELETE FROM channels WHERE active = 0 AND id NOT IN (SELECT channel_id FROM list_items UNION SELECT channel_id FROM hidden UNION SELECT channel_id FROM history)`);208209    if (meta.guides) {210      run("DELETE FROM guides");211      const st = d.prepare("INSERT INTO guides(channel_id, feed, site, site_id, site_name, lang) VALUES(?,?,?,?,?,?)");212      for (const g of meta.guides) if (g.channel && chans.has(g.channel)) st.run(g.channel, g.feed, g.site, g.site_id, g.site_name, g.lang);213      run("UPDATE channels SET has_guide = EXISTS(SELECT 1 FROM guides g WHERE g.channel_id = channels.id)");214    }215    if (firstSync && okSources.length) setSetting("initial_sync_at", String(t));216    setSetting("last_sync_at", String(t));217  });218  res.channels = get<{ n: number }>("SELECT COUNT(*) n FROM channels WHERE active = 1")!.n;219  res.streams = get<{ n: number }>("SELECT COUNT(*) n FROM streams")!.n;220  res.durationMs = now() - t0;221  bumpCatalog();222  log(res.sources.some((s) => !s.ok) || res.apiErrors.length ? "warn" : "info", "sync", `Synchronisation terminée : ${res.channels} chaînes, ${res.streams} flux`, res);223  return res;224}225
Files main
Menu

Theme