Repository Public
1import fs from "node:fs";2import path from "node:path";3import { all, bumpCatalog, db, get, log, now, run, setSetting, tx } from "./db";4import { parseM3u, slugKey, type M3uEntry } from "./m3u";5import { normalize } from "./catalog";6import { loadLive, type LiveSource } from "./engine";7import { getSetting } from "./db";8import { fetchText } from "./ssrf";910const API = "https://iptv-org.github.io/api";11const FAST = "https://raw.githubusercontent.com/BuddyChewChew/app-m3u-generator/main/playlists";12// Built-in sources. Each one was reviewed before being listed (docs/SOURCES.md): maintained repository, documented13// free-to-air policy, no credentials or bypass needed. `enabled: false` = listed for the administrator but not imported14// until switched on. Order matters: the first source that lists a URL owns the stream.15export const BUILTIN_SOURCES: { name: string; url: string; country: string | null; kind: string; enabled?: boolean }[] = [16 // Shipped with the application: streams read on the broadcasters' own sites and verified one by one (sources/official-verified.m3u).17 { name: "Sélection vérifiée — diffuseurs officiels (Radio-Canada, Assemblée nationale, CPAC, Global, France 24…)", url: "builtin:official-verified.m3u", country: null, kind: "curated" },18 { name: "IPTV-org — playlist principale", url: "https://iptv-org.github.io/iptv/index.m3u", country: null, kind: "iptv-org" },19 { name: "IPTV-org — Canada", url: "https://iptv-org.github.io/iptv/countries/ca.m3u", country: "CA", kind: "iptv-org" },20 { name: "IPTV-org — États-Unis", url: "https://iptv-org.github.io/iptv/countries/us.m3u", country: "US", kind: "iptv-org" },21 { name: "IPTV-org — France", url: "https://iptv-org.github.io/iptv/countries/fr.m3u", country: "FR", kind: "iptv-org" },22 { name: "Free-TV — chaînes gratuites du monde", url: "https://raw.githubusercontent.com/Free-TV/IPTV/master/playlist.m3u8", country: null, kind: "m3u" },23 { name: "Nexonic IPTV — chaînes officiellement gratuites", url: "https://raw.githubusercontent.com/Shahrukhalid/nexonic-iptv/main/playlists/all.m3u", country: null, kind: "m3u" },24 // Official free ad-supported services (FAST), regional line-ups generated daily from the services' own public APIs.25 { name: "Pluto TV — Canada", url: `${FAST}/plutotv_ca.m3u`, country: "CA", kind: "fast" },26 { name: "Samsung TV Plus — Canada", url: `${FAST}/samsungtvplus_ca.m3u`, country: "CA", kind: "fast" },27 { name: "Pluto TV — France", url: `${FAST}/plutotv_fr.m3u`, country: "FR", kind: "fast" },28 { name: "Samsung TV Plus — France", url: `${FAST}/samsungtvplus_fr.m3u`, country: "FR", kind: "fast" },29 { name: "Plex — Canada (jetons anonymes renouvelés chaque jour)", url: `${FAST}/plex_ca.m3u`, country: "CA", kind: "fast", enabled: false },30 // United States line-ups, measured playable from Canada on 2026-11-17 (Pluto 14/14, Roku 14/14, Tubi 13/14 sampled streams).31 // Samsung TV Plus US answers 403 from Canada (geo-restricted) and Plex US uses short-lived tokens: neither is added.32 { name: "Pluto TV — États-Unis", url: `${FAST}/plutotv_us.m3u`, country: "US", kind: "fast" },33 { name: "The Roku Channel", url: `${FAST}/roku_all.m3u`, country: "US", kind: "fast" },34 { name: "Tubi", url: `${FAST}/tubi_all.m3u`, country: "US", kind: "fast" },35 { name: "FreeCastHub — diffuseurs publics", url: "https://raw.githubusercontent.com/freecasthub/public-iptv/main/playlist.m3u", country: null, kind: "m3u", enabled: false },36];37// Programme guides published for the FAST line-ups above (same channel identifiers as the playlists).38export const BUILTIN_EPG = [39 { name: "Guide Pluto TV — Canada", url: "https://i.mjh.nz/PlutoTV/ca.xml.gz" },40 { name: "Guide Samsung TV Plus — Canada", url: "https://i.mjh.nz/SamsungTVPlus/ca.xml.gz" },41 { name: "Guide Pluto TV — France", url: "https://i.mjh.nz/PlutoTV/fr.xml.gz" },42 { name: "Guide Samsung TV Plus — France", url: "https://i.mjh.nz/SamsungTVPlus/fr.xml.gz" },43 { name: "Guide Pluto TV — États-Unis", url: "https://i.mjh.nz/PlutoTV/us.xml.gz" },44 { name: "Guide The Roku Channel", url: "https://i.mjh.nz/Roku/all.xml.gz" },45 { name: "Guide Tubi", url: `${FAST}/tubi_epg.xml` },46];4748// Page links (not streams) that some playlists list: they cannot be played by a video element and are not extracted.49const PAGE_HOSTS = /(^|\.)(youtube\.com|youtu\.be|twitch\.tv|dailymotion\.com|facebook\.com|vimeo\.com)$/i;50// Subscription brands: a public playlist that lists them is almost certainly relaying a pay channel. The free51// ad-supported variants that share the brand are let through explicitly.52const PREMIUM = /\b(hbo|cinemax|showtime|starz|dazn|sky\s+(sports?|cinema|atlantic|max)|canal\s?\+|bein\s+sports?|disney\s+(channel|junior|jr\.?|xd)|eurosport|nba\s+tv|nfl\s+network|tnt\s+sports|bt\s+sport|premier\s+sports|supersport|movistar|super\s?[ée]cran|crave|sportsnet|tsn\s?\d|rds\s?\d|rds$|fox\s+sports\s+\d|espn|lcn)\b/i;53const PREMIUM_FREE = /bein\s+sports?\s+xtra|the\s+ocho|rds\s+social/i;54export const looksPremium = (name: string) => PREMIUM.test(name) && !PREMIUM_FREE.test(name);55const cleanName = (n: string) => normalize(n.replace(/\b(uhd|fhd|hd|sd|4k)\b/gi, " ").replace(/\((en|fr|eng|fra)\)/gi, " "));5657export function ensureBuiltinSources() {58 for (const s of BUILTIN_SOURCES)59 run("INSERT INTO sources(kind, name, url, country, builtin, enabled, created_at) VALUES(?, ?, ?, ?, 1, ?, ?) ON CONFLICT(url) DO NOTHING", s.kind, s.name, s.url, s.country, s.enabled === false ? 0 : 1, now());60 for (const e of BUILTIN_EPG) run("INSERT INTO epg_sources(name, url, builtin, created_at) VALUES(?, ?, 1, ?) ON CONFLICT(url) DO NOTHING", e.name, e.url, now());61 if (!get("SELECT 1 FROM lists WHERE id = 1")) run("INSERT INTO lists(id, name, position, created_at) VALUES(1, 'Favoris', 0, ?)", now());62}6364interface ApiChannel { id: string; name: string; alt_names: string[]; network: string | null; country: string; categories: string[]; is_nsfw: boolean; closed: string | null; website: string | null }65interface ApiFeed { channel: string; id: string; is_main: boolean; languages: string[] }66interface ApiLogo { channel: string; feed: string | null; in_use: boolean; url: string }67interface ApiGuide { channel: string | null; feed: string | null; site: string; site_id: string; site_name: string; lang: string }6869async function api<T>(name: string, limitMb: number): Promise<T> {70 return JSON.parse(await fetchText(`${API}/${name}.json`, limitMb * 1024 * 1024, 120_000)) as T;71}7273export interface SyncResult {74 /** `entries` = lines in the playlist, `kept` = streams this source contributed after filters and de-duplication. */75 sources: { id: number; name: string; ok: boolean; entries: number; kept?: number; error?: string }[];76 channels: number; streams: number; newChannels: number; newStreams: number; removedStreams: number;77 duplicates: number; skippedBlocked: number; skippedProtocol: number; skippedPages: number; skippedPremium: number; mergedByName: number;78 parseErrors: number; apiErrors: string[]; durationMs: number;79}8081/** Full catalogue synchronisation: IPTV-org metadata API + every enabled M3U source. User state is never touched. */82export async function syncCatalog(): Promise<SyncResult> {83 const t0 = now();84 ensureBuiltinSources();85 const res: SyncResult = { sources: [], channels: 0, streams: 0, newChannels: 0, newStreams: 0, removedStreams: 0, duplicates: 0, skippedBlocked: 0, skippedProtocol: 0, skippedPages: 0, skippedPremium: 0, mergedByName: 0, parseErrors: 0, apiErrors: [], durationMs: 0 };8687 // 1. Metadata. A failure here is reported but does not stop the playlist import: existing metadata is kept.88 const meta: { channels?: ApiChannel[]; feeds?: ApiFeed[]; logos?: ApiLogo[]; guides?: ApiGuide[]; blocklist?: { channel: string }[]; categories?: { id: string; name: string }[]; countries?: { code: string; name: string; flag: string }[]; languages?: { code: string; name: string }[] } = {};89 await Promise.all(([["channels", 40], ["feeds", 40], ["logos", 40], ["guides", 80], ["blocklist", 5], ["categories", 1], ["countries", 1], ["languages", 5]] as const).map(async ([n, mb]) => {90 try { (meta as Record<string, unknown>)[n] = await api(n, mb); } catch (e) { res.apiErrors.push(`${n}.json : ${(e as Error).message}`); }91 }));92 const chanById = new Map((meta.channels ?? []).map((c) => [c.id, c]));93 const blocked = new Set((meta.blocklist ?? []).map((b) => b.channel));94 const catByName = new Map((meta.categories ?? all<{ id: string; name: string }>("SELECT id, name FROM categories")).map((c) => [c.name.toLowerCase(), c.id]));9596 // 2. Playlists, in source order (the main IPTV-org playlist first, so it owns the streams it lists).97 // Order = ownership of a stream listed twice: the playback priority set in the administration first, then the verified selection, then the built-in lists.98 const sources = all<LiveSource>("SELECT id, name, url, country, kind, config, secret, content FROM sources WHERE enabled = 1 ORDER BY priority, (kind = 'curated') DESC, builtin DESC, id");99 const premiumFilter = getSetting("premium_filter") !== "0";100 const sourceKind = new Map(sources.map((s) => [s.id, s.kind]));101 const perSource = new Map<number, number>();102 const okSources: number[] = [];103 const byUrl = new Map<string, { e: M3uEntry; sourceId: number; country: string | null }>();104 const regionOf = new Map<string, string>(); // stream url → country of the regional playlist that lists it105 for (const s of sources) {106 try {107 // Whatever the connector (playlist address, imported file, single stream, provider API), a live source yields playlist entries.108 const parsed = await loadLive(s, (name) => fs.readFileSync(path.join(/*turbopackIgnore: true*/ process.cwd(), "sources", path.basename(name)), "utf8"));109 res.parseErrors += parsed.errors.length;110 for (const e of parsed.entries) {111 if (!/^(https?|provider):\/\//i.test(e.url)) { res.skippedProtocol++; continue; }112 let host = ""; try { host = new URL(e.url).hostname; } catch { res.skippedProtocol++; continue; }113 if (PAGE_HOSTS.test(host)) { res.skippedPages++; continue; }114 if (e.channelId && (blocked.has(e.channelId) || chanById.get(e.channelId)?.is_nsfw)) { res.skippedBlocked++; continue; }115 if (premiumFilter && s.kind !== "fast" && s.kind !== "curated" && s.kind !== "xtream" && looksPremium(chanById.get(e.channelId ?? "")?.name ?? e.name)) { res.skippedPremium++; continue; }116 if (s.country) regionOf.set(e.url, s.country);117 if (byUrl.has(e.url)) { res.duplicates++; continue; }118 byUrl.set(e.url, { e, sourceId: s.id, country: s.country ?? e.country });119 perSource.set(s.id, (perSource.get(s.id) ?? 0) + 1);120 }121 okSources.push(s.id);122 res.sources.push({ id: s.id, name: s.name, ok: true, entries: parsed.entries.length, kept: perSource.get(s.id) ?? 0 });123 run("UPDATE sources SET last_sync_at = ?, last_status = 'ok', last_error = NULL, stream_count = ? WHERE id = ?", now(), parsed.entries.length, s.id);124 } catch (e) {125 const msg = (e as Error).message;126 res.sources.push({ id: s.id, name: s.name, ok: false, entries: 0, error: msg });127 run("UPDATE sources SET last_sync_at = ?, last_status = 'error', last_error = ? WHERE id = ?", now(), msg, s.id);128 log("error", "sync", `Source en échec : ${s.name}`, { url: /^https?:/.test(s.url) ? s.url : s.kind, error: msg });129 }130 }131132 // 3. Group streams into channels.133 const feedsByChannel = new Map<string, ApiFeed[]>();134 for (const f of meta.feeds ?? []) { const l = feedsByChannel.get(f.channel); if (l) l.push(f); else feedsByChannel.set(f.channel, [f]); }135 const logoByChannel = new Map<string, string>();136 for (const l of meta.logos ?? []) if (l.in_use && (!logoByChannel.has(l.channel) || l.feed === null)) logoByChannel.set(l.channel, l.url);137138 interface Chan { id: string; name: string; alt: string; country: string | null; categories: string; languages: Set<string>; logo: string | null; website: string | null; network: string | null; known: number }139 const chans = new Map<string, Chan>();140 const streamChannel = new Map<string, string>();141 // Name index of the IPTV-org database: lets a stream without a usable tvg-id join the channel it is a variant of.142 const byName = new Map<string, ApiChannel[]>();143 for (const c of meta.channels ?? []) {144 if (c.is_nsfw || blocked.has(c.id)) continue;145 for (const n of new Set([c.name, ...c.alt_names].map(cleanName))) { if (!n) continue; const l = byName.get(n); if (l) l.push(c); else byName.set(n, [c]); }146 }147 for (const [url, { e, country, sourceId }] of byUrl) {148 let known = e.channelId ? chanById.get(e.channelId) : undefined;149 if (!known && sourceKind.get(sourceId) !== "iptv-org") {150 // Same name and same country, or a name that designates a single channel worldwide: same channel.151 const cands = byName.get(cleanName(e.name)) ?? [];152 const hit = country ? cands.filter((c) => c.country === country) : cands;153 if (hit.length === 1) { known = hit[0]; res.mergedByName++; }154 }155 const id = known ? known.id : sourceKind.get(sourceId) === "iptv-org" || !country ? `x-${slugKey(e.name) || "sans-nom"}` : `x-${country.toLowerCase()}-${slugKey(e.name) || "sans-nom"}`;156 let c = chans.get(id);157 if (!c) {158 c = known159 ? { id, name: known.name, alt: known.alt_names.join("|"), country: known.country, categories: known.categories.join(";"), languages: new Set(), logo: e.logo || logoByChannel.get(id) || null, website: known.website, network: known.network, known: 1 }160 : { id, name: e.name, alt: "", country: country ?? regionOf.get(url) ?? null, categories: e.group.map((g) => catByName.get(g.toLowerCase())).filter(Boolean).join(";"), languages: new Set(), logo: e.logo, website: null, network: null, known: 0 };161 chans.set(id, c);162 }163 if (!c.logo && e.logo) c.logo = e.logo;164 if (!c.country) c.country = regionOf.get(url) ?? null;165 if (known) {166 const feeds = feedsByChannel.get(id) ?? [];167 const f = feeds.find((x) => x.id === e.feed) ?? feeds.find((x) => x.is_main);168 for (const l of f?.languages ?? []) c.languages.add(l);169 }170 streamChannel.set(url, id);171 }172173 // 4. Write. One transaction: readers keep seeing the previous catalogue until it commits.174 const firstSync = !get("SELECT 1 FROM settings WHERE key = 'initial_sync_at'");175 const d = db();176 const t = now();177 tx(() => {178 if (meta.categories) { run("DELETE FROM categories"); const st = d.prepare("INSERT INTO categories(id, name) VALUES(?, ?)"); for (const c of meta.categories) st.run(c.id, c.name); }179 if (meta.countries) { run("DELETE FROM countries"); const st = d.prepare("INSERT INTO countries(code, name, flag) VALUES(?, ?, ?)"); for (const c of meta.countries) st.run(c.code, c.name, c.flag); }180 if (meta.languages) { run("DELETE FROM languages"); const st = d.prepare("INSERT OR IGNORE INTO languages(code, name) VALUES(?, ?)"); for (const c of meta.languages) st.run(c.code, c.name); }181182 const existing = new Set(all<{ id: string }>("SELECT id FROM channels").map((r) => r.id));183 const upChan = d.prepare(`INSERT INTO channels(id, name, alt_names, country, categories, languages, logo, website, network, known, active, first_seen, updated_at)184 VALUES(?,?,?,?,?,?,?,?,?,?,1,?,?)185 ON CONFLICT(id) DO UPDATE SET name=excluded.name, alt_names=excluded.alt_names, country=COALESCE(excluded.country, country), categories=excluded.categories,186 languages=excluded.languages, logo=COALESCE(excluded.logo, logo), website=excluded.website, network=excluded.network, known=excluded.known, active=1, updated_at=excluded.updated_at`);187 for (const c of chans.values()) {188 if (!existing.has(c.id)) res.newChannels++;189 upChan.run(c.id, c.name, c.alt, c.country, c.categories, [...c.languages].join(";"), c.logo, c.website, c.network, c.known, t, t);190 }191 const existingUrls = new Set(all<{ url: string }>("SELECT url FROM streams").map((r) => r.url));192 const upStream = d.prepare(`INSERT INTO streams(channel_id, source_id, url, title, feed, quality, labels, referrer, user_agent, first_seen, last_seen, tvg_id)193 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)194 ON CONFLICT(url) DO UPDATE SET tvg_id=excluded.tvg_id, channel_id=excluded.channel_id, title=excluded.title, feed=excluded.feed, quality=excluded.quality, labels=excluded.labels,195 referrer=excluded.referrer, user_agent=excluded.user_agent, last_seen=excluded.last_seen`);196 for (const [url, { e, sourceId }] of byUrl) {197 if (!existingUrls.has(url)) res.newStreams++;198 upStream.run(streamChannel.get(url)!, sourceId, url, e.rawTitle, e.feed, e.quality, e.labels.join(";"), e.referrer, e.userAgent, t, t, e.tvgId);199 }200 // Streams that vanished from a source that synced correctly are removed; a failed source keeps its streams.201 if (okSources.length) {202 const marks = okSources.map(() => "?").join(",");203 res.removedStreams = Number(run(`DELETE FROM streams WHERE last_seen < ? AND source_id IN (${marks})`, t, ...okSources).changes);204 }205 run("UPDATE channels SET active = EXISTS(SELECT 1 FROM streams s WHERE s.channel_id = channels.id)");206 // A channel without streams survives only if the user kept something about it (favourite, hidden, history).207 run(`DELETE FROM channels WHERE active = 0 AND id NOT IN (SELECT channel_id FROM list_items UNION SELECT channel_id FROM hidden UNION SELECT channel_id FROM history)`);208209 if (meta.guides) {210 run("DELETE FROM guides");211 const st = d.prepare("INSERT INTO guides(channel_id, feed, site, site_id, site_name, lang) VALUES(?,?,?,?,?,?)");212 for (const g of meta.guides) if (g.channel && chans.has(g.channel)) st.run(g.channel, g.feed, g.site, g.site_id, g.site_name, g.lang);213 run("UPDATE channels SET has_guide = EXISTS(SELECT 1 FROM guides g WHERE g.channel_id = channels.id)");214 }215 if (firstSync && okSources.length) setSetting("initial_sync_at", String(t));216 setSetting("last_sync_at", String(t));217 });218 res.channels = get<{ n: number }>("SELECT COUNT(*) n FROM channels WHERE active = 1")!.n;219 res.streams = get<{ n: number }>("SELECT COUNT(*) n FROM streams")!.n;220 res.durationMs = now() - t0;221 bumpCatalog();222 log(res.sources.some((s) => !s.ok) || res.apiErrors.length ? "warn" : "info", "sync", `Synchronisation terminée : ${res.channels} chaînes, ${res.streams} flux`, res);223 return res;224}225