|
|
@@ -0,0 +1,312 @@
|
|
|
+// Curated IPTV playlist builder — sources from a clone of iptv-org/iptv.
|
|
|
+// The clone + validation cache live in /tmp (portable, throwaway — nothing lands
|
|
|
+// in the repo). Writes iptv.m3u next to this script; deploy.ts publishes it to GCS.
|
|
|
+//
|
|
|
+// node build-iptv.mjs --dry assemble + counts only, no validation
|
|
|
+// node build-iptv.mjs assemble + validate (cached) + write iptv.m3u
|
|
|
+// node build-iptv.mjs --revalidate re-probe every stream, ignoring the cache
|
|
|
+//
|
|
|
+// Update upstream first with: git -C /tmp/iptv-org-build/iptv-org-repo pull
|
|
|
+import { readFileSync, writeFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs';
|
|
|
+import { execSync } from 'node:child_process';
|
|
|
+import { fileURLToPath } from 'node:url';
|
|
|
+import path from 'node:path';
|
|
|
+
|
|
|
+const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
|
+const TMP = '/tmp/iptv-org-build'; // clone + probe cache: portable, throwaway
|
|
|
+const REPO = path.join(TMP, 'iptv-org-repo');
|
|
|
+const STREAMS = path.join(REPO, 'streams');
|
|
|
+const VCACHE = path.join(TMP, 'validation.json');
|
|
|
+const OUT = path.join(HERE, 'iptv.m3u');
|
|
|
+const REPORT = path.join(HERE, 'iptv-build-report.txt');
|
|
|
+const RAW = 'https://raw.githubusercontent.com/iptv-org/iptv/master/streams';
|
|
|
+const DRY = process.argv.includes('--dry');
|
|
|
+const REVALIDATE = process.argv.includes('--revalidate');
|
|
|
+const CONCURRENCY = 50;
|
|
|
+const TIMEOUT_MS = 7000;
|
|
|
+
|
|
|
+// ---------- bootstrap upstream clone ----------
|
|
|
+if (!existsSync(STREAMS)) {
|
|
|
+ mkdirSync(TMP, { recursive: true });
|
|
|
+ process.stderr.write(`Cloning iptv-org/iptv (shallow) into ${TMP} ...\n`);
|
|
|
+ execSync(`git clone --depth 1 https://github.com/iptv-org/iptv.git "${REPO}"`, { stdio: 'inherit' });
|
|
|
+}
|
|
|
+
|
|
|
+// ---------- parsing ----------
|
|
|
+const allFiles = readdirSync(STREAMS).filter((f) => f.endsWith('.m3u'));
|
|
|
+const cache = {};
|
|
|
+function parse(file) {
|
|
|
+ if (cache[file]) return cache[file];
|
|
|
+ let text;
|
|
|
+ try { text = readFileSync(path.join(STREAMS, file), 'utf8'); }
|
|
|
+ catch { return (cache[file] = []); }
|
|
|
+ const lines = text.split('\n');
|
|
|
+ const out = [];
|
|
|
+ for (let i = 0; i < lines.length; i++) {
|
|
|
+ if (!lines[i].startsWith('#EXTINF')) continue;
|
|
|
+ const extinf = lines[i];
|
|
|
+ const comma = extinf.indexOf(',');
|
|
|
+ const name = extinf.slice(comma + 1).trim();
|
|
|
+ let url = '';
|
|
|
+ for (let j = i + 1; j < lines.length; j++) {
|
|
|
+ const t = lines[j].trim();
|
|
|
+ if (!t) continue;
|
|
|
+ if (t.startsWith('#')) { if (t.startsWith('#EXTINF')) break; continue; }
|
|
|
+ url = t; break;
|
|
|
+ }
|
|
|
+ if (!url) continue;
|
|
|
+ const m = name.match(/\((\d+)p\)/);
|
|
|
+ out.push({ name, url, extinf, sourceFile: file, quality: m ? Number(m[1]) : 0,
|
|
|
+ geo: /\[Geo-blocked\]/i.test(name) });
|
|
|
+ }
|
|
|
+ return (cache[file] = out);
|
|
|
+}
|
|
|
+const everyEntry = () => allFiles.flatMap(parse);
|
|
|
+const filesEntries = (files) => files.flatMap(parse);
|
|
|
+
|
|
|
+// ---------- curated matchers ----------
|
|
|
+const NEWS = [
|
|
|
+ /^BBC News/i, /^Al Jazeera English/i, /^France 24( \(|$)/i, /^France 24 English/i,
|
|
|
+ /^DW( \(|$)/i, /^DW English/i, /^Sky News( \(|$)/i, /^Sky News Australia/i,
|
|
|
+ /^Euronews( \(|$)/i, /^Euronews English/i, /^CNA( \(|$)/i, /^Channel News ?Asia/i,
|
|
|
+ /^TRT World/i, /^Africanews( \(|$)/i, /^Africanews English/i, /^GB News/i, /^NHK World/i,
|
|
|
+ /^CGTN( \(|$)/i, /^CGTN Documentary/i, /^WION/i, /^Arirang TV( \(|$)/i, /^Arirang( \(|$)/i,
|
|
|
+ /^i24NEWS/i, /^TaiwanPlus/i, /^Bloomberg (Television|Originals|TV US|TV Australia|TV Plus|TV\+)/i,
|
|
|
+ /^Bloomberg TV( \(|$)/i, /^CNN International/i, /^CNN( \(|$)/i, /^ABC News Live( \(|\[|$)/i,
|
|
|
+ /^ABC News( \(|$)/i, /^NBC News Now/i, /^CBS News( \(|$|\[)/i, /^Scripps News/i,
|
|
|
+ /^LiveNOW from FOX/i, /^CBC News( \(|$)/i, /^CBC News Network/i, /^Newsmax/i, /^Global News National/i,
|
|
|
+];
|
|
|
+const NEWS_EXCLUDE = /Pashto|Arabic|Mubasher|Русск|Fran[cç]ais|Cha[iî]ne|Portugu|Deutsch|Espa[nñ]ol|Hebrew|Radio|Hindi|Urdu|Bangla|Mongolia|Bulgaria/i;
|
|
|
+const ESP_NEWS = [
|
|
|
+ /CNN en Espa/i, /France 24.*Espa/i, /^DW Espa/i, /Euronews.*Espa/i, /CGTN Espa/i,
|
|
|
+ /Deutsche Welle.*Espa/i, /Telemundo/i, /Univision/i, /TUDN/i, /Al Jazeera.*Espa/i,
|
|
|
+];
|
|
|
+// AR/CO/CL terrestrial & notable — Latin immersion (skips Comedy Central / flagship news already covered)
|
|
|
+const LATAM = [
|
|
|
+ /^Telefe/i, /^El Trece/i, /^El Nueve/i, /^TV Publica/i, /^Am[ée]rica TV/i, /^A24/i,
|
|
|
+ /^Bravo TV/i, /^Garage TV/i, /^Canal 9 /i,
|
|
|
+ /^Caracol/i, /RCN/i, /^Canal 1 /i, /^Se[ñn]al Colombia/i, /^Canal Capital/i,
|
|
|
+ /^Canal Institucional/i, /^NTN24/i, /^Teleantioquia/i, /^Telecaf/i, /^Telecaribe/i,
|
|
|
+ /^Telemedell/i, /^Cosmovision/i,
|
|
|
+ /^TVN( |3|$)/i, /^Mega( |noticias|$)/i, /^Canal 13/i, /^La Red/i, /^TV\+/i, /^UCV TV/i,
|
|
|
+];
|
|
|
+// Bespoke European terrestrial / region-defining (no Comedy Central, no flagship news dupes)
|
|
|
+const EUROPE = [
|
|
|
+ /^France 2( \(|$)/i, /^France 3( |\(|$)/i, /^France 4/i, /^France 5( \(|$)/i, /^Gulli/i, /^BFM TV/i, /^Arte/i,
|
|
|
+ /^Das Erste/i, /^ProSieben/i, /^3sat/i, /^Tagesschau 24/i, /^WDR Fernsehen( \(|$)/i,
|
|
|
+ /^NDR Fernsehen International/i, /^ZDF( \(|$)/i, /^ZDFneo/i,
|
|
|
+ /^Rai 1 HD/i, /^Rai 2 HD/i, /^Rai 3( \(|$)/i, /^Rai 4( \(|$)/i, /^Rai News/i, /^Rai Storia/i, /^Rai Movie/i, /^La7/i,
|
|
|
+ /^Canale 5/i, /^Italia 1/i, /^Rete 4/i,
|
|
|
+ /^La 1( \(|$)/i, /^Antena 3 Internacional/i, /^Canal Sur Andaluc/i, /^ETB ?[12] On/i,
|
|
|
+ /^BBC One( \(|$)/i, /^BBC Two( \(|$)/i, /^BBC Four/i, /^Channel 4( \(|$)/i, /^Channel 5( \(|$)/i, /^Film4/i,
|
|
|
+];
|
|
|
+const isNews = (n) => NEWS.some((r) => r.test(n)) && !NEWS_EXCLUDE.test(n);
|
|
|
+const isComedyCentral = (n) => /comedy central/i.test(n);
|
|
|
+const isEspNews = (n) => ESP_NEWS.some((r) => r.test(n));
|
|
|
+const isLatam = (n) => LATAM.some((r) => r.test(n));
|
|
|
+const isEurope = (n) => EUROPE.some((r) => r.test(n));
|
|
|
+
|
|
|
+// Streams confirmed good by hand (e.g. via ffprobe) that the HTTP probe flaps on.
|
|
|
+const FORCE_KEEP = [/^Comedy Central \(720p\)/i];
|
|
|
+
|
|
|
+// ---------- section assembly (priority order; first to claim a URL wins) ----------
|
|
|
+const seen = new Set();
|
|
|
+const sections = [];
|
|
|
+function add(meta, entries) {
|
|
|
+ const kept = [];
|
|
|
+ for (const e of entries) {
|
|
|
+ if (seen.has(e.url)) continue;
|
|
|
+ seen.add(e.url);
|
|
|
+ kept.push(e);
|
|
|
+ }
|
|
|
+ sections.push({ ...meta, entries: kept });
|
|
|
+}
|
|
|
+
|
|
|
+add({ title: 'News — flagships & world', group: 'News', dedupeByName: true,
|
|
|
+ sources: ['(cross-file pattern match across all country files)'],
|
|
|
+ filter: 'name matches flagship/major-news whitelist' },
|
|
|
+ everyEntry().filter((e) => isNews(e.name)));
|
|
|
+
|
|
|
+add({ title: 'Comedy Central', group: 'Comedy', dedupeByName: false,
|
|
|
+ sources: ['(cross-file pattern match across all country files)'],
|
|
|
+ filter: 'name contains "Comedy Central"' },
|
|
|
+ everyEntry().filter((e) => isComedyCentral(e.name)));
|
|
|
+
|
|
|
+add({ title: 'Espanol (Latin) — Mexico + Spanish news', group: 'Espanol', dedupeByName: false,
|
|
|
+ sources: ['mx.m3u', 'mx_pluto.m3u', 'mx_samsung.m3u', 'us_canelatv.m3u', '(+ Spanish-language news cross-file)'],
|
|
|
+ filter: 'all entries from MX/Canela files + Spanish-language news from any file' },
|
|
|
+ [...filesEntries(['mx.m3u', 'mx_pluto.m3u', 'mx_samsung.m3u', 'us_canelatv.m3u']),
|
|
|
+ ...everyEntry().filter((e) => isEspNews(e.name))]);
|
|
|
+
|
|
|
+add({ title: 'Latin America — AR/CO/CL terrestrial', group: 'LatAm', dedupeByName: false,
|
|
|
+ sources: ['ar.m3u', 'co.m3u', 'cl.m3u'],
|
|
|
+ filter: 'curated terrestrial/regional whitelist (Argentina, Colombia, Chile)' },
|
|
|
+ filesEntries(['ar.m3u', 'co.m3u', 'cl.m3u']).filter((e) => isLatam(e.name)));
|
|
|
+
|
|
|
+add({ title: 'Europe — terrestrial & regional', group: 'Europe', dedupeByName: false,
|
|
|
+ sources: ['fr.m3u', 'de.m3u', 'it.m3u', 'es.m3u', 'nl.m3u', 'uk.m3u'],
|
|
|
+ filter: 'curated iconic/terrestrial whitelist (excludes news & Comedy Central already covered)' },
|
|
|
+ filesEntries(['fr.m3u', 'de.m3u', 'it.m3u', 'es.m3u', 'nl.m3u', 'uk.m3u']).filter((e) => isEurope(e.name)));
|
|
|
+
|
|
|
+const bulk = [
|
|
|
+ ['Australia', 'AU', 'au.m3u'],
|
|
|
+ ['Australia — Samsung TV Plus', 'AU Samsung', 'au_samsung.m3u'],
|
|
|
+ ['US — Broadcast', 'US', 'us.m3u'],
|
|
|
+ ['US — Pluto TV', 'US Pluto', 'us_pluto.m3u'],
|
|
|
+ ['US — Samsung TV Plus', 'US Samsung', 'us_samsung.m3u'],
|
|
|
+ ['US — Tubi', 'US Tubi', 'us_tubi.m3u'],
|
|
|
+ ['US — Xumo', 'US Xumo', 'us_xumo.m3u'],
|
|
|
+];
|
|
|
+for (const [title, group, file] of bulk)
|
|
|
+ add({ title, group, dedupeByName: false, sources: [file], filter: 'all entries' }, parse(file));
|
|
|
+
|
|
|
+// ---------- validation (HTTP reachability, cached by URL) ----------
|
|
|
+async function check(url) {
|
|
|
+ const ctrl = new AbortController();
|
|
|
+ const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
|
|
|
+ try {
|
|
|
+ const res = await fetch(url, { method: 'GET', redirect: 'follow', signal: ctrl.signal,
|
|
|
+ headers: { 'User-Agent': 'VLC/3.0.20 LibVLC/3.0.20' } });
|
|
|
+ try { await res.body?.cancel(); } catch {}
|
|
|
+ return res.status;
|
|
|
+ } catch { return 0; }
|
|
|
+ finally { clearTimeout(timer); }
|
|
|
+}
|
|
|
+async function pool(items, n, fn) {
|
|
|
+ const ret = new Array(items.length);
|
|
|
+ let i = 0, done = 0;
|
|
|
+ async function worker() {
|
|
|
+ while (i < items.length) {
|
|
|
+ const idx = i++;
|
|
|
+ ret[idx] = await fn(items[idx]);
|
|
|
+ if (++done % 150 === 0) process.stderr.write(` probed ${done}/${items.length}\n`);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ await Promise.all(Array.from({ length: n }, worker));
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+
|
|
|
+const allEntries = sections.flatMap((s) => s.entries);
|
|
|
+process.stderr.write(`Assembled ${allEntries.length} unique candidates across ${sections.length} sections.\n`);
|
|
|
+
|
|
|
+if (!DRY) {
|
|
|
+ let vcache = {};
|
|
|
+ if (!REVALIDATE) { try { vcache = JSON.parse(readFileSync(VCACHE, 'utf8')); } catch {} }
|
|
|
+ const toProbe = allEntries.filter((e) => !(e.url in vcache));
|
|
|
+ process.stderr.write(`Validating ${allEntries.length} streams — ${toProbe.length} to probe, ${allEntries.length - toProbe.length} cached.\n`);
|
|
|
+ const statuses = await pool(toProbe, CONCURRENCY, (e) => check(e.url));
|
|
|
+ toProbe.forEach((e, k) => { vcache[e.url] = statuses[k]; });
|
|
|
+ mkdirSync(TMP, { recursive: true });
|
|
|
+ writeFileSync(VCACHE, JSON.stringify(vcache));
|
|
|
+ for (const e of allEntries) {
|
|
|
+ const st = vcache[e.url] ?? 0;
|
|
|
+ e.status = st;
|
|
|
+ e.alive = (st >= 200 && st < 400) || FORCE_KEEP.some((r) => r.test(e.name));
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// ---------- disambiguation ----------
|
|
|
+// When a channel name collides within a section, append a source tag (country,
|
|
|
+// then provider, then quality) — but only the dimensions that actually differ, so
|
|
|
+// single-source dupes don't get noise like "(AU)". Unique names are left untouched.
|
|
|
+function disambiguate(entries) {
|
|
|
+ const baseKey = (n) => n.toLowerCase()
|
|
|
+ .replace(/\(\d+p\)|\[[^\]]*\]|\((?:hevc|adaptive)\)/gi, '').replace(/\s+/g, ' ').trim();
|
|
|
+ const cc = (e) => (e.sourceFile.split(/[._]/)[0] || '').toUpperCase();
|
|
|
+ const prov = (e) => {
|
|
|
+ const p = e.sourceFile.replace(/\.m3u$/, '').split('_').slice(1).join(' ');
|
|
|
+ return p ? p.replace(/\b\w/g, (c) => c.toUpperCase()) : '';
|
|
|
+ };
|
|
|
+ const nameWith = (e, dims) => {
|
|
|
+ const tag = dims.map((d) => d(e)).filter(Boolean).join(' ');
|
|
|
+ return tag ? `${e.name} (${tag})` : e.name;
|
|
|
+ };
|
|
|
+ const groups = new Map();
|
|
|
+ for (const e of entries) {
|
|
|
+ const k = baseKey(e.name);
|
|
|
+ if (!groups.has(k)) groups.set(k, []);
|
|
|
+ groups.get(k).push(e);
|
|
|
+ }
|
|
|
+ for (const g of groups.values()) {
|
|
|
+ if (g.length < 2) continue;
|
|
|
+ // add the fewest source dimensions (country, then provider) needed to make
|
|
|
+ // names unique; quality already lives in the name so it's never re-appended.
|
|
|
+ const distinct = (dims) => new Set(g.map((e) => nameWith(e, dims))).size;
|
|
|
+ let active = [];
|
|
|
+ let best = distinct(active);
|
|
|
+ if (best < g.length) {
|
|
|
+ for (const d of [cc, prov]) {
|
|
|
+ const dn = distinct([...active, d]);
|
|
|
+ if (dn > best) { active = [...active, d]; best = dn; if (best === g.length) break; }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ const counts = {};
|
|
|
+ for (const e of g) {
|
|
|
+ let n = nameWith(e, active);
|
|
|
+ counts[n] = (counts[n] || 0) + 1;
|
|
|
+ if (counts[n] > 1) n = `${n} (${counts[n]})`;
|
|
|
+ if (n !== e.name) e.displayName = n;
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// ---------- post-process + write ----------
|
|
|
+const qrank = (e) => e.quality || 0;
|
|
|
+function finalize(s) {
|
|
|
+ let entries = DRY ? s.entries : s.entries.filter((e) => e.alive);
|
|
|
+ if (s.dedupeByName) {
|
|
|
+ const byName = new Map();
|
|
|
+ for (const e of entries) {
|
|
|
+ const key = e.name.toLowerCase().replace(/\(\d+p\)|\[[^\]]*\]/g, '').replace(/\s+/g, ' ').trim();
|
|
|
+ const cur = byName.get(key);
|
|
|
+ if (!cur || qrank(e) > qrank(cur)) byName.set(key, e);
|
|
|
+ }
|
|
|
+ entries = [...byName.values()];
|
|
|
+ }
|
|
|
+ entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
|
+ return entries;
|
|
|
+}
|
|
|
+function extinfLine(e, group) {
|
|
|
+ const comma = e.extinf.indexOf(',');
|
|
|
+ const attrs = e.extinf.slice(0, comma).replace(/\s*group-title="[^"]*"/, '');
|
|
|
+ return `${attrs} group-title="${group}",${e.displayName ?? e.name}`;
|
|
|
+}
|
|
|
+
|
|
|
+const lines = ['#EXTM3U'];
|
|
|
+const reportLines = [];
|
|
|
+lines.push(
|
|
|
+ '# ============================================================',
|
|
|
+ '# Curated IPTV playlist — source: iptv-org/iptv',
|
|
|
+ '# Built by: build-iptv.mjs (`bun refresh.ts` = playlist + EPG, `bun deploy.ts` = publish)',
|
|
|
+ '# Rebuild: bun build-iptv.mjs (cached, fast)',
|
|
|
+ '# Update: git -C /tmp/iptv-org-build/iptv-org-repo pull && bun build-iptv.mjs --revalidate',
|
|
|
+ `# Validation: ${DRY ? 'SKIPPED (dry run)' : 'HTTP reachability probed from this machine; dead/geo-blocked dropped'}`,
|
|
|
+ '# Colliding names get a source tag, e.g. "Comedy Central (US)" vs "(DK)".',
|
|
|
+ '# Per-section comments below record upstream sources so this can be regenerated.',
|
|
|
+ '# ============================================================');
|
|
|
+
|
|
|
+let totalKept = 0;
|
|
|
+for (const s of sections) {
|
|
|
+ const finals = finalize(s);
|
|
|
+ disambiguate(finals);
|
|
|
+ totalKept += finals.length;
|
|
|
+ reportLines.push(`${s.title.padEnd(40)} kept ${String(finals.length).padStart(4)} / ${String(s.entries.length).padStart(4)} candidates`);
|
|
|
+ lines.push('', '# ------------------------------------------------------------',
|
|
|
+ `# SECTION: ${s.title} (group-title="${s.group}")`,
|
|
|
+ ...s.sources.map((src) => src.startsWith('(') ? `# Upstream: ${src}` : `# Upstream: ${RAW}/${src}`),
|
|
|
+ `# Filter: ${s.filter}${s.dedupeByName ? ' + deduped by channel name (best quality)' : ''}`,
|
|
|
+ `# ${DRY ? 'Candidates' : 'Alive'}: ${finals.length}${DRY ? '' : ` / ${s.entries.length}`}`,
|
|
|
+ '# ------------------------------------------------------------');
|
|
|
+ for (const e of finals) {
|
|
|
+ if (/^(News|Comedy|Espanol|Latin|Europe)/.test(s.title)) lines.push(`# src: ${e.sourceFile}`);
|
|
|
+ lines.push(extinfLine(e, s.group), e.url);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+mkdirSync(path.dirname(OUT), { recursive: true });
|
|
|
+writeFileSync(OUT, lines.join('\n') + '\n');
|
|
|
+const report = ['IPTV build report (' + (DRY ? 'DRY — not validated' : 'validated') + ')',
|
|
|
+ '='.repeat(62), ...reportLines, '-'.repeat(62),
|
|
|
+ `TOTAL kept: ${totalKept}`, `Output: ${OUT}`].join('\n');
|
|
|
+writeFileSync(REPORT, report + '\n');
|
|
|
+process.stderr.write('\n' + report + '\n');
|
