iptv/build-iptv.mjs

// Curated IPTV playlist builder — sources from a clone of iptv-org/iptv.
// The clone + validation cache live in /tmp (portable, throwaway — nothing lands
// in the repo). Writes iptv.m3u next to this script; deploy.ts publishes it to GCS.
//
//   node build-iptv.mjs --dry          assemble + counts only, no validation
//   node build-iptv.mjs                assemble + validate (cached) + write iptv.m3u
//   node build-iptv.mjs --revalidate   re-probe every stream, ignoring the cache
//
// Update upstream first with: git -C /tmp/iptv-org-build/iptv-org-repo pull
import { readFileSync, writeFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs';
import { execSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import path from 'node:path';

const HERE = path.dirname(fileURLToPath(import.meta.url));
const TMP = '/tmp/iptv-org-build';            // clone + probe cache: portable, throwaway
const REPO = path.join(TMP, 'iptv-org-repo');
const STREAMS = path.join(REPO, 'streams');
const VCACHE = path.join(TMP, 'validation.json');
const OUT = path.join(HERE, 'iptv.m3u');
const REPORT = path.join(HERE, 'iptv-build-report.txt');
const RAW = 'https://raw.githubusercontent.com/iptv-org/iptv/master/streams';
const DRY = process.argv.includes('--dry');
const REVALIDATE = process.argv.includes('--revalidate');
const CONCURRENCY = 50;
const TIMEOUT_MS = 7000;

// ---------- bootstrap upstream clone ----------
if (!existsSync(STREAMS)) {
  mkdirSync(TMP, { recursive: true });
  process.stderr.write(`Cloning iptv-org/iptv (shallow) into ${TMP} ...\n`);
  execSync(`git clone --depth 1 https://github.com/iptv-org/iptv.git "${REPO}"`, { stdio: 'inherit' });
}

// ---------- parsing ----------
const allFiles = readdirSync(STREAMS).filter((f) => f.endsWith('.m3u'));
const cache = {};
function parse(file) {
  if (cache[file]) return cache[file];
  let text;
  try { text = readFileSync(path.join(STREAMS, file), 'utf8'); }
  catch { return (cache[file] = []); }
  const lines = text.split('\n');
  const out = [];
  for (let i = 0; i < lines.length; i++) {
    if (!lines[i].startsWith('#EXTINF')) continue;
    const extinf = lines[i];
    const comma = extinf.indexOf(',');
    const name = extinf.slice(comma + 1).trim();
    let url = '';
    for (let j = i + 1; j < lines.length; j++) {
      const t = lines[j].trim();
      if (!t) continue;
      if (t.startsWith('#')) { if (t.startsWith('#EXTINF')) break; continue; }
      url = t; break;
    }
    if (!url) continue;
    const m = name.match(/\((\d+)p\)/);
    out.push({ name, url, extinf, sourceFile: file, quality: m ? Number(m[1]) : 0,
      geo: /\[Geo-blocked\]/i.test(name) });
  }
  return (cache[file] = out);
}
const everyEntry = () => allFiles.flatMap(parse);
const filesEntries = (files) => files.flatMap(parse);

// ---------- curated matchers ----------
const NEWS = [
  /^BBC News/i, /^Al Jazeera English/i, /^France 24( \(|$)/i, /^France 24 English/i,
  /^DW( \(|$)/i, /^DW English/i, /^Sky News( \(|$)/i, /^Sky News Australia/i,
  /^Euronews( \(|$)/i, /^Euronews English/i, /^CNA( \(|$)/i, /^Channel News ?Asia/i,
  /^TRT World/i, /^Africanews( \(|$)/i, /^Africanews English/i, /^GB News/i, /^NHK World/i,
  /^CGTN( \(|$)/i, /^CGTN Documentary/i, /^WION/i, /^Arirang TV( \(|$)/i, /^Arirang( \(|$)/i,
  /^i24NEWS/i, /^TaiwanPlus/i, /^Bloomberg (Television|Originals|TV US|TV Australia|TV Plus|TV\+)/i,
  /^Bloomberg TV( \(|$)/i, /^CNN International/i, /^CNN( \(|$)/i, /^ABC News Live( \(|\[|$)/i,
  /^ABC News( \(|$)/i, /^NBC News Now/i, /^CBS News( \(|$|\[)/i, /^Scripps News/i,
  /^LiveNOW from FOX/i, /^CBC News( \(|$)/i, /^CBC News Network/i, /^Newsmax/i, /^Global News National/i,
];
const NEWS_EXCLUDE = /Pashto|Arabic|Mubasher|Русск|Fran[cç]ais|Cha[iî]ne|Portugu|Deutsch|Espa[nñ]ol|Hebrew|Radio|Hindi|Urdu|Bangla|Mongolia|Bulgaria/i;
const ESP_NEWS = [
  /CNN en Espa/i, /France 24.*Espa/i, /^DW Espa/i, /Euronews.*Espa/i, /CGTN Espa/i,
  /Deutsche Welle.*Espa/i, /Telemundo/i, /Univision/i, /TUDN/i, /Al Jazeera.*Espa/i,
];
// AR/CO/CL terrestrial & notable — Latin immersion (skips Comedy Central / flagship news already covered)
const LATAM = [
  /^Telefe/i, /^El Trece/i, /^El Nueve/i, /^TV Publica/i, /^Am[ée]rica TV/i, /^A24/i,
  /^Bravo TV/i, /^Garage TV/i, /^Canal 9 /i,
  /^Caracol/i, /RCN/i, /^Canal 1 /i, /^Se[ñn]al Colombia/i, /^Canal Capital/i,
  /^Canal Institucional/i, /^NTN24/i, /^Teleantioquia/i, /^Telecaf/i, /^Telecaribe/i,
  /^Telemedell/i, /^Cosmovision/i,
  /^TVN( |3|$)/i, /^Mega( |noticias|$)/i, /^Canal 13/i, /^La Red/i, /^TV\+/i, /^UCV TV/i,
];
// Bespoke European terrestrial / region-defining (no Comedy Central, no flagship news dupes)
const EUROPE = [
  /^France 2( \(|$)/i, /^France 3( |\(|$)/i, /^France 4/i, /^France 5( \(|$)/i, /^Gulli/i, /^BFM TV/i, /^Arte/i,
  /^Das Erste/i, /^ProSieben/i, /^3sat/i, /^Tagesschau 24/i, /^WDR Fernsehen( \(|$)/i,
  /^NDR Fernsehen International/i, /^ZDF( \(|$)/i, /^ZDFneo/i,
  /^Rai 1 HD/i, /^Rai 2 HD/i, /^Rai 3( \(|$)/i, /^Rai 4( \(|$)/i, /^Rai News/i, /^Rai Storia/i, /^Rai Movie/i, /^La7/i,
  /^Canale 5/i, /^Italia 1/i, /^Rete 4/i,
  /^La 1( \(|$)/i, /^Antena 3 Internacional/i, /^Canal Sur Andaluc/i, /^ETB ?[12] On/i,
  /^BBC One( \(|$)/i, /^BBC Two( \(|$)/i, /^BBC Four/i, /^Channel 4( \(|$)/i, /^Channel 5( \(|$)/i, /^Film4/i,
];
const isNews = (n) => NEWS.some((r) => r.test(n)) && !NEWS_EXCLUDE.test(n);
const isComedyCentral = (n) => /comedy central/i.test(n);
const isEspNews = (n) => ESP_NEWS.some((r) => r.test(n));
const isLatam = (n) => LATAM.some((r) => r.test(n));
const isEurope = (n) => EUROPE.some((r) => r.test(n));

// Streams confirmed good by hand (e.g. via ffprobe) that the HTTP probe flaps on.
const FORCE_KEEP = [/^Comedy Central \(720p\)/i];

// ---------- section assembly (priority order; first to claim a URL wins) ----------
const seen = new Set();
const sections = [];
function add(meta, entries) {
  const kept = [];
  for (const e of entries) {
    if (seen.has(e.url)) continue;
    seen.add(e.url);
    kept.push(e);
  }
  sections.push({ ...meta, entries: kept });
}

add({ title: 'News — flagships & world', group: 'News', dedupeByName: true,
      sources: ['(cross-file pattern match across all country files)'],
      filter: 'name matches flagship/major-news whitelist' },
    everyEntry().filter((e) => isNews(e.name)));

add({ title: 'Comedy Central', group: 'Comedy', dedupeByName: false,
      sources: ['(cross-file pattern match across all country files)'],
      filter: 'name contains "Comedy Central"' },
    everyEntry().filter((e) => isComedyCentral(e.name)));

add({ title: 'Espanol (Latin) — Mexico + Spanish news', group: 'Espanol', dedupeByName: false,
      sources: ['mx.m3u', 'mx_pluto.m3u', 'mx_samsung.m3u', 'us_canelatv.m3u', '(+ Spanish-language news cross-file)'],
      filter: 'all entries from MX/Canela files + Spanish-language news from any file' },
    [...filesEntries(['mx.m3u', 'mx_pluto.m3u', 'mx_samsung.m3u', 'us_canelatv.m3u']),
     ...everyEntry().filter((e) => isEspNews(e.name))]);

add({ title: 'Latin America — AR/CO/CL terrestrial', group: 'LatAm', dedupeByName: false,
      sources: ['ar.m3u', 'co.m3u', 'cl.m3u'],
      filter: 'curated terrestrial/regional whitelist (Argentina, Colombia, Chile)' },
    filesEntries(['ar.m3u', 'co.m3u', 'cl.m3u']).filter((e) => isLatam(e.name)));

add({ title: 'Europe — terrestrial & regional', group: 'Europe', dedupeByName: false,
      sources: ['fr.m3u', 'de.m3u', 'it.m3u', 'es.m3u', 'nl.m3u', 'uk.m3u'],
      filter: 'curated iconic/terrestrial whitelist (excludes news & Comedy Central already covered)' },
    filesEntries(['fr.m3u', 'de.m3u', 'it.m3u', 'es.m3u', 'nl.m3u', 'uk.m3u']).filter((e) => isEurope(e.name)));

const bulk = [
  ['Australia', 'AU', 'au.m3u'],
  ['Australia — Samsung TV Plus', 'AU Samsung', 'au_samsung.m3u'],
  ['US — Broadcast', 'US', 'us.m3u'],
  ['US — Pluto TV', 'US Pluto', 'us_pluto.m3u'],
  ['US — Samsung TV Plus', 'US Samsung', 'us_samsung.m3u'],
  ['US — Tubi', 'US Tubi', 'us_tubi.m3u'],
  ['US — Xumo', 'US Xumo', 'us_xumo.m3u'],
];
for (const [title, group, file] of bulk)
  add({ title, group, dedupeByName: false, sources: [file], filter: 'all entries' }, parse(file));

// ---------- validation (HTTP reachability, cached by URL) ----------
async function check(url) {
  const ctrl = new AbortController();
  const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
  try {
    const res = await fetch(url, { method: 'GET', redirect: 'follow', signal: ctrl.signal,
      headers: { 'User-Agent': 'VLC/3.0.20 LibVLC/3.0.20' } });
    try { await res.body?.cancel(); } catch {}
    return res.status;
  } catch { return 0; }
  finally { clearTimeout(timer); }
}
async function pool(items, n, fn) {
  const ret = new Array(items.length);
  let i = 0, done = 0;
  async function worker() {
    while (i < items.length) {
      const idx = i++;
      ret[idx] = await fn(items[idx]);
      if (++done % 150 === 0) process.stderr.write(`  probed ${done}/${items.length}\n`);
    }
  }
  await Promise.all(Array.from({ length: n }, worker));
  return ret;
}

const allEntries = sections.flatMap((s) => s.entries);
process.stderr.write(`Assembled ${allEntries.length} unique candidates across ${sections.length} sections.\n`);

if (!DRY) {
  let vcache = {};
  if (!REVALIDATE) { try { vcache = JSON.parse(readFileSync(VCACHE, 'utf8')); } catch {} }
  const toProbe = allEntries.filter((e) => !(e.url in vcache));
  process.stderr.write(`Validating ${allEntries.length} streams — ${toProbe.length} to probe, ${allEntries.length - toProbe.length} cached.\n`);
  const statuses = await pool(toProbe, CONCURRENCY, (e) => check(e.url));
  toProbe.forEach((e, k) => { vcache[e.url] = statuses[k]; });
  mkdirSync(TMP, { recursive: true });
  writeFileSync(VCACHE, JSON.stringify(vcache));
  for (const e of allEntries) {
    const st = vcache[e.url] ?? 0;
    e.status = st;
    e.alive = (st >= 200 && st < 400) || FORCE_KEEP.some((r) => r.test(e.name));
  }
}

// ---------- disambiguation ----------
// When a channel name collides within a section, append a source tag (country,
// then provider, then quality) — but only the dimensions that actually differ, so
// single-source dupes don't get noise like "(AU)". Unique names are left untouched.
function disambiguate(entries) {
  const baseKey = (n) => n.toLowerCase()
    .replace(/\(\d+p\)|\[[^\]]*\]|\((?:hevc|adaptive)\)/gi, '').replace(/\s+/g, ' ').trim();
  const cc = (e) => (e.sourceFile.split(/[._]/)[0] || '').toUpperCase();
  const prov = (e) => {
    const p = e.sourceFile.replace(/\.m3u$/, '').split('_').slice(1).join(' ');
    return p ? p.replace(/\b\w/g, (c) => c.toUpperCase()) : '';
  };
  const nameWith = (e, dims) => {
    const tag = dims.map((d) => d(e)).filter(Boolean).join(' ');
    return tag ? `${e.name} (${tag})` : e.name;
  };
  const groups = new Map();
  for (const e of entries) {
    const k = baseKey(e.name);
    if (!groups.has(k)) groups.set(k, []);
    groups.get(k).push(e);
  }
  for (const g of groups.values()) {
    if (g.length < 2) continue;
    // add the fewest source dimensions (country, then provider) needed to make
    // names unique; quality already lives in the name so it's never re-appended.
    const distinct = (dims) => new Set(g.map((e) => nameWith(e, dims))).size;
    let active = [];
    let best = distinct(active);
    if (best < g.length) {
      for (const d of [cc, prov]) {
        const dn = distinct([...active, d]);
        if (dn > best) { active = [...active, d]; best = dn; if (best === g.length) break; }
      }
    }
    const counts = {};
    for (const e of g) {
      let n = nameWith(e, active);
      counts[n] = (counts[n] || 0) + 1;
      if (counts[n] > 1) n = `${n} (${counts[n]})`;
      if (n !== e.name) e.displayName = n;
    }
  }
}

// ---------- post-process + write ----------
const qrank = (e) => e.quality || 0;
function finalize(s) {
  let entries = DRY ? s.entries : s.entries.filter((e) => e.alive);
  if (s.dedupeByName) {
    const byName = new Map();
    for (const e of entries) {
      const key = e.name.toLowerCase().replace(/\(\d+p\)|\[[^\]]*\]/g, '').replace(/\s+/g, ' ').trim();
      const cur = byName.get(key);
      if (!cur || qrank(e) > qrank(cur)) byName.set(key, e);
    }
    entries = [...byName.values()];
  }
  entries.sort((a, b) => a.name.localeCompare(b.name));
  return entries;
}
function extinfLine(e, group) {
  const comma = e.extinf.indexOf(',');
  const attrs = e.extinf.slice(0, comma).replace(/\s*group-title="[^"]*"/, '');
  return `${attrs} group-title="${group}",${e.displayName ?? e.name}`;
}

const lines = ['#EXTM3U'];
const reportLines = [];
lines.push(
  '# ============================================================',
  '# Curated IPTV playlist — source: iptv-org/iptv',
  '# Built by: build-iptv.mjs  (`bun refresh.ts` = playlist + EPG, `bun deploy.ts` = publish)',
  '# Rebuild:  bun build-iptv.mjs            (cached, fast)',
  '# Update:   git -C /tmp/iptv-org-build/iptv-org-repo pull && bun build-iptv.mjs --revalidate',
  `# Validation: ${DRY ? 'SKIPPED (dry run)' : 'HTTP reachability probed from this machine; dead/geo-blocked dropped'}`,
  '# Colliding names get a source tag, e.g. "Comedy Central (US)" vs "(DK)".',
  '# Per-section comments below record upstream sources so this can be regenerated.',
  '# ============================================================');

let totalKept = 0;
for (const s of sections) {
  const finals = finalize(s);
  disambiguate(finals);
  totalKept += finals.length;
  reportLines.push(`${s.title.padEnd(40)} kept ${String(finals.length).padStart(4)} / ${String(s.entries.length).padStart(4)} candidates`);
  lines.push('', '# ------------------------------------------------------------',
    `# SECTION: ${s.title}   (group-title="${s.group}")`,
    ...s.sources.map((src) => src.startsWith('(') ? `# Upstream: ${src}` : `# Upstream: ${RAW}/${src}`),
    `# Filter: ${s.filter}${s.dedupeByName ? ' + deduped by channel name (best quality)' : ''}`,
    `# ${DRY ? 'Candidates' : 'Alive'}: ${finals.length}${DRY ? '' : ` / ${s.entries.length}`}`,
    '# ------------------------------------------------------------');
  for (const e of finals) {
    if (/^(News|Comedy|Espanol|Latin|Europe)/.test(s.title)) lines.push(`# src: ${e.sourceFile}`);
    lines.push(extinfLine(e, s.group), e.url);
  }
}

mkdirSync(path.dirname(OUT), { recursive: true });
writeFileSync(OUT, lines.join('\n') + '\n');
const report = ['IPTV build report (' + (DRY ? 'DRY — not validated' : 'validated') + ')',
  '='.repeat(62), ...reportLines, '-'.repeat(62),
  `TOTAL kept: ${totalKept}`, `Output: ${OUT}`].join('\n');
writeFileSync(REPORT, report + '\n');
process.stderr.write('\n' + report + '\n');