iptv/grab-epg.ts

#!/usr/bin/env bun
// Batched EPG grab — keeps memory bounded. The iptv-org grabber holds every
// programme for every channel in RAM until the end (1000+ channels = many GB),
// so we grab ~BATCH channels at a time, EACH IN ITS OWN bun process (memory is
// reclaimed when the process exits), then merge the per-batch guides into one
// XMLTV file. Tunable via env: BATCH, DAYS, MAXCONN, LIMIT, CHANNELS, OUT.
import { $ } from 'bun';
import { readFileSync, writeFileSync, mkdirSync, rmSync, existsSync } from 'node:fs';

const HERE = import.meta.dir;
const TMP = '/tmp/iptv-org-build';
const EPG = `${TMP}/epg`;
const CHANNELS = process.env.CHANNELS || `${TMP}/custom.channels.xml`;
const OUT = process.env.OUT || `${HERE}/guide.xml`;
const BATCH = Number(process.env.BATCH || 25);  // ~3 GB peak per batch (measured)
const DAYS = Number(process.env.DAYS || 2);
const MAXCONN = Number(process.env.MAXCONN || 10);
const LIMIT = Number(process.env.LIMIT || 0); // 0 = all batches (for quick tests)

const channels = [...readFileSync(CHANNELS, 'utf8').matchAll(/<channel\b[\s\S]*?<\/channel>/g)].map((m) => m[0]);
const dir = `${TMP}/batches`;
rmSync(dir, { recursive: true, force: true });
mkdirSync(dir, { recursive: true });

let groups = [];
for (let i = 0; i < channels.length; i += BATCH) groups.push(channels.slice(i, i + BATCH));
if (LIMIT) groups = groups.slice(0, LIMIT);
console.log(`${channels.length} channels -> ${groups.length} batch(es) of ${BATCH}  (days=${DAYS}, maxConn=${MAXCONN})`);

const guides = [];
for (let i = 0; i < groups.length; i++) {
  const cf = `${dir}/b${i}.channels.xml`;
  const gf = `${dir}/g${i}.xml`;
  writeFileSync(cf, `<?xml version="1.0" encoding="UTF-8"?>\n<channels>\n${groups[i].join('\n')}\n</channels>\n`);
  await $`bun scripts/commands/epg/grab.ts --channels ${cf} --output ${gf} --days ${DAYS} --maxConnections ${MAXCONN}`
    .cwd(EPG).quiet().nothrow();
  const n = existsSync(gf) ? (readFileSync(gf, 'utf8').match(/<programme\b/g) || []).length : -1;
  console.log(`  [${i + 1}/${groups.length}] ${n < 0 ? 'FAILED' : `${n} programmes`}`);
  if (existsSync(gf)) guides.push(gf);
}

// merge: every channel is in exactly one batch, so no dedup needed
const chans = [], progs = [];
for (const g of guides) {
  const t = readFileSync(g, 'utf8');
  for (const m of t.matchAll(/<channel\b[\s\S]*?<\/channel>/g)) chans.push(m[0]);
  for (const m of t.matchAll(/<programme\b[\s\S]*?<\/programme>/g)) progs.push(m[0]);
}
writeFileSync(OUT, `<?xml version="1.0" encoding="UTF-8"?>\n<tv>\n${chans.join('\n')}\n${progs.join('\n')}\n</tv>\n`);
console.log(`merged -> ${OUT}  (${chans.length} channels, ${progs.length} programmes)`);