mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-11 02:46:16 -08:00
Merge pull request #864 from sapireli/epg-hardening-invalid-rows
Harden EPG channel lookup and invalid rows
This commit is contained in:
13 files changed
+793
-344
No files matched your search
@@ -2,6 +2,7 @@ import type EpgEventsType from './epg.events';
|
||||
|
||||
const mockWorkerInstances: any[] = [];
|
||||
const resolveWorkerRuntimeBootstrap = jest.fn();
|
||||
const getDatabase = jest.fn();
|
||||
|
||||
jest.mock('electron', () => ({
|
||||
app: {
|
||||
@@ -39,7 +40,7 @@ jest.mock('../workers/worker-runtime-paths', () => ({
|
||||
}));
|
||||
|
||||
jest.mock('../database/connection', () => ({
|
||||
getDatabase: jest.fn(),
|
||||
getDatabase: (...args: unknown[]) => getDatabase(...args),
|
||||
}));
|
||||
|
||||
describe('EpgEvents', () => {
|
||||
@@ -67,6 +68,7 @@ describe('EpgEvents', () => {
|
||||
afterEach(() => {
|
||||
consoleLogSpy.mockRestore();
|
||||
consoleErrorSpy.mockRestore();
|
||||
getDatabase.mockReset();
|
||||
});
|
||||
|
||||
async function flushPromises(): Promise<void> {
|
||||
@@ -128,4 +130,107 @@ describe('EpgEvents', () => {
|
||||
message: expect.stringContaining('epg-parser.worker.js'),
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to case-insensitive channel id lookup for EPG programs', async () => {
|
||||
const select = jest.fn();
|
||||
const programLimitExact = jest.fn().mockResolvedValue([]);
|
||||
const channelLimit = jest
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: 'BBC.ONE.UK', displayName: 'BBC One' }]);
|
||||
const programLimitResolved = jest.fn().mockResolvedValue([
|
||||
{
|
||||
id: 1,
|
||||
channelId: 'BBC.ONE.UK',
|
||||
start: '2026-04-14T10:00:00Z',
|
||||
stop: '2026-04-14T11:00:00Z',
|
||||
title: 'News',
|
||||
description: null,
|
||||
category: null,
|
||||
iconUrl: null,
|
||||
rating: null,
|
||||
episodeNum: null,
|
||||
},
|
||||
]);
|
||||
|
||||
const from = jest
|
||||
.fn()
|
||||
.mockReturnValueOnce({
|
||||
where: jest.fn().mockReturnValue({
|
||||
orderBy: jest.fn().mockReturnValue({
|
||||
limit: programLimitExact,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
.mockReturnValueOnce({
|
||||
where: jest.fn().mockReturnValue({
|
||||
limit: channelLimit,
|
||||
}),
|
||||
})
|
||||
.mockReturnValueOnce({
|
||||
where: jest.fn().mockReturnValue({
|
||||
orderBy: jest.fn().mockReturnValue({
|
||||
limit: programLimitResolved,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
select.mockImplementation(() => ({ from }));
|
||||
|
||||
getDatabase.mockResolvedValue({ select });
|
||||
|
||||
const programs = await (EpgEvents as unknown as Record<string, any>)[
|
||||
'handleGetChannelPrograms'
|
||||
]('bbc.one.uk');
|
||||
|
||||
expect(programs).toHaveLength(1);
|
||||
expect(programs[0].channel).toBe('BBC.ONE.UK');
|
||||
});
|
||||
|
||||
it('drops malformed EPG rows with invalid stop dates', async () => {
|
||||
const select = jest.fn();
|
||||
const from = jest.fn();
|
||||
const where = jest.fn();
|
||||
const orderBy = jest.fn();
|
||||
const limit = jest.fn();
|
||||
|
||||
select.mockImplementation(() => ({ from }));
|
||||
from.mockReturnValue({ where });
|
||||
where.mockReturnValue({ orderBy });
|
||||
orderBy.mockReturnValue({ limit });
|
||||
limit.mockResolvedValue([
|
||||
{
|
||||
id: 1,
|
||||
channelId: 'id2e2cd03c90ad',
|
||||
start: '2026-04-14T20:00:00+00:00',
|
||||
stop: '2026-04-14T21:00:00+00:00',
|
||||
title: 'valid',
|
||||
description: null,
|
||||
category: null,
|
||||
iconUrl: null,
|
||||
rating: null,
|
||||
episodeNum: null,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
channelId: 'id2e2cd03c90ad',
|
||||
start: '2026-04-14T21:00:00+00:00',
|
||||
stop: '',
|
||||
title: 'invalid',
|
||||
description: null,
|
||||
category: null,
|
||||
iconUrl: null,
|
||||
rating: null,
|
||||
episodeNum: null,
|
||||
},
|
||||
]);
|
||||
|
||||
getDatabase.mockResolvedValue({ select });
|
||||
|
||||
const programs = await (EpgEvents as unknown as Record<string, any>)[
|
||||
'handleGetChannelPrograms'
|
||||
]('id2e2cd03c90ad');
|
||||
|
||||
expect(programs).toHaveLength(1);
|
||||
expect(programs[0].title).toBe('valid');
|
||||
});
|
||||
});
|
||||
@@ -410,6 +410,15 @@ export default class EpgEvents {
|
||||
};
|
||||
}
|
||||
|
||||
private static isValidEpgProgram(program: EpgProgram): boolean {
|
||||
return Boolean(
|
||||
program.start &&
|
||||
program.stop &&
|
||||
!Number.isNaN(new Date(program.start).getTime()) &&
|
||||
!Number.isNaN(new Date(program.stop).getTime())
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get programs for a specific channel from database
|
||||
*/
|
||||
@@ -433,12 +442,39 @@ export default class EpgEvents {
|
||||
.limit(500);
|
||||
|
||||
if (results.length > 0) {
|
||||
return results.map(this.transformDbRowToEpgProgram);
|
||||
return results
|
||||
.map(this.transformDbRowToEpgProgram)
|
||||
.filter(this.isValidEpgProgram);
|
||||
}
|
||||
|
||||
// Some playlists provide the right tvg-id with different casing than
|
||||
// the XMLTV feed. Resolve the canonical channel row before giving up.
|
||||
let channel = await db
|
||||
.select()
|
||||
.from(schema.epgChannels)
|
||||
.where(
|
||||
sql`${schema.epgChannels.id} = ${trimmedChannelId} COLLATE NOCASE`
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (channel.length > 0) {
|
||||
results = await db
|
||||
.select()
|
||||
.from(schema.epgPrograms)
|
||||
.where(eq(schema.epgPrograms.channelId, channel[0].id))
|
||||
.orderBy(schema.epgPrograms.start)
|
||||
.limit(500);
|
||||
|
||||
if (results.length > 0) {
|
||||
return results
|
||||
.map(this.transformDbRowToEpgProgram)
|
||||
.filter(this.isValidEpgProgram);
|
||||
}
|
||||
}
|
||||
|
||||
// Try exact display name match before giving up. Using wildcard LIKE
|
||||
// here can scan the whole table on the Electron main process.
|
||||
let channel = await db
|
||||
channel = await db
|
||||
.select()
|
||||
.from(schema.epgChannels)
|
||||
.where(eq(schema.epgChannels.displayName, trimmedChannelId))
|
||||
@@ -462,7 +498,9 @@ export default class EpgEvents {
|
||||
.orderBy(schema.epgPrograms.start)
|
||||
.limit(500);
|
||||
|
||||
return results.map(this.transformDbRowToEpgProgram);
|
||||
return results
|
||||
.map(this.transformDbRowToEpgProgram)
|
||||
.filter(this.isValidEpgProgram);
|
||||
}
|
||||
|
||||
return [];
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
import type BetterSqlite3 from 'better-sqlite3';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import { getIptvnatorDatabasePath } from 'database-path-utils';
|
||||
import { SaxesParser, SaxesTagPlain } from 'saxes';
|
||||
import { Readable } from 'stream';
|
||||
import { parentPort, workerData } from 'worker_threads';
|
||||
import { createGunzip } from 'zlib';
|
||||
import {
|
||||
ParsedChannel,
|
||||
ParsedProgram,
|
||||
StreamingEpgParser,
|
||||
} from './epg-streaming-parser';
|
||||
import { shouldGunzipEpgResponse } from './epg-response-utils';
|
||||
import {
|
||||
getNativeModuleSearchPaths,
|
||||
getWorkerDataNativeModuleSearchPaths,
|
||||
@@ -38,51 +43,6 @@ function loadBetterSqlite3(): typeof BetterSqlite3 {
|
||||
|
||||
Database = loadBetterSqlite3();
|
||||
|
||||
/**
|
||||
* Internal parsing types with arrays for XML parsing
|
||||
* These are different from the flat EpgProgram interface used by the frontend
|
||||
*/
|
||||
interface ParsedTextValue {
|
||||
lang: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface ParsedIcon {
|
||||
src: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
|
||||
interface ParsedRating {
|
||||
system: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface ParsedEpisodeNum {
|
||||
system: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface ParsedChannel {
|
||||
id: string;
|
||||
displayName: ParsedTextValue[];
|
||||
icon: ParsedIcon[];
|
||||
url: string[];
|
||||
}
|
||||
|
||||
interface ParsedProgram {
|
||||
start: string;
|
||||
stop: string;
|
||||
channel: string;
|
||||
title: ParsedTextValue[];
|
||||
desc: ParsedTextValue[];
|
||||
category: ParsedTextValue[];
|
||||
date: string;
|
||||
episodeNum: ParsedEpisodeNum[];
|
||||
icon: ParsedIcon[];
|
||||
rating: ParsedRating[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming EPG Parser Worker
|
||||
* Uses SAX parsing to process XML incrementally without loading entire file into memory.
|
||||
@@ -243,285 +203,12 @@ class EpgDatabase {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse XMLTV datetime format to ISO string
|
||||
* Format: YYYYMMDDHHmmss +HHMM or YYYYMMDDHHmmss
|
||||
*/
|
||||
function parseXmltvDate(dateStr: string): string {
|
||||
if (!dateStr) return '';
|
||||
|
||||
const match = dateStr.match(
|
||||
/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})\s*([+-]\d{4})?$/
|
||||
);
|
||||
|
||||
if (!match) return dateStr;
|
||||
|
||||
const [, year, month, day, hour, minute, second, tz] = match;
|
||||
|
||||
let isoString = `${year}-${month}-${day}T${hour}:${minute}:${second}`;
|
||||
|
||||
if (tz) {
|
||||
isoString += `${tz.slice(0, 3)}:${tz.slice(3)}`;
|
||||
} else {
|
||||
isoString += 'Z';
|
||||
}
|
||||
|
||||
return isoString;
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming EPG parser using SAX
|
||||
*/
|
||||
class StreamingEpgParser {
|
||||
private parser: SaxesParser;
|
||||
private channels: ParsedChannel[] = [];
|
||||
private programs: ParsedProgram[] = [];
|
||||
private totalChannels = 0;
|
||||
private totalPrograms = 0;
|
||||
|
||||
// Current element being parsed
|
||||
private currentChannel: Partial<ParsedChannel> | null = null;
|
||||
private currentProgram: Partial<ParsedProgram> | null = null;
|
||||
private currentTextContent = '';
|
||||
private currentLang = '';
|
||||
|
||||
// For nested elements
|
||||
private elementStack: string[] = [];
|
||||
|
||||
constructor(
|
||||
private onChannelsBatch: (channels: ParsedChannel[]) => void,
|
||||
private onProgramsBatch: (programs: ParsedProgram[]) => void,
|
||||
private onProgress: (channels: number, programs: number) => void
|
||||
) {
|
||||
this.parser = new SaxesParser();
|
||||
this.setupParser();
|
||||
}
|
||||
|
||||
private setupParser(): void {
|
||||
this.parser.on('opentag', (tag: SaxesTagPlain) => {
|
||||
this.elementStack.push(tag.name);
|
||||
this.currentTextContent = '';
|
||||
|
||||
switch (tag.name) {
|
||||
case 'channel':
|
||||
this.currentChannel = {
|
||||
id: (tag.attributes['id'] as string) || '',
|
||||
displayName: [],
|
||||
icon: [],
|
||||
url: [],
|
||||
};
|
||||
break;
|
||||
|
||||
case 'programme':
|
||||
this.currentProgram = {
|
||||
start: parseXmltvDate(
|
||||
(tag.attributes['start'] as string) || ''
|
||||
),
|
||||
stop: parseXmltvDate(
|
||||
(tag.attributes['stop'] as string) || ''
|
||||
),
|
||||
channel: (tag.attributes['channel'] as string) || '',
|
||||
title: [],
|
||||
desc: [],
|
||||
category: [],
|
||||
date: '',
|
||||
episodeNum: [],
|
||||
icon: [],
|
||||
rating: [],
|
||||
};
|
||||
break;
|
||||
|
||||
case 'icon':
|
||||
if (this.currentChannel) {
|
||||
this.currentChannel.icon!.push({
|
||||
src: (tag.attributes['src'] as string) || '',
|
||||
width: tag.attributes['width']
|
||||
? parseInt(tag.attributes['width'] as string)
|
||||
: undefined,
|
||||
height: tag.attributes['height']
|
||||
? parseInt(tag.attributes['height'] as string)
|
||||
: undefined,
|
||||
});
|
||||
} else if (this.currentProgram) {
|
||||
this.currentProgram.icon!.push({
|
||||
src: (tag.attributes['src'] as string) || '',
|
||||
width: tag.attributes['width']
|
||||
? parseInt(tag.attributes['width'] as string)
|
||||
: undefined,
|
||||
height: tag.attributes['height']
|
||||
? parseInt(tag.attributes['height'] as string)
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'display-name':
|
||||
case 'title':
|
||||
case 'desc':
|
||||
case 'category':
|
||||
this.currentLang = (tag.attributes['lang'] as string) || '';
|
||||
break;
|
||||
|
||||
case 'rating':
|
||||
if (this.currentProgram) {
|
||||
const system =
|
||||
(tag.attributes['system'] as string) || '';
|
||||
this.currentProgram.rating!.push({ system, value: '' });
|
||||
}
|
||||
break;
|
||||
|
||||
case 'episode-num':
|
||||
if (this.currentProgram) {
|
||||
const system =
|
||||
(tag.attributes['system'] as string) || '';
|
||||
this.currentProgram.episodeNum!.push({
|
||||
system,
|
||||
value: '',
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
this.parser.on('text', (text: string) => {
|
||||
this.currentTextContent += text;
|
||||
});
|
||||
|
||||
this.parser.on('closetag', (tag: SaxesTagPlain) => {
|
||||
const text = this.currentTextContent.trim();
|
||||
|
||||
if (this.currentChannel) {
|
||||
switch (tag.name) {
|
||||
case 'display-name':
|
||||
this.currentChannel.displayName!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'url':
|
||||
if (text) this.currentChannel.url!.push(text);
|
||||
break;
|
||||
case 'channel':
|
||||
this.channels.push(
|
||||
this.currentChannel as ParsedChannel
|
||||
);
|
||||
this.totalChannels++;
|
||||
this.currentChannel = null;
|
||||
|
||||
if (this.channels.length >= CHANNEL_BATCH_SIZE) {
|
||||
this.flushChannels();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.currentProgram) {
|
||||
switch (tag.name) {
|
||||
case 'title':
|
||||
this.currentProgram.title!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'desc':
|
||||
this.currentProgram.desc!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'category':
|
||||
this.currentProgram.category!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'date':
|
||||
this.currentProgram.date = text;
|
||||
break;
|
||||
case 'value':
|
||||
if (
|
||||
this.elementStack.includes('rating') &&
|
||||
this.currentProgram.rating!.length > 0
|
||||
) {
|
||||
this.currentProgram.rating![
|
||||
this.currentProgram.rating!.length - 1
|
||||
].value = text;
|
||||
}
|
||||
break;
|
||||
case 'episode-num':
|
||||
if (this.currentProgram.episodeNum!.length > 0) {
|
||||
this.currentProgram.episodeNum![
|
||||
this.currentProgram.episodeNum!.length - 1
|
||||
].value = text;
|
||||
}
|
||||
break;
|
||||
case 'programme':
|
||||
this.programs.push(
|
||||
this.currentProgram as ParsedProgram
|
||||
);
|
||||
this.totalPrograms++;
|
||||
|
||||
if (this.programs.length >= PROGRAM_BATCH_SIZE) {
|
||||
this.flushChannels();
|
||||
this.flushPrograms();
|
||||
}
|
||||
this.currentProgram = null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
this.elementStack.pop();
|
||||
this.currentTextContent = '';
|
||||
});
|
||||
|
||||
this.parser.on('error', (err: Error) => {
|
||||
console.error(loggerLabel, 'Parser error:', err.message);
|
||||
});
|
||||
}
|
||||
|
||||
private flushChannels(): void {
|
||||
if (this.channels.length > 0) {
|
||||
this.onChannelsBatch([...this.channels]);
|
||||
this.channels = [];
|
||||
this.onProgress(this.totalChannels, this.totalPrograms);
|
||||
}
|
||||
}
|
||||
|
||||
private flushPrograms(): void {
|
||||
if (this.programs.length > 0) {
|
||||
this.onProgramsBatch([...this.programs]);
|
||||
this.programs = [];
|
||||
this.onProgress(this.totalChannels, this.totalPrograms);
|
||||
}
|
||||
}
|
||||
|
||||
write(chunk: string): void {
|
||||
this.parser.write(chunk);
|
||||
}
|
||||
|
||||
finish(): { totalChannels: number; totalPrograms: number } {
|
||||
this.parser.close();
|
||||
this.flushChannels();
|
||||
this.flushPrograms();
|
||||
|
||||
return {
|
||||
totalChannels: this.totalChannels,
|
||||
totalPrograms: this.totalPrograms,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches and parses EPG data from URL using streaming
|
||||
* Inserts directly into SQLite to avoid blocking main thread
|
||||
*/
|
||||
async function fetchAndParseEpgStreaming(url: string): Promise<void> {
|
||||
const isGzipped = url.endsWith('.gz');
|
||||
|
||||
console.log(
|
||||
loggerLabel,
|
||||
`Fetching EPG from ${url} (gzipped: ${isGzipped})`
|
||||
);
|
||||
console.log(loggerLabel, `Fetching EPG from ${url}`);
|
||||
|
||||
// Create database connection in worker
|
||||
const epgDb = new EpgDatabase();
|
||||
@@ -532,6 +219,19 @@ async function fetchAndParseEpgStreaming(url: string): Promise<void> {
|
||||
epgDb.clearSourceData(url);
|
||||
|
||||
const response = await fetch(url.trim());
|
||||
const isGzipped = shouldGunzipEpgResponse(url, response);
|
||||
|
||||
if (response.url && response.url !== url) {
|
||||
console.log(
|
||||
loggerLabel,
|
||||
`Resolved EPG redirect: ${url} -> ${response.url}`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
loggerLabel,
|
||||
`EPG response detected as gzipped: ${isGzipped}`
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
@@ -557,7 +257,9 @@ async function fetchAndParseEpgStreaming(url: string): Promise<void> {
|
||||
stats: { totalChannels, totalPrograms },
|
||||
};
|
||||
parentPort?.postMessage(response);
|
||||
}
|
||||
},
|
||||
CHANNEL_BATCH_SIZE,
|
||||
PROGRAM_BATCH_SIZE
|
||||
);
|
||||
|
||||
// Convert web stream to Node.js stream
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { shouldGunzipEpgResponse } from './epg-response-utils';
|
||||
|
||||
describe('shouldGunzipEpgResponse', () => {
|
||||
it('returns true for original .gz URLs', () => {
|
||||
expect(
|
||||
shouldGunzipEpgResponse('https://example.com/guide.xml.gz', {
|
||||
headers: new Headers(),
|
||||
url: 'https://example.com/guide.xml.gz',
|
||||
})
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when a redirect lands on a .gz URL', () => {
|
||||
expect(
|
||||
shouldGunzipEpgResponse('http://epg.vcboy.com', {
|
||||
headers: new Headers(),
|
||||
url: 'http://iptv-worker.sapireli.workers.dev/epg.xml.gz',
|
||||
})
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for gzip content-encoding without a gzip payload', () => {
|
||||
expect(
|
||||
shouldGunzipEpgResponse('https://example.com/guide', {
|
||||
headers: new Headers([['content-encoding', 'gzip']]),
|
||||
url: 'https://example.com/guide',
|
||||
})
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('returns true for gzip mime types', () => {
|
||||
expect(
|
||||
shouldGunzipEpgResponse('https://example.com/guide', {
|
||||
headers: new Headers([['content-type', 'application/gzip']]),
|
||||
url: 'https://example.com/guide',
|
||||
})
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when content-disposition advertises a .gz filename', () => {
|
||||
expect(
|
||||
shouldGunzipEpgResponse('https://example.com/guide', {
|
||||
headers: new Headers([
|
||||
['content-disposition', 'attachment; filename="guide.xml.gz"'],
|
||||
]),
|
||||
url: 'https://example.com/guide',
|
||||
})
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for plain XML responses', () => {
|
||||
expect(
|
||||
shouldGunzipEpgResponse('https://example.com/guide.xml', {
|
||||
headers: new Headers([['content-type', 'application/xml']]),
|
||||
url: 'https://example.com/guide.xml',
|
||||
})
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
function hasGzipPath(url: string | null | undefined): boolean {
|
||||
if (!url) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
return new URL(url).pathname.toLowerCase().endsWith('.gz');
|
||||
} catch {
|
||||
return url.toLowerCase().endsWith('.gz');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect whether an EPG response should be gunzipped.
|
||||
* Some providers redirect plain-looking URLs to a `.gz` payload.
|
||||
*/
|
||||
export function shouldGunzipEpgResponse(
|
||||
originalUrl: string,
|
||||
response: { headers: Headers; url?: string }
|
||||
): boolean {
|
||||
if (hasGzipPath(originalUrl) || hasGzipPath(response.url)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type');
|
||||
if (
|
||||
contentType &&
|
||||
/(application\/gzip|application\/x-gzip)/i.test(contentType)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const contentDisposition = response.headers.get('content-disposition');
|
||||
if (contentDisposition?.toLowerCase().includes('.gz')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import {
|
||||
parseXmltvDate,
|
||||
StreamingEpgParser,
|
||||
type ParsedChannel,
|
||||
type ParsedProgram,
|
||||
} from './epg-streaming-parser';
|
||||
|
||||
describe('parseXmltvDate', () => {
|
||||
it('normalizes XMLTV timestamps with timezone offsets', () => {
|
||||
expect(parseXmltvDate('20260415053700 +0000')).toBe(
|
||||
'2026-04-15T05:37:00+00:00'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StreamingEpgParser', () => {
|
||||
it('flushes pending channels before the first programme batch', () => {
|
||||
const callbackOrder: string[] = [];
|
||||
const channelIdsByBatch: string[][] = [];
|
||||
const programmeChannelsByBatch: string[][] = [];
|
||||
|
||||
const parser = new StreamingEpgParser(
|
||||
(channels: ParsedChannel[]) => {
|
||||
callbackOrder.push('channels');
|
||||
channelIdsByBatch.push(channels.map((channel) => channel.id));
|
||||
},
|
||||
(programs: ParsedProgram[]) => {
|
||||
callbackOrder.push('programs');
|
||||
programmeChannelsByBatch.push(
|
||||
programs.map((program) => program.channel)
|
||||
);
|
||||
},
|
||||
() => undefined
|
||||
);
|
||||
|
||||
const xml = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<tv>',
|
||||
...Array.from({ length: 101 }, (_, index) => {
|
||||
const id = `channel-${index + 1}`;
|
||||
return `<channel id="${id}"><display-name>${id}</display-name></channel>`;
|
||||
}),
|
||||
'<programme start="20260415053700 +0000" stop="20260415062100 +0000" channel="channel-101"><title>Late channel</title></programme>',
|
||||
'</tv>',
|
||||
].join('');
|
||||
|
||||
parser.write(xml);
|
||||
parser.finish();
|
||||
|
||||
expect(callbackOrder).toEqual(['channels', 'channels', 'programs']);
|
||||
expect(channelIdsByBatch).toEqual([
|
||||
Array.from({ length: 100 }, (_, index) => `channel-${index + 1}`),
|
||||
['channel-101'],
|
||||
]);
|
||||
expect(programmeChannelsByBatch).toEqual([['channel-101']]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,309 @@
|
||||
import { SaxesParser, SaxesTagPlain } from 'saxes';
|
||||
|
||||
export interface ParsedTextValue {
|
||||
lang: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface ParsedIcon {
|
||||
src: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
|
||||
export interface ParsedRating {
|
||||
system: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface ParsedEpisodeNum {
|
||||
system: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface ParsedChannel {
|
||||
id: string;
|
||||
displayName: ParsedTextValue[];
|
||||
icon: ParsedIcon[];
|
||||
url: string[];
|
||||
}
|
||||
|
||||
export interface ParsedProgram {
|
||||
start: string;
|
||||
stop: string;
|
||||
channel: string;
|
||||
title: ParsedTextValue[];
|
||||
desc: ParsedTextValue[];
|
||||
category: ParsedTextValue[];
|
||||
date: string;
|
||||
episodeNum: ParsedEpisodeNum[];
|
||||
icon: ParsedIcon[];
|
||||
rating: ParsedRating[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse XMLTV datetime format to ISO string
|
||||
* Format: YYYYMMDDHHmmss +HHMM or YYYYMMDDHHmmss
|
||||
*/
|
||||
export function parseXmltvDate(dateStr: string): string {
|
||||
if (!dateStr) return '';
|
||||
|
||||
const match = dateStr.match(
|
||||
/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})\s*([+-]\d{4})?$/
|
||||
);
|
||||
|
||||
if (!match) return dateStr;
|
||||
|
||||
const [, year, month, day, hour, minute, second, tz] = match;
|
||||
|
||||
let isoString = `${year}-${month}-${day}T${hour}:${minute}:${second}`;
|
||||
|
||||
if (tz) {
|
||||
isoString += `${tz.slice(0, 3)}:${tz.slice(3)}`;
|
||||
} else {
|
||||
isoString += 'Z';
|
||||
}
|
||||
|
||||
return isoString;
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming EPG parser using SAX
|
||||
*/
|
||||
export class StreamingEpgParser {
|
||||
private parser: SaxesParser;
|
||||
private channels: ParsedChannel[] = [];
|
||||
private programs: ParsedProgram[] = [];
|
||||
private totalChannels = 0;
|
||||
private totalPrograms = 0;
|
||||
|
||||
// Current element being parsed
|
||||
private currentChannel: Partial<ParsedChannel> | null = null;
|
||||
private currentProgram: Partial<ParsedProgram> | null = null;
|
||||
private currentTextContent = '';
|
||||
private currentLang = '';
|
||||
|
||||
// For nested elements
|
||||
private elementStack: string[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly onChannelsBatch: (channels: ParsedChannel[]) => void,
|
||||
private readonly onProgramsBatch: (programs: ParsedProgram[]) => void,
|
||||
private readonly onProgress: (channels: number, programs: number) => void,
|
||||
private readonly channelBatchSize = 100,
|
||||
private readonly programBatchSize = 1000
|
||||
) {
|
||||
this.parser = new SaxesParser();
|
||||
this.setupParser();
|
||||
}
|
||||
|
||||
private setupParser(): void {
|
||||
this.parser.on('opentag', (tag: SaxesTagPlain) => {
|
||||
this.elementStack.push(tag.name);
|
||||
this.currentTextContent = '';
|
||||
|
||||
switch (tag.name) {
|
||||
case 'channel':
|
||||
this.currentChannel = {
|
||||
id: (tag.attributes['id'] as string) || '',
|
||||
displayName: [],
|
||||
icon: [],
|
||||
url: [],
|
||||
};
|
||||
break;
|
||||
|
||||
case 'programme':
|
||||
// XMLTV feeds typically emit all <channel> entries before
|
||||
// the first <programme>. Flush any pending channel batch
|
||||
// so downstream program consumers can resolve late-channel
|
||||
// IDs immediately instead of dropping the first rows.
|
||||
this.flushChannels();
|
||||
this.currentProgram = {
|
||||
start: parseXmltvDate(
|
||||
(tag.attributes['start'] as string) || ''
|
||||
),
|
||||
stop: parseXmltvDate(
|
||||
(tag.attributes['stop'] as string) || ''
|
||||
),
|
||||
channel: (tag.attributes['channel'] as string) || '',
|
||||
title: [],
|
||||
desc: [],
|
||||
category: [],
|
||||
date: '',
|
||||
episodeNum: [],
|
||||
icon: [],
|
||||
rating: [],
|
||||
};
|
||||
break;
|
||||
|
||||
case 'icon':
|
||||
if (this.currentChannel) {
|
||||
this.currentChannel.icon!.push({
|
||||
src: (tag.attributes['src'] as string) || '',
|
||||
width: tag.attributes['width']
|
||||
? parseInt(tag.attributes['width'] as string)
|
||||
: undefined,
|
||||
height: tag.attributes['height']
|
||||
? parseInt(tag.attributes['height'] as string)
|
||||
: undefined,
|
||||
});
|
||||
} else if (this.currentProgram) {
|
||||
this.currentProgram.icon!.push({
|
||||
src: (tag.attributes['src'] as string) || '',
|
||||
width: tag.attributes['width']
|
||||
? parseInt(tag.attributes['width'] as string)
|
||||
: undefined,
|
||||
height: tag.attributes['height']
|
||||
? parseInt(tag.attributes['height'] as string)
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'display-name':
|
||||
case 'title':
|
||||
case 'desc':
|
||||
case 'category':
|
||||
this.currentLang = (tag.attributes['lang'] as string) || '';
|
||||
break;
|
||||
|
||||
case 'rating':
|
||||
if (this.currentProgram) {
|
||||
const system =
|
||||
(tag.attributes['system'] as string) || '';
|
||||
this.currentProgram.rating!.push({ system, value: '' });
|
||||
}
|
||||
break;
|
||||
|
||||
case 'episode-num':
|
||||
if (this.currentProgram) {
|
||||
const system =
|
||||
(tag.attributes['system'] as string) || '';
|
||||
this.currentProgram.episodeNum!.push({
|
||||
system,
|
||||
value: '',
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
this.parser.on('text', (text: string) => {
|
||||
this.currentTextContent += text;
|
||||
});
|
||||
|
||||
this.parser.on('closetag', (tag: SaxesTagPlain) => {
|
||||
const text = this.currentTextContent.trim();
|
||||
|
||||
if (this.currentChannel) {
|
||||
switch (tag.name) {
|
||||
case 'display-name':
|
||||
this.currentChannel.displayName!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'url':
|
||||
if (text) this.currentChannel.url!.push(text);
|
||||
break;
|
||||
case 'channel':
|
||||
this.channels.push(this.currentChannel as ParsedChannel);
|
||||
this.totalChannels++;
|
||||
this.currentChannel = null;
|
||||
|
||||
if (this.channels.length >= this.channelBatchSize) {
|
||||
this.flushChannels();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.currentProgram) {
|
||||
switch (tag.name) {
|
||||
case 'title':
|
||||
this.currentProgram.title!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'desc':
|
||||
this.currentProgram.desc!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'category':
|
||||
this.currentProgram.category!.push({
|
||||
lang: this.currentLang,
|
||||
value: text,
|
||||
});
|
||||
break;
|
||||
case 'date':
|
||||
this.currentProgram.date = text;
|
||||
break;
|
||||
case 'value':
|
||||
if (
|
||||
this.elementStack.includes('rating') &&
|
||||
this.currentProgram.rating!.length > 0
|
||||
) {
|
||||
this.currentProgram.rating![
|
||||
this.currentProgram.rating!.length - 1
|
||||
].value = text;
|
||||
}
|
||||
break;
|
||||
case 'episode-num':
|
||||
if (this.currentProgram.episodeNum!.length > 0) {
|
||||
this.currentProgram.episodeNum![
|
||||
this.currentProgram.episodeNum!.length - 1
|
||||
].value = text;
|
||||
}
|
||||
break;
|
||||
case 'programme':
|
||||
this.programs.push(this.currentProgram as ParsedProgram);
|
||||
this.totalPrograms++;
|
||||
|
||||
if (this.programs.length >= this.programBatchSize) {
|
||||
this.flushChannels();
|
||||
this.flushPrograms();
|
||||
}
|
||||
this.currentProgram = null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
this.elementStack.pop();
|
||||
this.currentTextContent = '';
|
||||
});
|
||||
}
|
||||
|
||||
private flushChannels(): void {
|
||||
if (this.channels.length > 0) {
|
||||
this.onChannelsBatch([...this.channels]);
|
||||
this.channels = [];
|
||||
this.onProgress(this.totalChannels, this.totalPrograms);
|
||||
}
|
||||
}
|
||||
|
||||
private flushPrograms(): void {
|
||||
if (this.programs.length > 0) {
|
||||
this.onProgramsBatch([...this.programs]);
|
||||
this.programs = [];
|
||||
this.onProgress(this.totalChannels, this.totalPrograms);
|
||||
}
|
||||
}
|
||||
|
||||
write(chunk: string): void {
|
||||
this.parser.write(chunk);
|
||||
}
|
||||
|
||||
finish(): { totalChannels: number; totalPrograms: number } {
|
||||
this.parser.close();
|
||||
this.flushChannels();
|
||||
this.flushPrograms();
|
||||
|
||||
return {
|
||||
totalChannels: this.totalChannels,
|
||||
totalPrograms: this.totalPrograms,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { EpgProgram } from 'shared-interfaces';
|
||||
import { normalizeEpgPrograms } from './epg-program-normalization.util';
|
||||
|
||||
function buildProgram(overrides: Partial<EpgProgram> = {}): EpgProgram {
|
||||
return {
|
||||
start: '2026-04-15T20:00:00Z',
|
||||
stop: '2026-04-15T21:00:00Z',
|
||||
channel: 'channel-1',
|
||||
title: 'Sample',
|
||||
desc: null,
|
||||
category: null,
|
||||
iconUrl: null,
|
||||
rating: null,
|
||||
episodeNum: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('normalizeEpgPrograms', () => {
|
||||
it('keeps valid rows and normalizes dates', () => {
|
||||
expect(normalizeEpgPrograms([buildProgram()])).toEqual([
|
||||
expect.objectContaining({
|
||||
start: '2026-04-15T20:00:00.000Z',
|
||||
stop: '2026-04-15T21:00:00.000Z',
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops rows with invalid stop dates', () => {
|
||||
expect(
|
||||
normalizeEpgPrograms([
|
||||
buildProgram({ title: 'ok' }),
|
||||
buildProgram({ title: 'bad', stop: '' }),
|
||||
])
|
||||
).toEqual([expect.objectContaining({ title: 'ok' })]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { EpgProgram } from 'shared-interfaces';
|
||||
|
||||
function toIsoDate(value: string | undefined): string | null {
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return date.toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop malformed EPG rows instead of failing the whole channel request.
|
||||
*/
|
||||
export function normalizeEpgPrograms(programs: EpgProgram[]): EpgProgram[] {
|
||||
const normalizedPrograms: EpgProgram[] = [];
|
||||
|
||||
for (const program of programs) {
|
||||
const start = toIsoDate(program.start);
|
||||
const stop = toIsoDate(program.stop);
|
||||
|
||||
if (!start || !stop) {
|
||||
continue;
|
||||
}
|
||||
|
||||
normalizedPrograms.push({
|
||||
...program,
|
||||
start,
|
||||
stop,
|
||||
});
|
||||
}
|
||||
|
||||
return normalizedPrograms;
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { TranslateService } from '@ngx-translate/core';
|
||||
import { BehaviorSubject, forkJoin, from, Observable, of } from 'rxjs';
|
||||
import { catchError, map, tap, timeout } from 'rxjs/operators';
|
||||
import { EpgProgram } from 'shared-interfaces';
|
||||
import { normalizeEpgPrograms } from './epg-program-normalization.util';
|
||||
|
||||
interface CachedProgram {
|
||||
program: EpgProgram | null;
|
||||
@@ -69,13 +70,7 @@ export class EpgService {
|
||||
from(window.electron.getChannelPrograms(channelId))
|
||||
.pipe(
|
||||
timeout(3000),
|
||||
map((programs: EpgProgram[]) =>
|
||||
programs.map((program) => ({
|
||||
...program,
|
||||
start: new Date(program.start).toISOString(),
|
||||
stop: new Date(program.stop).toISOString(),
|
||||
}))
|
||||
),
|
||||
map((programs: EpgProgram[]) => normalizeEpgPrograms(programs)),
|
||||
catchError((err) => {
|
||||
console.error('EPG get programs error:', err);
|
||||
this.showErrorSnackbar();
|
||||
@@ -123,8 +118,9 @@ export class EpgService {
|
||||
|
||||
// Fetch from backend
|
||||
return from(window.electron.getChannelPrograms(channelId)).pipe(
|
||||
map((programs: EpgProgram[]) => normalizeEpgPrograms(programs)),
|
||||
map((programs: EpgProgram[]) => {
|
||||
if (!programs || programs.length === 0) {
|
||||
if (!programs.length) {
|
||||
this.programCache.set(channelId, {
|
||||
program: null,
|
||||
timestamp: now,
|
||||
@@ -132,16 +128,7 @@ export class EpgService {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Normalize date formats to ISO strings for consistency
|
||||
const transformedPrograms = programs.map((program) => ({
|
||||
...program,
|
||||
start: new Date(program.start).toISOString(),
|
||||
stop: new Date(program.stop).toISOString(),
|
||||
}));
|
||||
|
||||
// Find current program from transformed programs
|
||||
const currentProgram =
|
||||
this.findCurrentProgram(transformedPrograms);
|
||||
const currentProgram = this.findCurrentProgram(programs);
|
||||
|
||||
// Cache the result
|
||||
this.programCache.set(channelId, {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { Channel } from 'shared-interfaces';
|
||||
import { resolveChannelEpgLookupKey } from './channel-epg-lookup.util';
|
||||
|
||||
function buildChannel(overrides: Partial<Channel> = {}): Channel {
|
||||
return {
|
||||
id: 'channel-1',
|
||||
url: 'https://example.com/live.m3u8',
|
||||
name: 'Fallback Name',
|
||||
group: { title: 'News' },
|
||||
tvg: {
|
||||
id: 'epg-id',
|
||||
name: 'Guide Name',
|
||||
url: '',
|
||||
logo: '',
|
||||
rec: '',
|
||||
},
|
||||
http: {
|
||||
referrer: '',
|
||||
'user-agent': '',
|
||||
origin: '',
|
||||
},
|
||||
radio: 'false',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('resolveChannelEpgLookupKey', () => {
|
||||
it('prefers tvg id when present', () => {
|
||||
expect(resolveChannelEpgLookupKey(buildChannel())).toBe('epg-id');
|
||||
});
|
||||
|
||||
it('falls back to tvg name before channel name', () => {
|
||||
expect(
|
||||
resolveChannelEpgLookupKey(
|
||||
buildChannel({
|
||||
tvg: {
|
||||
id: ' ',
|
||||
name: 'Guide Name',
|
||||
url: '',
|
||||
logo: '',
|
||||
rec: '',
|
||||
},
|
||||
name: 'Channel Name',
|
||||
})
|
||||
)
|
||||
).toBe('Guide Name');
|
||||
});
|
||||
|
||||
it('falls back to channel name when tvg fields are empty', () => {
|
||||
expect(
|
||||
resolveChannelEpgLookupKey(
|
||||
buildChannel({
|
||||
tvg: {
|
||||
id: ' ',
|
||||
name: ' ',
|
||||
url: '',
|
||||
logo: '',
|
||||
rec: '',
|
||||
},
|
||||
name: 'Channel Name',
|
||||
})
|
||||
)
|
||||
).toBe('Channel Name');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Channel } from 'shared-interfaces';
|
||||
|
||||
export function resolveChannelEpgLookupKey(
|
||||
channel: Channel | null | undefined
|
||||
): string {
|
||||
return (
|
||||
channel?.tvg?.id?.trim() ||
|
||||
channel?.tvg?.name?.trim() ||
|
||||
channel?.name?.trim() ||
|
||||
''
|
||||
);
|
||||
}
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
selectChannels,
|
||||
selectFavorites,
|
||||
} from './selectors';
|
||||
import { resolveChannelEpgLookupKey } from './channel-epg-lookup.util';
|
||||
import { buildExternalPlayerPayload } from './external-player-payload.util';
|
||||
|
||||
@Injectable({ providedIn: 'any' })
|
||||
@@ -152,7 +153,7 @@ export class PlaylistEffects {
|
||||
const { channel } = action;
|
||||
|
||||
// Use modern EPG service to get channel programs
|
||||
const channelId = channel.tvg?.id || channel.name;
|
||||
const channelId = resolveChannelEpgLookupKey(channel);
|
||||
if (channelId) {
|
||||
this.epgService.getChannelPrograms(channelId);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user