From 36a7ce9f3d42b5fa825f4bcbe41ed2dd1181cc41 Mon Sep 17 00:00:00 2001 From: 4gray <4gray@users.noreply.github.com> Date: Sun, 21 Jun 2026 09:34:37 +0200 Subject: [PATCH] fix(epg): decode compressed XMLTV responses Decode HTTP Content-Encoding for EPG XMLTV streams before SAX parsing and harden gzip payload handling for mislabelled providers. Validation: - pnpm nx test electron-backend --testFile=epg-stream-decoder.spec.ts - pnpm nx test electron-backend --testFile=epg-response-utils.spec.ts - pnpm nx test electron-backend - pnpm nx lint electron-backend - pnpm nx run electron-backend:build-worker --- .../src/app/workers/epg-parser.worker.ts | 33 +++-- .../app/workers/epg-response-utils.spec.ts | 47 ++++++- .../src/app/workers/epg-response-utils.ts | 35 ++++- .../app/workers/epg-stream-decoder.spec.ts | 130 ++++++++++++++++++ .../src/app/workers/epg-stream-decoder.ts | 50 +++++++ 5 files changed, 279 insertions(+), 16 deletions(-) create mode 100644 apps/electron-backend/src/app/workers/epg-stream-decoder.spec.ts create mode 100644 apps/electron-backend/src/app/workers/epg-stream-decoder.ts diff --git a/apps/electron-backend/src/app/workers/epg-parser.worker.ts b/apps/electron-backend/src/app/workers/epg-parser.worker.ts index 31f647506..1eb71fc1a 100644 --- a/apps/electron-backend/src/app/workers/epg-parser.worker.ts +++ b/apps/electron-backend/src/app/workers/epg-parser.worker.ts @@ -6,10 +6,13 @@ import { } from '@iptvnator/shared/interfaces'; import { Readable } from 'stream'; import { parentPort, workerData } from 'worker_threads'; -import { createGunzip } from 'zlib'; import { EpgDatabase, EpgDatabaseClearOperation } from './epg-database'; +import { createDecodedEpgStream } from './epg-stream-decoder'; import { StreamingEpgParser } from './epg-streaming-parser'; -import { shouldGunzipEpgResponse } from './epg-response-utils'; +import { + getEpgResponseContentEncoding, + shouldGunzipEpgResponse, +} from './epg-response-utils'; import { isPrivateNetworkUrlAccessAllowed, UnsafeUrlError, @@ -130,6 +133,9 @@ async function fetchAndParseEpgStreaming( headers: response.headers, url: responseUrl, }); + const contentEncoding = getEpgResponseContentEncoding( + response.headers + ); if (responseUrl && responseUrl !== url) { console.log( @@ -142,6 +148,12 @@ async function fetchAndParseEpgStreaming( loggerLabel, `EPG response detected as gzipped: ${isGzipped}` ); + if (contentEncoding) { + console.log( + loggerLabel, + `EPG response content-encoding: ${contentEncoding}` + ); + } if (response.status < 200 || response.status >= 300) { throw new Error(`HTTP error! status: ${response.status}`); @@ -175,18 +187,11 @@ async function fetchAndParseEpgStreaming( ); return new Promise((resolve, reject) => { - let dataStream: Readable = response.data; - - if (isGzipped) { - const gunzip = createGunzip(); - dataStream = response.data.pipe(gunzip); - - gunzip.on('error', (err) => { - console.error(loggerLabel, 'Gunzip error:', err); - epgDb.close(); - reject(err); - }); - } + const dataStream = createDecodedEpgStream( + response.data, + response.headers, + isGzipped + ); dataStream.on('data', (chunk: Buffer) => { try { diff --git a/apps/electron-backend/src/app/workers/epg-response-utils.spec.ts b/apps/electron-backend/src/app/workers/epg-response-utils.spec.ts index 314c10192..dd4683b9f 100644 --- a/apps/electron-backend/src/app/workers/epg-response-utils.spec.ts +++ b/apps/electron-backend/src/app/workers/epg-response-utils.spec.ts @@ -1,4 +1,7 @@ -import { shouldGunzipEpgResponse } from './epg-response-utils'; +import { + getEpgResponseContentEncoding, + shouldGunzipEpgResponse, +} from './epg-response-utils'; describe('shouldGunzipEpgResponse', () => { it('returns true for original .gz URLs', () => { @@ -71,3 +74,45 @@ describe('shouldGunzipEpgResponse', () => { ).toBe(false); }); }); + +describe('getEpgResponseContentEncoding', () => { + it('detects Brotli transfer encoding from response headers', () => { + expect( + getEpgResponseContentEncoding({ + 'content-encoding': 'br', + }) + ).toBe('br'); + }); + + it('detects gzip transfer encoding from response headers', () => { + expect( + getEpgResponseContentEncoding( + new Headers([['content-encoding', 'gzip']]) + ) + ).toBe('gzip'); + }); + + it('detects deflate transfer encoding from response headers', () => { + expect( + getEpgResponseContentEncoding({ + 'Content-Encoding': 'deflate', + }) + ).toBe('deflate'); + }); + + it('ignores unsupported transfer encodings', () => { + expect( + getEpgResponseContentEncoding({ + 'content-encoding': 'zstd', + }) + ).toBe(null); + }); + + it('returns the outermost supported transfer encoding first', () => { + expect( + getEpgResponseContentEncoding({ + 'content-encoding': 'gzip, br', + }) + ).toBe('br'); + }); +}); diff --git a/apps/electron-backend/src/app/workers/epg-response-utils.ts b/apps/electron-backend/src/app/workers/epg-response-utils.ts index 363dfd00a..e4ed92cce 100644 --- a/apps/electron-backend/src/app/workers/epg-response-utils.ts +++ b/apps/electron-backend/src/app/workers/epg-response-utils.ts @@ -1,9 +1,17 @@ -type HeaderReader = +export type HeaderReader = | Record | { get(name: string): unknown; }; +export type EpgResponseContentEncoding = 'br' | 'gzip' | 'deflate'; + +const SUPPORTED_CONTENT_ENCODINGS: readonly EpgResponseContentEncoding[] = [ + 'br', + 'gzip', + 'deflate', +]; + function getHeaderValue(headers: HeaderReader, name: string): string | null { const value = 'get' in headers && typeof headers.get === 'function' @@ -27,6 +35,31 @@ function hasGzipPath(url: string | null | undefined): boolean { } } +export function getEpgResponseContentEncoding( + headers: HeaderReader +): EpgResponseContentEncoding | null { + const contentEncoding = getEpgResponseContentEncodings(headers).at(0); + + return contentEncoding ?? null; +} + +export function getEpgResponseContentEncodings( + headers: HeaderReader +): EpgResponseContentEncoding[] { + return ( + getHeaderValue(headers, 'content-encoding') + ?.toLowerCase() + .split(',') + .map((encoding) => encoding.trim()) + .filter((encoding): encoding is EpgResponseContentEncoding => + SUPPORTED_CONTENT_ENCODINGS.includes( + encoding as EpgResponseContentEncoding + ) + ) + .reverse() ?? [] + ); +} + /** * Detect whether an EPG response should be gunzipped. * Some providers redirect plain-looking URLs to a `.gz` payload. diff --git a/apps/electron-backend/src/app/workers/epg-stream-decoder.spec.ts b/apps/electron-backend/src/app/workers/epg-stream-decoder.spec.ts new file mode 100644 index 000000000..73c6d2882 --- /dev/null +++ b/apps/electron-backend/src/app/workers/epg-stream-decoder.spec.ts @@ -0,0 +1,130 @@ +import { PassThrough, Readable } from 'stream'; +import { brotliCompressSync, gzipSync } from 'zlib'; +import { createDecodedEpgStream } from './epg-stream-decoder'; +import { StreamingEpgParser } from './epg-streaming-parser'; + +const xmltvFixture = + '' + + 'Test Channel'; + +async function collectDecodedText(stream: Readable): Promise { + const chunks: Buffer[] = []; + + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + + return Buffer.concat(chunks).toString('utf-8'); +} + +function parseChannelCount(xml: string): number { + const parser = new StreamingEpgParser( + () => undefined, + () => undefined, + () => undefined + ); + + parser.write(xml); + return parser.finish().totalChannels; +} + +describe('createDecodedEpgStream', () => { + it('decodes Brotli transfer-encoded XML before SAX parsing', async () => { + const decodedText = await collectDecodedText( + createDecodedEpgStream( + Readable.from([brotliCompressSync(xmltvFixture)]), + { 'content-encoding': 'br' }, + false + ) + ); + + expect(decodedText.startsWith(' { + const decodedText = await collectDecodedText( + createDecodedEpgStream( + Readable.from([gzipSync(xmltvFixture)]), + { 'content-encoding': 'gzip' }, + false + ) + ); + + expect(decodedText.startsWith(' { + const decodedText = await collectDecodedText( + createDecodedEpgStream( + Readable.from([Buffer.from(xmltvFixture)]), + {}, + false + ) + ); + + expect(decodedText.startsWith(' { + const transferEncodedPayload = brotliCompressSync( + gzipSync(xmltvFixture) + ); + + const decodedText = await collectDecodedText( + createDecodedEpgStream( + Readable.from([transferEncodedPayload]), + { 'content-encoding': 'br' }, + true + ) + ); + + expect(decodedText.startsWith(' { + const decodedText = await collectDecodedText( + createDecodedEpgStream( + Readable.from([gzipSync(xmltvFixture)]), + { 'content-encoding': 'gzip' }, + true + ) + ); + + expect(decodedText.startsWith(' { + const encodedPayload = brotliCompressSync(gzipSync(xmltvFixture)); + + const decodedText = await collectDecodedText( + createDecodedEpgStream( + Readable.from([encodedPayload]), + { 'content-encoding': 'gzip, br' }, + false + ) + ); + + expect(decodedText.startsWith(' { + const source = new PassThrough(); + const decodedStream = createDecodedEpgStream( + source, + { 'content-encoding': 'gzip' }, + false + ); + + const readPromise = collectDecodedText(decodedStream); + source.end(Buffer.from('not gzip')); + + await expect(readPromise).rejects.toThrow(); + expect(source.destroyed).toBe(true); + }); +}); diff --git a/apps/electron-backend/src/app/workers/epg-stream-decoder.ts b/apps/electron-backend/src/app/workers/epg-stream-decoder.ts new file mode 100644 index 000000000..726294e0a --- /dev/null +++ b/apps/electron-backend/src/app/workers/epg-stream-decoder.ts @@ -0,0 +1,50 @@ +import { PassThrough, Readable, Transform, pipeline } from 'stream'; +import { + createBrotliDecompress, + createGunzip, + createInflate, +} from 'zlib'; +import { + EpgResponseContentEncoding, + getEpgResponseContentEncodings, + HeaderReader, +} from './epg-response-utils'; + +function createContentEncodingDecoder( + contentEncoding: EpgResponseContentEncoding +): Transform { + switch (contentEncoding) { + case 'br': + return createBrotliDecompress(); + case 'gzip': + return createGunzip(); + case 'deflate': + return createInflate(); + } +} + +export function createDecodedEpgStream( + source: Readable, + headers: HeaderReader, + shouldGunzipPayload: boolean +): Readable { + const contentEncodings = getEpgResponseContentEncodings(headers); + const transforms = contentEncodings.map(createContentEncodingDecoder); + + if (shouldGunzipPayload && !contentEncodings.includes('gzip')) { + transforms.push(createGunzip()); + } + + if (transforms.length === 0) { + return source; + } + + const output = new PassThrough(); + pipeline([source, ...transforms, output], (error) => { + if (error) { + output.destroy(error); + } + }); + + return output; +}