fix(epg): decode compressed XMLTV responses

Decode HTTP Content-Encoding for EPG XMLTV streams before SAX parsing and harden gzip payload handling for mislabelled providers.

Validation:
- pnpm nx test electron-backend --testFile=epg-stream-decoder.spec.ts
- pnpm nx test electron-backend --testFile=epg-response-utils.spec.ts
- pnpm nx test electron-backend
- pnpm nx lint electron-backend
- pnpm nx run electron-backend:build-worker
This commit is contained in:
4gray authored and GitHub committed 2026-06-21 09:34:37 +02:00
1 parent d37e0f84b9
commit 36a7ce9f3d
5 files changed
+279 -16

No files matched your search

@@ -6,10 +6,13 @@ import {
} from '@iptvnator/shared/interfaces';
import { Readable } from 'stream';
import { parentPort, workerData } from 'worker_threads';
import { createGunzip } from 'zlib';
import { EpgDatabase, EpgDatabaseClearOperation } from './epg-database';
import { createDecodedEpgStream } from './epg-stream-decoder';
import { StreamingEpgParser } from './epg-streaming-parser';
import { shouldGunzipEpgResponse } from './epg-response-utils';
import {
getEpgResponseContentEncoding,
shouldGunzipEpgResponse,
} from './epg-response-utils';
import {
isPrivateNetworkUrlAccessAllowed,
UnsafeUrlError,
@@ -130,6 +133,9 @@ async function fetchAndParseEpgStreaming(
headers: response.headers,
url: responseUrl,
});
const contentEncoding = getEpgResponseContentEncoding(
response.headers
);
if (responseUrl && responseUrl !== url) {
console.log(
@@ -142,6 +148,12 @@ async function fetchAndParseEpgStreaming(
loggerLabel,
`EPG response detected as gzipped: ${isGzipped}`
);
if (contentEncoding) {
console.log(
loggerLabel,
`EPG response content-encoding: ${contentEncoding}`
);
}
if (response.status < 200 || response.status >= 300) {
throw new Error(`HTTP error! status: ${response.status}`);
@@ -175,18 +187,11 @@ async function fetchAndParseEpgStreaming(
);
return new Promise((resolve, reject) => {
let dataStream: Readable = response.data;
if (isGzipped) {
const gunzip = createGunzip();
dataStream = response.data.pipe(gunzip);
gunzip.on('error', (err) => {
console.error(loggerLabel, 'Gunzip error:', err);
epgDb.close();
reject(err);
});
}
const dataStream = createDecodedEpgStream(
response.data,
response.headers,
isGzipped
);
dataStream.on('data', (chunk: Buffer) => {
try {
@@ -1,4 +1,7 @@
import { shouldGunzipEpgResponse } from './epg-response-utils';
import {
getEpgResponseContentEncoding,
shouldGunzipEpgResponse,
} from './epg-response-utils';
describe('shouldGunzipEpgResponse', () => {
it('returns true for original .gz URLs', () => {
@@ -71,3 +74,45 @@ describe('shouldGunzipEpgResponse', () => {
).toBe(false);
});
});
describe('getEpgResponseContentEncoding', () => {
it('detects Brotli transfer encoding from response headers', () => {
expect(
getEpgResponseContentEncoding({
'content-encoding': 'br',
})
).toBe('br');
});
it('detects gzip transfer encoding from response headers', () => {
expect(
getEpgResponseContentEncoding(
new Headers([['content-encoding', 'gzip']])
)
).toBe('gzip');
});
it('detects deflate transfer encoding from response headers', () => {
expect(
getEpgResponseContentEncoding({
'Content-Encoding': 'deflate',
})
).toBe('deflate');
});
it('ignores unsupported transfer encodings', () => {
expect(
getEpgResponseContentEncoding({
'content-encoding': 'zstd',
})
).toBe(null);
});
it('returns the outermost supported transfer encoding first', () => {
expect(
getEpgResponseContentEncoding({
'content-encoding': 'gzip, br',
})
).toBe('br');
});
});
@@ -1,9 +1,17 @@
type HeaderReader =
export type HeaderReader =
| Record<string, unknown>
| {
get(name: string): unknown;
};
export type EpgResponseContentEncoding = 'br' | 'gzip' | 'deflate';
const SUPPORTED_CONTENT_ENCODINGS: readonly EpgResponseContentEncoding[] = [
'br',
'gzip',
'deflate',
];
function getHeaderValue(headers: HeaderReader, name: string): string | null {
const value =
'get' in headers && typeof headers.get === 'function'
@@ -27,6 +35,31 @@ function hasGzipPath(url: string | null | undefined): boolean {
}
}
export function getEpgResponseContentEncoding(
headers: HeaderReader
): EpgResponseContentEncoding | null {
const contentEncoding = getEpgResponseContentEncodings(headers).at(0);
return contentEncoding ?? null;
}
export function getEpgResponseContentEncodings(
headers: HeaderReader
): EpgResponseContentEncoding[] {
return (
getHeaderValue(headers, 'content-encoding')
?.toLowerCase()
.split(',')
.map((encoding) => encoding.trim())
.filter((encoding): encoding is EpgResponseContentEncoding =>
SUPPORTED_CONTENT_ENCODINGS.includes(
encoding as EpgResponseContentEncoding
)
)
.reverse() ?? []
);
}
/**
* Detect whether an EPG response should be gunzipped.
* Some providers redirect plain-looking URLs to a `.gz` payload.
@@ -0,0 +1,130 @@
import { PassThrough, Readable } from 'stream';
import { brotliCompressSync, gzipSync } from 'zlib';
import { createDecodedEpgStream } from './epg-stream-decoder';
import { StreamingEpgParser } from './epg-streaming-parser';
const xmltvFixture =
'<?xml version="1.0" encoding="utf-8" ?>' +
'<tv><channel id="test"><display-name>Test Channel</display-name></channel></tv>';
async function collectDecodedText(stream: Readable): Promise<string> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks).toString('utf-8');
}
function parseChannelCount(xml: string): number {
const parser = new StreamingEpgParser(
() => undefined,
() => undefined,
() => undefined
);
parser.write(xml);
return parser.finish().totalChannels;
}
describe('createDecodedEpgStream', () => {
it('decodes Brotli transfer-encoded XML before SAX parsing', async () => {
const decodedText = await collectDecodedText(
createDecodedEpgStream(
Readable.from([brotliCompressSync(xmltvFixture)]),
{ 'content-encoding': 'br' },
false
)
);
expect(decodedText.startsWith('<?xml')).toBe(true);
expect(parseChannelCount(decodedText)).toBe(1);
});
it('decodes gzip transfer-encoded XML before SAX parsing', async () => {
const decodedText = await collectDecodedText(
createDecodedEpgStream(
Readable.from([gzipSync(xmltvFixture)]),
{ 'content-encoding': 'gzip' },
false
)
);
expect(decodedText.startsWith('<?xml')).toBe(true);
expect(parseChannelCount(decodedText)).toBe(1);
});
it('keeps plain XML readable when no transfer encoding is present', async () => {
const decodedText = await collectDecodedText(
createDecodedEpgStream(
Readable.from([Buffer.from(xmltvFixture)]),
{},
false
)
);
expect(decodedText.startsWith('<?xml')).toBe(true);
expect(parseChannelCount(decodedText)).toBe(1);
});
it('decodes gzip XML payloads after transfer decoding', async () => {
const transferEncodedPayload = brotliCompressSync(
gzipSync(xmltvFixture)
);
const decodedText = await collectDecodedText(
createDecodedEpgStream(
Readable.from([transferEncodedPayload]),
{ 'content-encoding': 'br' },
true
)
);
expect(decodedText.startsWith('<?xml')).toBe(true);
expect(parseChannelCount(decodedText)).toBe(1);
});
it('does not double-gunzip mislabelled gzip XML payloads', async () => {
const decodedText = await collectDecodedText(
createDecodedEpgStream(
Readable.from([gzipSync(xmltvFixture)]),
{ 'content-encoding': 'gzip' },
true
)
);
expect(decodedText.startsWith('<?xml')).toBe(true);
expect(parseChannelCount(decodedText)).toBe(1);
});
it('decodes multi-value content-encoding in reverse order', async () => {
const encodedPayload = brotliCompressSync(gzipSync(xmltvFixture));
const decodedText = await collectDecodedText(
createDecodedEpgStream(
Readable.from([encodedPayload]),
{ 'content-encoding': 'gzip, br' },
false
)
);
expect(decodedText.startsWith('<?xml')).toBe(true);
expect(parseChannelCount(decodedText)).toBe(1);
});
it('destroys the source stream when a decoder fails', async () => {
const source = new PassThrough();
const decodedStream = createDecodedEpgStream(
source,
{ 'content-encoding': 'gzip' },
false
);
const readPromise = collectDecodedText(decodedStream);
source.end(Buffer.from('not gzip'));
await expect(readPromise).rejects.toThrow();
expect(source.destroyed).toBe(true);
});
});
@@ -0,0 +1,50 @@
import { PassThrough, Readable, Transform, pipeline } from 'stream';
import {
createBrotliDecompress,
createGunzip,
createInflate,
} from 'zlib';
import {
EpgResponseContentEncoding,
getEpgResponseContentEncodings,
HeaderReader,
} from './epg-response-utils';
function createContentEncodingDecoder(
contentEncoding: EpgResponseContentEncoding
): Transform {
switch (contentEncoding) {
case 'br':
return createBrotliDecompress();
case 'gzip':
return createGunzip();
case 'deflate':
return createInflate();
}
}
export function createDecodedEpgStream(
source: Readable,
headers: HeaderReader,
shouldGunzipPayload: boolean
): Readable {
const contentEncodings = getEpgResponseContentEncodings(headers);
const transforms = contentEncodings.map(createContentEncodingDecoder);
if (shouldGunzipPayload && !contentEncodings.includes('gzip')) {
transforms.push(createGunzip());
}
if (transforms.length === 0) {
return source;
}
const output = new PassThrough();
pipeline([source, ...transforms, output], (error) => {
if (error) {
output.destroy(error);
}
});
return output;
}