mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-10 01:56:16 -08:00
fix(epg): decode compressed XMLTV responses
Decode HTTP Content-Encoding for EPG XMLTV streams before SAX parsing and harden gzip payload handling for mislabelled providers. Validation: - pnpm nx test electron-backend --testFile=epg-stream-decoder.spec.ts - pnpm nx test electron-backend --testFile=epg-response-utils.spec.ts - pnpm nx test electron-backend - pnpm nx lint electron-backend - pnpm nx run electron-backend:build-worker
This commit is contained in:
1 parent
d37e0f84b9
commit
36a7ce9f3d
5 files changed
+279
-16
No files matched your search
@@ -6,10 +6,13 @@ import {
|
||||
} from '@iptvnator/shared/interfaces';
|
||||
import { Readable } from 'stream';
|
||||
import { parentPort, workerData } from 'worker_threads';
|
||||
import { createGunzip } from 'zlib';
|
||||
import { EpgDatabase, EpgDatabaseClearOperation } from './epg-database';
|
||||
import { createDecodedEpgStream } from './epg-stream-decoder';
|
||||
import { StreamingEpgParser } from './epg-streaming-parser';
|
||||
import { shouldGunzipEpgResponse } from './epg-response-utils';
|
||||
import {
|
||||
getEpgResponseContentEncoding,
|
||||
shouldGunzipEpgResponse,
|
||||
} from './epg-response-utils';
|
||||
import {
|
||||
isPrivateNetworkUrlAccessAllowed,
|
||||
UnsafeUrlError,
|
||||
@@ -130,6 +133,9 @@ async function fetchAndParseEpgStreaming(
|
||||
headers: response.headers,
|
||||
url: responseUrl,
|
||||
});
|
||||
const contentEncoding = getEpgResponseContentEncoding(
|
||||
response.headers
|
||||
);
|
||||
|
||||
if (responseUrl && responseUrl !== url) {
|
||||
console.log(
|
||||
@@ -142,6 +148,12 @@ async function fetchAndParseEpgStreaming(
|
||||
loggerLabel,
|
||||
`EPG response detected as gzipped: ${isGzipped}`
|
||||
);
|
||||
if (contentEncoding) {
|
||||
console.log(
|
||||
loggerLabel,
|
||||
`EPG response content-encoding: ${contentEncoding}`
|
||||
);
|
||||
}
|
||||
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
@@ -175,18 +187,11 @@ async function fetchAndParseEpgStreaming(
|
||||
);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let dataStream: Readable = response.data;
|
||||
|
||||
if (isGzipped) {
|
||||
const gunzip = createGunzip();
|
||||
dataStream = response.data.pipe(gunzip);
|
||||
|
||||
gunzip.on('error', (err) => {
|
||||
console.error(loggerLabel, 'Gunzip error:', err);
|
||||
epgDb.close();
|
||||
reject(err);
|
||||
});
|
||||
}
|
||||
const dataStream = createDecodedEpgStream(
|
||||
response.data,
|
||||
response.headers,
|
||||
isGzipped
|
||||
);
|
||||
|
||||
dataStream.on('data', (chunk: Buffer) => {
|
||||
try {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { shouldGunzipEpgResponse } from './epg-response-utils';
|
||||
import {
|
||||
getEpgResponseContentEncoding,
|
||||
shouldGunzipEpgResponse,
|
||||
} from './epg-response-utils';
|
||||
|
||||
describe('shouldGunzipEpgResponse', () => {
|
||||
it('returns true for original .gz URLs', () => {
|
||||
@@ -71,3 +74,45 @@ describe('shouldGunzipEpgResponse', () => {
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getEpgResponseContentEncoding', () => {
|
||||
it('detects Brotli transfer encoding from response headers', () => {
|
||||
expect(
|
||||
getEpgResponseContentEncoding({
|
||||
'content-encoding': 'br',
|
||||
})
|
||||
).toBe('br');
|
||||
});
|
||||
|
||||
it('detects gzip transfer encoding from response headers', () => {
|
||||
expect(
|
||||
getEpgResponseContentEncoding(
|
||||
new Headers([['content-encoding', 'gzip']])
|
||||
)
|
||||
).toBe('gzip');
|
||||
});
|
||||
|
||||
it('detects deflate transfer encoding from response headers', () => {
|
||||
expect(
|
||||
getEpgResponseContentEncoding({
|
||||
'Content-Encoding': 'deflate',
|
||||
})
|
||||
).toBe('deflate');
|
||||
});
|
||||
|
||||
it('ignores unsupported transfer encodings', () => {
|
||||
expect(
|
||||
getEpgResponseContentEncoding({
|
||||
'content-encoding': 'zstd',
|
||||
})
|
||||
).toBe(null);
|
||||
});
|
||||
|
||||
it('returns the outermost supported transfer encoding first', () => {
|
||||
expect(
|
||||
getEpgResponseContentEncoding({
|
||||
'content-encoding': 'gzip, br',
|
||||
})
|
||||
).toBe('br');
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,17 @@
|
||||
type HeaderReader =
|
||||
export type HeaderReader =
|
||||
| Record<string, unknown>
|
||||
| {
|
||||
get(name: string): unknown;
|
||||
};
|
||||
|
||||
export type EpgResponseContentEncoding = 'br' | 'gzip' | 'deflate';
|
||||
|
||||
const SUPPORTED_CONTENT_ENCODINGS: readonly EpgResponseContentEncoding[] = [
|
||||
'br',
|
||||
'gzip',
|
||||
'deflate',
|
||||
];
|
||||
|
||||
function getHeaderValue(headers: HeaderReader, name: string): string | null {
|
||||
const value =
|
||||
'get' in headers && typeof headers.get === 'function'
|
||||
@@ -27,6 +35,31 @@ function hasGzipPath(url: string | null | undefined): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
export function getEpgResponseContentEncoding(
|
||||
headers: HeaderReader
|
||||
): EpgResponseContentEncoding | null {
|
||||
const contentEncoding = getEpgResponseContentEncodings(headers).at(0);
|
||||
|
||||
return contentEncoding ?? null;
|
||||
}
|
||||
|
||||
export function getEpgResponseContentEncodings(
|
||||
headers: HeaderReader
|
||||
): EpgResponseContentEncoding[] {
|
||||
return (
|
||||
getHeaderValue(headers, 'content-encoding')
|
||||
?.toLowerCase()
|
||||
.split(',')
|
||||
.map((encoding) => encoding.trim())
|
||||
.filter((encoding): encoding is EpgResponseContentEncoding =>
|
||||
SUPPORTED_CONTENT_ENCODINGS.includes(
|
||||
encoding as EpgResponseContentEncoding
|
||||
)
|
||||
)
|
||||
.reverse() ?? []
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect whether an EPG response should be gunzipped.
|
||||
* Some providers redirect plain-looking URLs to a `.gz` payload.
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import { PassThrough, Readable } from 'stream';
|
||||
import { brotliCompressSync, gzipSync } from 'zlib';
|
||||
import { createDecodedEpgStream } from './epg-stream-decoder';
|
||||
import { StreamingEpgParser } from './epg-streaming-parser';
|
||||
|
||||
const xmltvFixture =
|
||||
'<?xml version="1.0" encoding="utf-8" ?>' +
|
||||
'<tv><channel id="test"><display-name>Test Channel</display-name></channel></tv>';
|
||||
|
||||
async function collectDecodedText(stream: Readable): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
|
||||
return Buffer.concat(chunks).toString('utf-8');
|
||||
}
|
||||
|
||||
function parseChannelCount(xml: string): number {
|
||||
const parser = new StreamingEpgParser(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
() => undefined
|
||||
);
|
||||
|
||||
parser.write(xml);
|
||||
return parser.finish().totalChannels;
|
||||
}
|
||||
|
||||
describe('createDecodedEpgStream', () => {
|
||||
it('decodes Brotli transfer-encoded XML before SAX parsing', async () => {
|
||||
const decodedText = await collectDecodedText(
|
||||
createDecodedEpgStream(
|
||||
Readable.from([brotliCompressSync(xmltvFixture)]),
|
||||
{ 'content-encoding': 'br' },
|
||||
false
|
||||
)
|
||||
);
|
||||
|
||||
expect(decodedText.startsWith('<?xml')).toBe(true);
|
||||
expect(parseChannelCount(decodedText)).toBe(1);
|
||||
});
|
||||
|
||||
it('decodes gzip transfer-encoded XML before SAX parsing', async () => {
|
||||
const decodedText = await collectDecodedText(
|
||||
createDecodedEpgStream(
|
||||
Readable.from([gzipSync(xmltvFixture)]),
|
||||
{ 'content-encoding': 'gzip' },
|
||||
false
|
||||
)
|
||||
);
|
||||
|
||||
expect(decodedText.startsWith('<?xml')).toBe(true);
|
||||
expect(parseChannelCount(decodedText)).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps plain XML readable when no transfer encoding is present', async () => {
|
||||
const decodedText = await collectDecodedText(
|
||||
createDecodedEpgStream(
|
||||
Readable.from([Buffer.from(xmltvFixture)]),
|
||||
{},
|
||||
false
|
||||
)
|
||||
);
|
||||
|
||||
expect(decodedText.startsWith('<?xml')).toBe(true);
|
||||
expect(parseChannelCount(decodedText)).toBe(1);
|
||||
});
|
||||
|
||||
it('decodes gzip XML payloads after transfer decoding', async () => {
|
||||
const transferEncodedPayload = brotliCompressSync(
|
||||
gzipSync(xmltvFixture)
|
||||
);
|
||||
|
||||
const decodedText = await collectDecodedText(
|
||||
createDecodedEpgStream(
|
||||
Readable.from([transferEncodedPayload]),
|
||||
{ 'content-encoding': 'br' },
|
||||
true
|
||||
)
|
||||
);
|
||||
|
||||
expect(decodedText.startsWith('<?xml')).toBe(true);
|
||||
expect(parseChannelCount(decodedText)).toBe(1);
|
||||
});
|
||||
|
||||
it('does not double-gunzip mislabelled gzip XML payloads', async () => {
|
||||
const decodedText = await collectDecodedText(
|
||||
createDecodedEpgStream(
|
||||
Readable.from([gzipSync(xmltvFixture)]),
|
||||
{ 'content-encoding': 'gzip' },
|
||||
true
|
||||
)
|
||||
);
|
||||
|
||||
expect(decodedText.startsWith('<?xml')).toBe(true);
|
||||
expect(parseChannelCount(decodedText)).toBe(1);
|
||||
});
|
||||
|
||||
it('decodes multi-value content-encoding in reverse order', async () => {
|
||||
const encodedPayload = brotliCompressSync(gzipSync(xmltvFixture));
|
||||
|
||||
const decodedText = await collectDecodedText(
|
||||
createDecodedEpgStream(
|
||||
Readable.from([encodedPayload]),
|
||||
{ 'content-encoding': 'gzip, br' },
|
||||
false
|
||||
)
|
||||
);
|
||||
|
||||
expect(decodedText.startsWith('<?xml')).toBe(true);
|
||||
expect(parseChannelCount(decodedText)).toBe(1);
|
||||
});
|
||||
|
||||
it('destroys the source stream when a decoder fails', async () => {
|
||||
const source = new PassThrough();
|
||||
const decodedStream = createDecodedEpgStream(
|
||||
source,
|
||||
{ 'content-encoding': 'gzip' },
|
||||
false
|
||||
);
|
||||
|
||||
const readPromise = collectDecodedText(decodedStream);
|
||||
source.end(Buffer.from('not gzip'));
|
||||
|
||||
await expect(readPromise).rejects.toThrow();
|
||||
expect(source.destroyed).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import { PassThrough, Readable, Transform, pipeline } from 'stream';
|
||||
import {
|
||||
createBrotliDecompress,
|
||||
createGunzip,
|
||||
createInflate,
|
||||
} from 'zlib';
|
||||
import {
|
||||
EpgResponseContentEncoding,
|
||||
getEpgResponseContentEncodings,
|
||||
HeaderReader,
|
||||
} from './epg-response-utils';
|
||||
|
||||
function createContentEncodingDecoder(
|
||||
contentEncoding: EpgResponseContentEncoding
|
||||
): Transform {
|
||||
switch (contentEncoding) {
|
||||
case 'br':
|
||||
return createBrotliDecompress();
|
||||
case 'gzip':
|
||||
return createGunzip();
|
||||
case 'deflate':
|
||||
return createInflate();
|
||||
}
|
||||
}
|
||||
|
||||
export function createDecodedEpgStream(
|
||||
source: Readable,
|
||||
headers: HeaderReader,
|
||||
shouldGunzipPayload: boolean
|
||||
): Readable {
|
||||
const contentEncodings = getEpgResponseContentEncodings(headers);
|
||||
const transforms = contentEncodings.map(createContentEncodingDecoder);
|
||||
|
||||
if (shouldGunzipPayload && !contentEncodings.includes('gzip')) {
|
||||
transforms.push(createGunzip());
|
||||
}
|
||||
|
||||
if (transforms.length === 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
const output = new PassThrough();
|
||||
pipeline([source, ...transforms, output], (error) => {
|
||||
if (error) {
|
||||
output.destroy(error);
|
||||
}
|
||||
});
|
||||
|
||||
return output;
|
||||
}
|
||||
Reference in new issue
Block a user