fix(epg): preserve UTF-8 characters across XMLTV stream chunks (#1946)

* fix(epg): preserve UTF-8 across XMLTV stream chunks

* test(sources): wait for nested menu detachment before Escape

.changes/epg-utf8-stream.md

@@ -0,0 +1,6 @@
+---
+type: fix
+area: epg
+---
+
+XMLTV imports now preserve accented letters, non-Latin text and emoji when characters span download or decompression chunks. Channel identifiers and programme titles remain intact.

apps/electron-backend-e2e/src/epg.e2e.ts

@@ -105,17 +105,25 @@ test.describe('Electron EPG', () => {
         ['double', true],
         ['single', false],
     ] as const) {
-        test(`@epg @electron imports ${name} gzip with a .gz URL and HTTP gzip`, async ({
+        test(`@epg @electron imports ${name} gzip with UTF-8 across decompression chunks`, async ({
             dataDir,
         }) => {
-            const server = await createGzipEpgServer(
-                createCurrentXmltvFixture(
-                    'gzip-news',
-                    'Gzip News',
-                    'Gzip Bulletin'
-                ),
-                doubleGzip
+            const title = 'Été 日本 🎬 Новости';
+            const xml = createCurrentXmltvFixture(
+                'gzip-news',
+                'Gzip Новости 東京 📺',
+                title
             );
+            // Gunzip emits 16 KiB chunks. Put the first byte of É at the end
+            // of one chunk so the real parser worker must carry UTF-8 state.
+            const titleOffset = Buffer.byteLength(
+                xml.slice(0, xml.indexOf(title))
+            );
+            const padding = ' '.repeat(16 * 1024 - 1 - titleOffset - 7);
+            const paddedXml = xml.replace('<tv>', `<tv><!--${padding}-->`);
+            expect(Buffer.from(paddedXml)[16 * 1024 - 1]).toBe(0xc3);
+            expect(Buffer.from(paddedXml)[16 * 1024]).toBe(0x89);
+            const server = await createGzipEpgServer(paddedXml, doubleGzip);
             try {
                 const app = await launchElectronApp(dataDir);
                 try {
@@ -129,13 +137,13 @@ test.describe('Electron EPG', () => {
                         window.electron.getEpgChannelMetadata(['gzip-news'])
                     );
                     expect(metadata['gzip-news']).toMatchObject({
-                        displayName: 'Gzip News',
+                        displayName: 'Gzip Новости 東京 📺',
                     });
                     const programs = await app.mainWindow.evaluate(() =>
                         window.electron.getChannelPrograms('gzip-news')
                     );
                     expect(programs.map((program) => program.title)).toEqual([
-                        'Gzip Bulletin',
+                        title,
                     ]);
                 } finally {
                     await closeElectronApp(app);

apps/electron-backend-e2e/src/sources.e2e.ts

@@ -786,15 +786,24 @@ https://streams.example.test/original-url.m3u8
             await dropM3uPlaylistOntoWorkspace(app.mainWindow, localFilePath);
             await waitForM3uCatalog(app.mainWindow);
             // Refresh for the active source lives in its switcher row menu.
-            await app.mainWindow.getByTestId('playlist-switcher-trigger').click();
+            await app.mainWindow
+                .getByTestId('playlist-switcher-trigger')
+                .click();
             const currentRow = app.mainWindow.locator(
                 'app-playlist-switcher-panel [role="option"][aria-current="true"]'
             );
             await currentRow.locator('.source-row__more').click();
             await expect(
-                app.mainWindow.getByRole('menuitem', { name: 'Refresh playlist' })
+                app.mainWindow.getByRole('menuitem', {
+                    name: 'Refresh playlist',
+                })
             ).toBeVisible();
             await app.mainWindow.keyboard.press('Escape');
+            // Material retains the nested menu overlay during its exit animation.
+            // Wait for detachment before sending Escape to the switcher beneath it.
+            await expect(
+                app.mainWindow.getByRole('menu', { includeHidden: true })
+            ).toHaveCount(0);
             await app.mainWindow.keyboard.press('Escape');
             await expect(
                 app.mainWindow.locator('app-playlist-switcher-panel')

apps/electron-backend/src/app/workers/epg-parser.worker.spec.ts

@@ -0,0 +1,125 @@
+import { Readable } from 'stream';
+import type { ParsedChannel, ParsedProgram } from './epg-streaming-parser';
+
+const postMessage = jest.fn();
+const insertChannels = jest.fn();
+const insertPrograms = jest.fn();
+const closeDatabase = jest.fn();
+const openEpgSourceStream = jest.fn();
+let handleMessage: (message: { type: string; url: string }) => Promise<void>;
+
+jest.mock('worker_threads', () => ({
+    workerData: {},
+    parentPort: {
+        on: (_event: string, listener: typeof handleMessage) => {
+            handleMessage = listener;
+        },
+        postMessage: (...args: unknown[]) => postMessage(...args),
+    },
+}));
+jest.mock('./worker-runtime-paths', () => ({
+    getWorkerDataNativeModuleSearchPaths: () => [],
+    getNativeModuleSearchPaths: () => [],
+    registerNativeModuleSearchPaths: jest.fn(),
+    loadNativeModuleFromSearchPaths: () => jest.fn(),
+}));
+jest.mock('./epg-database', () => ({
+    EpgDatabase: jest.fn().mockImplementation(() => ({
+        insertChannels,
+        insertPrograms,
+        close: closeDatabase,
+    })),
+}));
+jest.mock('./epg-source-stream', () => ({
+    openEpgSourceStream: (...args: unknown[]) => openEpgSourceStream(...args),
+}));
+jest.mock('../util/network-defaults', () => ({
+    applyElectronNetworkDefaults: jest.fn(),
+}));
+jest.mock('../util/epg-logger', () => ({
+    epgLogger: { log: jest.fn(), error: jest.fn() },
+}));
+
+import './epg-parser.worker';
+
+const sourceUrl = 'https://example.com/guide.xml';
+const channelId = 'café-東京-📺';
+const channelName = 'Новости café 東京 📺';
+const title = 'Été 日本 🎬';
+const description = 'Программа café 東京 📻';
+const xml = `<?xml version="1.0" encoding="UTF-8"?><tv>
+<channel id="${channelId}"><display-name>${channelName}</display-name></channel>
+<programme start="20261015080000 +0000" stop="20261015090000 +0000" channel="${channelId}"><title>${title}</title><desc>${description}</desc></programme>
+</tv>`;
+
+function storedChannels(): ParsedChannel[] {
+    return insertChannels.mock.calls.flatMap(([channels]) => channels);
+}
+
+function storedPrograms(): ParsedProgram[] {
+    return insertPrograms.mock.calls.flatMap(([programs]) => programs);
+}
+
+describe('EPG worker UTF-8 input', () => {
+    beforeEach(() => jest.clearAllMocks());
+
+    it.each([1, 2, 3, 7])(
+        'preserves XMLTV identifiers and text with %i-byte chunks',
+        async (chunkSize) => {
+            const bytes = Buffer.from(xml);
+            const chunks: Buffer[] = [];
+            for (let offset = 0; offset < bytes.length; offset += chunkSize) {
+                chunks.push(bytes.subarray(offset, offset + chunkSize));
+            }
+            openEpgSourceStream.mockResolvedValue(Readable.from(chunks));
+
+            await handleMessage({ type: 'FETCH_EPG', url: sourceUrl });
+
+            expect(storedChannels()).toEqual([
+                expect.objectContaining({
+                    id: channelId,
+                    displayName: [{ lang: '', value: channelName }],
+                }),
+            ]);
+            expect(storedPrograms()).toEqual([
+                expect.objectContaining({
+                    channel: channelId,
+                    title: [{ lang: '', value: title }],
+                    desc: [{ lang: '', value: description }],
+                }),
+            ]);
+            expect(postMessage).toHaveBeenCalledWith({
+                type: 'EPG_COMPLETE',
+                url: sourceUrl,
+                stats: { totalChannels: 1, totalPrograms: 1 },
+            });
+            expect(closeDatabase).toHaveBeenCalledTimes(1);
+        }
+    );
+
+    it('reports a stream failure while a UTF-8 character is incomplete', async () => {
+        openEpgSourceStream.mockResolvedValue(
+            Readable.from(
+                (async function* () {
+                    yield Buffer.from('<tv><channel id="');
+                    yield Buffer.from([0xf0, 0x9f]);
+                    throw new Error('interrupted source');
+                })()
+            )
+        );
+
+        await handleMessage({ type: 'FETCH_EPG', url: sourceUrl });
+
+        expect(postMessage).toHaveBeenCalledWith(
+            expect.objectContaining({
+                type: 'EPG_ERROR',
+                error: 'interrupted source',
+            })
+        );
+        expect(postMessage).not.toHaveBeenCalledWith(
+            expect.objectContaining({ type: 'EPG_COMPLETE' })
+        );
+        expect(insertChannels).not.toHaveBeenCalled();
+        expect(closeDatabase).toHaveBeenCalled();
+    });
+});

apps/electron-backend/src/app/workers/epg-parser.worker.ts

@@ -135,11 +135,13 @@ async function fetchAndParseEpgStreaming(
         );
 
         return new Promise((resolve, reject) => {
-            const dataStream = decodedStream;
+            // Decode after decompression, retaining partial UTF-8 sequences
+            // between chunks and flushing the decoder before the end event.
+            const dataStream = decodedStream.setEncoding('utf8');
 
-            dataStream.on('data', (chunk: Buffer) => {
+            dataStream.on('data', (chunk: string) => {
                 try {
-                    parser.write(chunk.toString('utf-8'));
+                    parser.write(chunk);
                 } catch (err) {
                     epgLogger.error(loggerLabel, 'Parse error:', err);
                     epgDb.close();

docs/architecture/m3u-playlist-module.md

@@ -1180,6 +1180,11 @@ retried as plain XML. Source errors and consumer cancellation terminate the
 whole decoding chain. The decoder does not recursively unpack file layers or
 change XML parsing, source reconciliation, or database persistence contracts.
 
+After all decompression stages, the parser worker uses a single UTF-8 stream
+decoder per import. It retains incomplete multibyte characters across chunks
+and flushes before finishing SAX parsing, preserving channel IDs and programme
+text in both remote and local feeds without buffering the whole guide.
+
 ### Local XMLTV files
 
 An EPG source is not necessarily a URL. Settings → EPG and the playlist