mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-13 12:36:15 -08:00
fix(epg): remove cached XMLTV data after source deletion (#1548)
* fix(epg): remove cached XMLTV data after source deletion * fix(epg): close source reconciliation review races * fix(epg): serialize cleanup with replacement imports * fix(epg): report retired worker exits as cancellations * fix(epg): preserve source metadata through cache cleanup * refactor(epg): separate worker runtime and import lifecycle * fix(epg): skip cleanup for unchanged source settings * fix(epg): cancel retired error rows and pending retries * fix(epg): redact diagnostics and mirror committed settings after cleanup errors * fix(epg): preserve metadata writer order independently of timestamps
This commit is contained in:
1 parent
baba0529ef
commit
9de480826c
55 files changed
+3538
-615
No files matched your search
@@ -536,6 +536,12 @@ export const epgPreloadCases: PreloadInvokeCase[] = [
|
||||
channel: 'EPG_CLEAR_ALL',
|
||||
forwardedArgs: [],
|
||||
},
|
||||
{
|
||||
method: 'reconcileEpgSources',
|
||||
args: [epgUrls],
|
||||
channel: 'EPG_RECONCILE_SOURCES',
|
||||
forwardedArgs: [{ urls: epgUrls }],
|
||||
},
|
||||
{
|
||||
method: 'clearEpgDataForSource',
|
||||
args: ['https://example.com/guide.xml'],
|
||||
|
||||
@@ -615,11 +615,7 @@ const electronApi: ElectronBridgeApi = {
|
||||
sessionId: string,
|
||||
style: EmbeddedMpvSubtitleStyle
|
||||
): Promise<EmbeddedMpvSession | null> =>
|
||||
ipcRenderer.invoke(
|
||||
'EMBEDDED_MPV_SET_SUBTITLE_STYLE',
|
||||
sessionId,
|
||||
style
|
||||
),
|
||||
ipcRenderer.invoke('EMBEDDED_MPV_SET_SUBTITLE_STYLE', sessionId, style),
|
||||
selectEmbeddedMpvSubtitleFile: (): Promise<string | null> =>
|
||||
ipcRenderer.invoke('EMBEDDED_MPV_SELECT_SUBTITLE_FILE'),
|
||||
setEmbeddedMpvSpeed: (
|
||||
@@ -681,6 +677,8 @@ const electronApi: ElectronBridgeApi = {
|
||||
forceFetchEpg: (url: string, options?: ElectronBridgeTrustOptions) =>
|
||||
ipcRenderer.invoke('EPG_FORCE_FETCH', { url, options }),
|
||||
clearEpgData: () => ipcRenderer.invoke('EPG_CLEAR_ALL'),
|
||||
reconcileEpgSources: (urls: string[]) =>
|
||||
ipcRenderer.invoke('EPG_RECONCILE_SOURCES', { urls }),
|
||||
clearEpgDataForSource: (sourceUrl: string) =>
|
||||
ipcRenderer.invoke('EPG_CLEAR_SOURCE', { sourceUrl }),
|
||||
checkEpgFreshness: (urls: string[], maxAgeHours?: number) =>
|
||||
@@ -1118,8 +1116,7 @@ const electronApi: ElectronBridgeApi = {
|
||||
recordingsUpdatePrograms: (
|
||||
targetPath: string,
|
||||
programs: RecordingProgramSnapshot[]
|
||||
) =>
|
||||
ipcRenderer.invoke('RECORDINGS_UPDATE_PROGRAMS', targetPath, programs),
|
||||
) => ipcRenderer.invoke('RECORDINGS_UPDATE_PROGRAMS', targetPath, programs),
|
||||
recordingsRevealFile: (filePath: string) =>
|
||||
ipcRenderer.invoke('RECORDINGS_REVEAL_FILE', filePath),
|
||||
recordingsPlayFile: (filePath: string) =>
|
||||
|
||||
@@ -4,40 +4,41 @@
|
||||
*/
|
||||
|
||||
export {
|
||||
// Tables
|
||||
playlists,
|
||||
categories,
|
||||
content,
|
||||
recentlyViewed,
|
||||
favorites,
|
||||
epgChannels,
|
||||
epgChannelMappings,
|
||||
epgPrograms,
|
||||
playbackPositions,
|
||||
downloads,
|
||||
recordings,
|
||||
appState,
|
||||
// Types
|
||||
type Playlist,
|
||||
type NewPlaylist,
|
||||
type AppState,
|
||||
type NewAppState,
|
||||
type Category,
|
||||
type NewCategory,
|
||||
type Content,
|
||||
type NewContent,
|
||||
type RecentlyViewed,
|
||||
type NewRecentlyViewed,
|
||||
type Favorite,
|
||||
type NewFavorite,
|
||||
type EpgChannel,
|
||||
type NewEpgChannel,
|
||||
type EpgProgramDb,
|
||||
type NewEpgProgramDb,
|
||||
type PlaybackPosition,
|
||||
type NewPlaybackPosition,
|
||||
type Download,
|
||||
type NewDownload,
|
||||
type Recording,
|
||||
type NewRecording,
|
||||
// Tables
|
||||
playlists,
|
||||
categories,
|
||||
content,
|
||||
recentlyViewed,
|
||||
favorites,
|
||||
epgChannels,
|
||||
epgChannelSources,
|
||||
epgChannelMappings,
|
||||
epgPrograms,
|
||||
playbackPositions,
|
||||
downloads,
|
||||
recordings,
|
||||
appState,
|
||||
// Types
|
||||
type Playlist,
|
||||
type NewPlaylist,
|
||||
type AppState,
|
||||
type NewAppState,
|
||||
type Category,
|
||||
type NewCategory,
|
||||
type Content,
|
||||
type NewContent,
|
||||
type RecentlyViewed,
|
||||
type NewRecentlyViewed,
|
||||
type Favorite,
|
||||
type NewFavorite,
|
||||
type EpgChannel,
|
||||
type NewEpgChannel,
|
||||
type EpgProgramDb,
|
||||
type NewEpgProgramDb,
|
||||
type PlaybackPosition,
|
||||
type NewPlaybackPosition,
|
||||
type Download,
|
||||
type NewDownload,
|
||||
type Recording,
|
||||
type NewRecording,
|
||||
} from '@iptvnator/shared/database';
|
||||
@@ -0,0 +1,281 @@
|
||||
import { epgLogger } from '../util/epg-logger';
|
||||
import type { Worker } from 'worker_threads';
|
||||
import type {
|
||||
ElectronBridgeSecurityErrorCode,
|
||||
ElectronBridgeTrustOptions,
|
||||
} from '@iptvnator/shared/interfaces';
|
||||
import { epgSourceGeneration } from './epg-source-generation';
|
||||
import type { EpgWorkerRuntime } from './epg-worker-runtime';
|
||||
import type { EpgProgressStats, EpgWorkerService } from './epg-worker.service';
|
||||
|
||||
interface EpgWorkerMessage {
|
||||
type: string;
|
||||
error?: string;
|
||||
errorCode?: ElectronBridgeSecurityErrorCode;
|
||||
errorHost?: string;
|
||||
url?: string;
|
||||
stats?: EpgProgressStats;
|
||||
}
|
||||
|
||||
interface FetchContext {
|
||||
runtime: EpgWorkerRuntime;
|
||||
workers: Map<string, Worker>;
|
||||
fetchedUrls: Set<string>;
|
||||
loggerLabel: string;
|
||||
fetchTimeoutMs: number;
|
||||
sendProgressToRenderer: EpgWorkerService['sendProgressToRenderer'];
|
||||
}
|
||||
|
||||
/** One import's message, cancellation, timeout and exit lifecycle. */
|
||||
export function runEpgFetch(
|
||||
url: string,
|
||||
options: ElectronBridgeTrustOptions,
|
||||
context: FetchContext
|
||||
): Promise<void> {
|
||||
const generation = epgSourceGeneration(url);
|
||||
return new Promise((resolve, reject) => {
|
||||
let worker: Worker;
|
||||
try {
|
||||
worker = context.runtime.createEpgWorker();
|
||||
} catch (error) {
|
||||
epgLogger.error(
|
||||
context.loggerLabel,
|
||||
'Failed to create worker:',
|
||||
error
|
||||
);
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
context.workers.set(url, worker);
|
||||
|
||||
// Guards against double-settling and keeps the outer loop moving
|
||||
// when the worker dies or hangs without sending EPG_COMPLETE/EPG_ERROR.
|
||||
let settled = false;
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
let lastProgressStats: EpgProgressStats = {
|
||||
totalChannels: 0,
|
||||
totalPrograms: 0,
|
||||
};
|
||||
|
||||
const clearFetchTimeout = () => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
timeoutId = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const settle = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearFetchTimeout();
|
||||
fn();
|
||||
};
|
||||
|
||||
const cancelIfRetired = (exited = false): boolean => {
|
||||
if (generation === epgSourceGeneration(url)) return false;
|
||||
if (context.workers.get(url) === worker)
|
||||
context.workers.delete(url);
|
||||
settle(() => {
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'cancelled',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
generation
|
||||
);
|
||||
if (exited) resolve();
|
||||
else
|
||||
void context.runtime
|
||||
.terminateWorker(worker, 'retired fetch')
|
||||
.then(() => resolve());
|
||||
});
|
||||
return true;
|
||||
};
|
||||
|
||||
const scheduleFetchTimeout = () => {
|
||||
clearFetchTimeout();
|
||||
timeoutId = setTimeout(() => {
|
||||
handleFetchTimeout();
|
||||
}, context.fetchTimeoutMs);
|
||||
};
|
||||
|
||||
const hasProgressMoved = (stats: EpgProgressStats): boolean =>
|
||||
stats.totalChannels > lastProgressStats.totalChannels ||
|
||||
stats.totalPrograms > lastProgressStats.totalPrograms;
|
||||
|
||||
const recordProgress = (stats: EpgProgressStats): void => {
|
||||
lastProgressStats = {
|
||||
totalChannels: Math.max(
|
||||
lastProgressStats.totalChannels,
|
||||
stats.totalChannels
|
||||
),
|
||||
totalPrograms: Math.max(
|
||||
lastProgressStats.totalPrograms,
|
||||
stats.totalPrograms
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
const handleFetchTimeout = () => {
|
||||
if (settled || cancelIfRetired()) return;
|
||||
const errorMessage = `EPG fetch timed out after ${
|
||||
context.fetchTimeoutMs / 1000
|
||||
}s without progress`;
|
||||
epgLogger.error(context.loggerLabel, errorMessage);
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
errorMessage
|
||||
);
|
||||
context.workers.delete(url);
|
||||
// Settle only after the worker thread is really gone: a
|
||||
// terminated-but-still-running worker can keep holding the
|
||||
// SQLite lock and block the next EPG fetch.
|
||||
settle(() => {
|
||||
void context.runtime
|
||||
.terminateWorker(worker, 'timed out fetch')
|
||||
.then(() => reject(new Error(errorMessage)));
|
||||
});
|
||||
};
|
||||
|
||||
scheduleFetchTimeout();
|
||||
|
||||
worker.on('message', async (message: EpgWorkerMessage) => {
|
||||
if (settled || generation !== epgSourceGeneration(url)) return;
|
||||
try {
|
||||
switch (message.type) {
|
||||
case 'READY':
|
||||
scheduleFetchTimeout();
|
||||
context.sendProgressToRenderer(url, 'loading', {
|
||||
totalChannels: 0,
|
||||
totalPrograms: 0,
|
||||
});
|
||||
worker.postMessage({
|
||||
type: 'FETCH_EPG',
|
||||
url,
|
||||
options,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'EPG_PROGRESS':
|
||||
if (message.stats) {
|
||||
if (hasProgressMoved(message.stats)) {
|
||||
recordProgress(message.stats);
|
||||
scheduleFetchTimeout();
|
||||
}
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'loading',
|
||||
message.stats
|
||||
);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'EPG_COMPLETE':
|
||||
epgLogger.log(
|
||||
context.loggerLabel,
|
||||
'EPG parsing complete:',
|
||||
message.stats
|
||||
);
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'complete',
|
||||
message.stats
|
||||
);
|
||||
context.fetchedUrls.add(url);
|
||||
context.workers.delete(url);
|
||||
settle(() => {
|
||||
void context.runtime
|
||||
.terminateWorker(worker, 'completed fetch')
|
||||
.then(() => resolve());
|
||||
});
|
||||
break;
|
||||
|
||||
case 'EPG_ERROR':
|
||||
epgLogger.error(
|
||||
context.loggerLabel,
|
||||
'Worker error:',
|
||||
message.error
|
||||
);
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
message.error,
|
||||
undefined,
|
||||
message.errorCode,
|
||||
message.errorHost
|
||||
);
|
||||
context.workers.delete(url);
|
||||
settle(() => {
|
||||
void context.runtime
|
||||
.terminateWorker(worker, 'failed fetch')
|
||||
.then(() =>
|
||||
reject(
|
||||
new Error(
|
||||
message.error || 'Unknown error'
|
||||
)
|
||||
)
|
||||
);
|
||||
});
|
||||
break;
|
||||
}
|
||||
} catch (err) {
|
||||
if (settled || cancelIfRetired()) return;
|
||||
epgLogger.error(
|
||||
context.loggerLabel,
|
||||
'Error handling message:',
|
||||
err
|
||||
);
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
err instanceof Error ? err.message : String(err)
|
||||
);
|
||||
context.workers.delete(url);
|
||||
settle(() => {
|
||||
void context.runtime
|
||||
.terminateWorker(worker, 'failed message handling')
|
||||
.then(() => reject(err));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
worker.on('error', (error) => {
|
||||
if (settled || cancelIfRetired()) return;
|
||||
epgLogger.error(context.loggerLabel, 'Worker error event:', error);
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
error.message
|
||||
);
|
||||
context.workers.delete(url);
|
||||
settle(() => {
|
||||
void context.runtime
|
||||
.terminateWorker(worker, 'errored fetch')
|
||||
.then(() => reject(error));
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
if (settled || cancelIfRetired(true)) return;
|
||||
const errorMessage = `Worker exited unexpectedly (code ${code})`;
|
||||
epgLogger.error(context.loggerLabel, errorMessage);
|
||||
context.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
errorMessage
|
||||
);
|
||||
context.workers.delete(url);
|
||||
settle(() => reject(new Error(errorMessage)));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { epgLogger } from '../util/epg-logger';
|
||||
import { epgSourceGeneration, requestEpgSource } from './epg-source-generation';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { ElectronBridgeTrustOptions } from '@iptvnator/shared/interfaces';
|
||||
import { getDatabase } from '../database/connection';
|
||||
@@ -45,12 +47,13 @@ export async function checkEpgFreshness(
|
||||
const db = await getDatabase();
|
||||
|
||||
for (const url of urls) {
|
||||
const generation = epgSourceGeneration(url);
|
||||
if (!url?.trim()) continue;
|
||||
|
||||
const result = await db
|
||||
.select({ updatedAt: schema.epgChannels.updatedAt })
|
||||
.from(schema.epgChannels)
|
||||
.where(eq(schema.epgChannels.sourceUrl, url))
|
||||
.select({ updatedAt: schema.epgChannelSources.updatedAt })
|
||||
.from(schema.epgChannelSources)
|
||||
.where(eq(schema.epgChannelSources.sourceUrl, url))
|
||||
.limit(1);
|
||||
|
||||
const isFresh =
|
||||
@@ -58,6 +61,7 @@ export async function checkEpgFreshness(
|
||||
result[0].updatedAt &&
|
||||
result[0].updatedAt >= cutoffTime;
|
||||
|
||||
if (generation !== epgSourceGeneration(url)) continue;
|
||||
if (isFresh) {
|
||||
freshUrls.push(url);
|
||||
epgWorkerService.markFetchedUrl(url);
|
||||
@@ -66,18 +70,18 @@ export async function checkEpgFreshness(
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(loggerLabel, 'Error checking EPG freshness:', error);
|
||||
epgLogger.error(loggerLabel, 'Error checking EPG freshness:', error);
|
||||
return { staleUrls: urls, freshUrls: [] };
|
||||
}
|
||||
|
||||
if (freshUrls.length > 0) {
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
loggerLabel,
|
||||
`EPG fresh (skipping): ${freshUrls.length} source(s)`
|
||||
);
|
||||
}
|
||||
if (staleUrls.length > 0) {
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
loggerLabel,
|
||||
`EPG stale (will fetch): ${staleUrls.length} source(s)`
|
||||
);
|
||||
@@ -95,7 +99,12 @@ export async function handleFetchEpg(
|
||||
urls: string[],
|
||||
options: ElectronBridgeTrustOptions = {}
|
||||
): Promise<EpgFetchResult> {
|
||||
const validUrls = urls.filter((url) => url?.trim());
|
||||
const validUrls = urls
|
||||
.filter((url) => url?.trim())
|
||||
.map((url) => url.trim());
|
||||
const generations = new Map(
|
||||
validUrls.map((url) => [url, requestEpgSource(url)])
|
||||
);
|
||||
|
||||
if (validUrls.length === 0) {
|
||||
return { success: false, message: 'No valid URLs provided' };
|
||||
@@ -118,11 +127,13 @@ export async function handleFetchEpg(
|
||||
// a 'queued' status, then fetchEpgFromUrl silently skips the URL and no
|
||||
// completion update ever arrives, leaving the UI stuck at "queued".
|
||||
const urlsToFetch = staleUrls.filter(
|
||||
(url) => !epgWorkerService.hasFetchedUrl(url)
|
||||
(url) =>
|
||||
generations.get(url) === epgSourceGeneration(url) &&
|
||||
!epgWorkerService.hasFetchedUrl(url)
|
||||
);
|
||||
|
||||
if (urlsToFetch.length === 0) {
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
loggerLabel,
|
||||
`All ${staleUrls.length} stale URL(s) already fetched this session; skipping`
|
||||
);
|
||||
@@ -142,16 +153,23 @@ export async function handleFetchEpg(
|
||||
const errors: string[] = [];
|
||||
for (const url of urlsToFetch) {
|
||||
try {
|
||||
if (generations.get(url) !== epgSourceGeneration(url)) {
|
||||
epgWorkerService.sendProgressToRenderer(
|
||||
url,
|
||||
'cancelled',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
generations.get(url)
|
||||
);
|
||||
continue;
|
||||
}
|
||||
await epgWorkerService.fetchEpgFromUrl(url, options);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
loggerLabel,
|
||||
`Error fetching EPG from ${url}:`,
|
||||
error
|
||||
);
|
||||
errors.push(
|
||||
error instanceof Error ? error.message : String(error)
|
||||
);
|
||||
epgLogger.error(loggerLabel, 'Error fetching EPG source:', error);
|
||||
errors.push(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { epgLogger } from '../util/epg-logger';
|
||||
import { and, eq, inArray, isNull, or, sql, type SQL } from 'drizzle-orm';
|
||||
import {
|
||||
EpgChannelMetadata,
|
||||
@@ -154,7 +155,7 @@ export class EpgQueryService {
|
||||
|
||||
return [];
|
||||
} catch (error) {
|
||||
console.error(
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Error getting channel programs:',
|
||||
error
|
||||
@@ -310,7 +311,7 @@ export class EpgQueryService {
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error(
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Error getting batch current programs:',
|
||||
error
|
||||
@@ -335,7 +336,7 @@ export class EpgQueryService {
|
||||
|
||||
return { channels, programs: [] };
|
||||
} catch (error) {
|
||||
console.error(
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Error getting all channels:',
|
||||
error
|
||||
@@ -391,7 +392,7 @@ export class EpgQueryService {
|
||||
])
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Error getting channel metadata:',
|
||||
error
|
||||
@@ -439,7 +440,7 @@ export class EpgQueryService {
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Error getting channels by range:',
|
||||
error
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/** Retires queued imports as well as workers already parsing a removed URL. */
|
||||
const generations = new Map<string, number>();
|
||||
const requests = new Map<string, symbol>();
|
||||
export function epgSourceGeneration(url: string): number {
|
||||
return generations.get(url.trim()) ?? 0;
|
||||
}
|
||||
export function requestEpgSource(url: string): number {
|
||||
requests.set(url.trim(), Symbol());
|
||||
return epgSourceGeneration(url);
|
||||
}
|
||||
export function retireEpgSource(url: string): void {
|
||||
generations.set(url.trim(), epgSourceGeneration(url) + 1);
|
||||
}
|
||||
export function requestedEpgSources(): Map<string, symbol> {
|
||||
return new Map(requests);
|
||||
}
|
||||
export function forgetEpgSourceRequest(
|
||||
url: string,
|
||||
request: symbol | undefined
|
||||
): void {
|
||||
// A request received during cleanup still needs reconciliation next time.
|
||||
if (requests.get(url.trim()) === request) requests.delete(url.trim());
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import {
|
||||
appState,
|
||||
epgChannels,
|
||||
epgChannelSources,
|
||||
epgPrograms,
|
||||
playlists,
|
||||
} from '../database/schema';
|
||||
import { getDatabase } from '../database/connection';
|
||||
import { epgWorkerService } from './epg-worker.service';
|
||||
import { epgSourceGeneration, requestEpgSource } from './epg-source-generation';
|
||||
import { reconcileEpgSources } from './epg-source-settings.service';
|
||||
|
||||
jest.mock('../database/connection', () => ({ getDatabase: jest.fn() }));
|
||||
jest.mock('./epg-worker.service', () => ({
|
||||
epgWorkerService: {
|
||||
clearEpgDataForSource: jest.fn().mockResolvedValue(undefined),
|
||||
sendProgressToRenderer: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
describe('committed EPG source reconciliation', () => {
|
||||
let rows: Map<unknown, unknown[]>;
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
rows = new Map<unknown, unknown[]>([
|
||||
[appState, [{ value: '1' }]],
|
||||
[playlists, []],
|
||||
[epgChannels, [{ url: 'removed' }, { url: 'shared' }]],
|
||||
[epgPrograms, [{ url: 'removed' }, { url: 'second' }]],
|
||||
]);
|
||||
const select = () => ({
|
||||
from: (table: unknown) => {
|
||||
const result = rows.get(table) ?? [];
|
||||
return Object.assign(Promise.resolve(result), {
|
||||
where: () => Promise.resolve(result),
|
||||
});
|
||||
},
|
||||
});
|
||||
(getDatabase as jest.Mock).mockResolvedValue({
|
||||
select,
|
||||
selectDistinct: select,
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves global and every enabled M3U source, including an overlapping channel owner', async () => {
|
||||
rows.set(playlists, [{ type: 'm3u-url', urls: '["shared"]' }]);
|
||||
const generation = epgSourceGeneration('removed');
|
||||
await reconcileEpgSources([' second ']);
|
||||
expect(epgWorkerService.clearEpgDataForSource).toHaveBeenCalledWith(
|
||||
'removed'
|
||||
);
|
||||
expect(epgWorkerService.clearEpgDataForSource).not.toHaveBeenCalledWith(
|
||||
'shared'
|
||||
);
|
||||
expect(epgWorkerService.clearEpgDataForSource).not.toHaveBeenCalledWith(
|
||||
'second'
|
||||
);
|
||||
expect(epgSourceGeneration('removed')).toBeGreaterThan(generation);
|
||||
});
|
||||
|
||||
it('cleans already orphaned sources when the saved global list is empty', async () => {
|
||||
await reconcileEpgSources([]);
|
||||
for (const url of ['removed', 'shared', 'second']) {
|
||||
expect(epgWorkerService.clearEpgDataForSource).toHaveBeenCalledWith(
|
||||
url
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('finds metadata-only source owners after restart', async () => {
|
||||
rows.set(epgChannels, [{ url: 'active' }]);
|
||||
rows.set(epgPrograms, []);
|
||||
rows.set(epgChannelSources, [
|
||||
{ url: 'metadata-only' },
|
||||
{ url: 'active' },
|
||||
]);
|
||||
await reconcileEpgSources(['active']);
|
||||
expect(epgWorkerService.clearEpgDataForSource).toHaveBeenCalledWith(
|
||||
'metadata-only'
|
||||
);
|
||||
expect(epgWorkerService.clearEpgDataForSource).not.toHaveBeenCalledWith(
|
||||
'active'
|
||||
);
|
||||
});
|
||||
|
||||
it('cancels completed error rows even without cached data or a running worker', async () => {
|
||||
rows.set(epgChannels, []);
|
||||
rows.set(epgPrograms, []);
|
||||
const generation = requestEpgSource('failed-import');
|
||||
await reconcileEpgSources([]);
|
||||
expect(epgWorkerService.sendProgressToRenderer).toHaveBeenCalledWith(
|
||||
'failed-import',
|
||||
'cancelled',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
generation
|
||||
);
|
||||
});
|
||||
|
||||
it('cancels every retired row before the first source clear can fail', async () => {
|
||||
const generation = epgSourceGeneration('second');
|
||||
(
|
||||
epgWorkerService.clearEpgDataForSource as jest.Mock
|
||||
).mockImplementationOnce(async () => {
|
||||
expect(
|
||||
epgWorkerService.sendProgressToRenderer
|
||||
).toHaveBeenCalledWith(
|
||||
'second',
|
||||
'cancelled',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
generation
|
||||
);
|
||||
throw new Error('clear failed');
|
||||
});
|
||||
await expect(reconcileEpgSources(['shared'])).rejects.toThrow(
|
||||
'clear failed'
|
||||
);
|
||||
expect(
|
||||
(
|
||||
epgWorkerService.sendProgressToRenderer as jest.Mock
|
||||
).mock.calls.map(([url]) => url)
|
||||
).not.toContain('shared');
|
||||
});
|
||||
|
||||
it('does not clear historical request keys again after successful cleanup', async () => {
|
||||
requestEpgSource('removed');
|
||||
await reconcileEpgSources([]);
|
||||
rows.set(epgChannels, []);
|
||||
rows.set(epgPrograms, []);
|
||||
jest.clearAllMocks();
|
||||
await reconcileEpgSources([]);
|
||||
expect(epgWorkerService.clearEpgDataForSource).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retries a failed cleanup even when the queued source had no database rows', async () => {
|
||||
rows.set(epgChannels, []);
|
||||
rows.set(epgPrograms, []);
|
||||
requestEpgSource('queued-only');
|
||||
(
|
||||
epgWorkerService.clearEpgDataForSource as jest.Mock
|
||||
).mockRejectedValueOnce(new Error('worker failure'));
|
||||
await expect(reconcileEpgSources([])).rejects.toThrow('worker failure');
|
||||
await reconcileEpgSources([]);
|
||||
expect(epgWorkerService.clearEpgDataForSource).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('preserves a new request received while the previous request is being cleared', async () => {
|
||||
rows.set(epgChannels, []);
|
||||
rows.set(epgPrograms, []);
|
||||
requestEpgSource('requested-again');
|
||||
(
|
||||
epgWorkerService.clearEpgDataForSource as jest.Mock
|
||||
).mockImplementationOnce(async () => {
|
||||
requestEpgSource('requested-again');
|
||||
});
|
||||
await reconcileEpgSources([]);
|
||||
await reconcileEpgSources([]);
|
||||
expect(epgWorkerService.clearEpgDataForSource).toHaveBeenCalledTimes(2);
|
||||
jest.clearAllMocks();
|
||||
await reconcileEpgSources([]);
|
||||
expect(epgWorkerService.clearEpgDataForSource).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not prune sources before playlist migration succeeds', async () => {
|
||||
rows.set(appState, []);
|
||||
await expect(reconcileEpgSources([])).rejects.toThrow(
|
||||
'migrated playlists'
|
||||
);
|
||||
expect(epgWorkerService.clearEpgDataForSource).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not guess ownership when persisted playlist source metadata is invalid', async () => {
|
||||
rows.set(playlists, [{ type: 'm3u-url', urls: '{broken' }]);
|
||||
await expect(reconcileEpgSources([])).rejects.toThrow();
|
||||
expect(epgWorkerService.clearEpgDataForSource).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { getDatabase } from '../database/connection';
|
||||
import {
|
||||
appState,
|
||||
epgChannels,
|
||||
epgChannelSources,
|
||||
epgPrograms,
|
||||
playlists,
|
||||
} from '../database/schema';
|
||||
import { epgWorkerService } from './epg-worker.service';
|
||||
import {
|
||||
epgSourceGeneration,
|
||||
forgetEpgSourceRequest,
|
||||
requestedEpgSources,
|
||||
retireEpgSource,
|
||||
} from './epg-source-generation';
|
||||
|
||||
let reconciliation = Promise.resolve();
|
||||
|
||||
/** Called only with settings successfully read from or written to IndexedDB. */
|
||||
export function reconcileEpgSources(globalUrls: string[]): Promise<void> {
|
||||
const next = reconciliation
|
||||
.catch(() => undefined)
|
||||
.then(async () => {
|
||||
const db = await getDatabase();
|
||||
const migration = await db
|
||||
.select()
|
||||
.from(appState)
|
||||
.where(
|
||||
eq(appState.key, 'm3u-playlists-indexeddb-to-sqlite-v1')
|
||||
);
|
||||
// A failed/incomplete migration cannot establish all playlist owners.
|
||||
if (migration[0]?.value !== '1') {
|
||||
throw new Error(
|
||||
'EPG source reconciliation requires migrated playlists'
|
||||
);
|
||||
}
|
||||
const active = new Set(
|
||||
globalUrls.map((url) => url.trim()).filter(Boolean)
|
||||
);
|
||||
const savedPlaylists = await db
|
||||
.select({ type: playlists.type, urls: playlists.epgUrls })
|
||||
.from(playlists);
|
||||
for (const playlist of savedPlaylists) {
|
||||
if (!playlist.type.startsWith('m3u-')) continue;
|
||||
const urls: unknown = JSON.parse(playlist.urls || '[]');
|
||||
if (
|
||||
!Array.isArray(urls) ||
|
||||
urls.some((url) => typeof url !== 'string')
|
||||
) {
|
||||
throw new Error('Invalid saved playlist EPG ownership');
|
||||
}
|
||||
for (const url of urls as string[])
|
||||
if (url.trim()) active.add(url.trim());
|
||||
}
|
||||
const channels = await db
|
||||
.selectDistinct({ url: epgChannels.sourceUrl })
|
||||
.from(epgChannels);
|
||||
const programs = await db
|
||||
.selectDistinct({ url: epgPrograms.sourceUrl })
|
||||
.from(epgPrograms);
|
||||
const metadata = await db
|
||||
.selectDistinct({ url: epgChannelSources.sourceUrl })
|
||||
.from(epgChannelSources);
|
||||
const requested = requestedEpgSources();
|
||||
const removed = [
|
||||
...new Set(
|
||||
[
|
||||
...channels.map((row) => row.url),
|
||||
...programs.map((row) => row.url),
|
||||
...metadata.map((row) => row.url),
|
||||
...requested.keys(),
|
||||
].filter(
|
||||
(url): url is string =>
|
||||
!!url?.trim() && !active.has(url.trim())
|
||||
)
|
||||
),
|
||||
];
|
||||
// Fence the whole obsolete set before awaiting the first worker exit.
|
||||
removed.forEach((url) => {
|
||||
const generation = epgSourceGeneration(url);
|
||||
retireEpgSource(url);
|
||||
// Finished imports can leave actionable error rows with Retry.
|
||||
epgWorkerService.sendProgressToRenderer(
|
||||
url,
|
||||
'cancelled',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
generation
|
||||
);
|
||||
});
|
||||
for (const url of removed) {
|
||||
await epgWorkerService.clearEpgDataForSource(url);
|
||||
forgetEpgSourceRequest(url, requested.get(url.trim()));
|
||||
}
|
||||
});
|
||||
reconciliation = next;
|
||||
return next;
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { epgLogger } from '../util/epg-logger';
|
||||
import { app } from 'electron';
|
||||
import * as path from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { Worker } from 'worker_threads';
|
||||
import { resolveWorkerRuntimeBootstrap } from '../workers/worker-runtime-paths';
|
||||
|
||||
interface ClearWorkerMessage {
|
||||
type: 'CLEAR_EPG' | 'CLEAR_EPG_SOURCE';
|
||||
sourceUrl?: string;
|
||||
}
|
||||
|
||||
/** Worker bootstrap, shutdown and one-shot cache-clear protocol. */
|
||||
export class EpgWorkerRuntime {
|
||||
constructor(
|
||||
private readonly loggerLabel: string,
|
||||
private readonly fetchTimeoutMs: number
|
||||
) {}
|
||||
|
||||
runClearWorker(options: {
|
||||
timeoutLabel: string;
|
||||
exitLabel: string;
|
||||
readyMessage: ClearWorkerMessage;
|
||||
completeWorkerLabel: string;
|
||||
failedWorkerLabel: string;
|
||||
erroredWorkerLabel: string;
|
||||
onComplete: (worker: Worker) => Promise<void>;
|
||||
}): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let worker: Worker;
|
||||
try {
|
||||
worker = this.createEpgWorker();
|
||||
} catch (error) {
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Failed to create worker for clear:',
|
||||
error
|
||||
);
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
let settled = false;
|
||||
const settle = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeoutId);
|
||||
fn();
|
||||
};
|
||||
|
||||
const timeoutId = setTimeout(() => {
|
||||
const errorMessage = `${options.timeoutLabel} timed out after ${
|
||||
this.fetchTimeoutMs / 1000
|
||||
}s`;
|
||||
epgLogger.error(this.loggerLabel, errorMessage);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
`timed out ${options.timeoutLabel}`
|
||||
).then(() => reject(new Error(errorMessage)));
|
||||
});
|
||||
}, this.fetchTimeoutMs);
|
||||
|
||||
worker.on(
|
||||
'message',
|
||||
(message: { type: string; error?: string }) => {
|
||||
if (message.type === 'READY') {
|
||||
worker.postMessage(options.readyMessage);
|
||||
} else if (message.type === 'CLEAR_COMPLETE') {
|
||||
settle(() => {
|
||||
void options
|
||||
.onComplete(worker)
|
||||
.then(() => resolve(), reject);
|
||||
});
|
||||
} else if (message.type === 'EPG_ERROR') {
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Worker clear error:',
|
||||
message.error
|
||||
);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
options.failedWorkerLabel
|
||||
).then(() =>
|
||||
reject(
|
||||
new Error(message.error || 'Clear failed')
|
||||
)
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
worker.on('error', (error) => {
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
'Worker error during clear:',
|
||||
error
|
||||
);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
options.erroredWorkerLabel
|
||||
).then(() => reject(error));
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
if (settled) return;
|
||||
const errorMessage = `${options.exitLabel} exited unexpectedly (code ${code})`;
|
||||
epgLogger.error(this.loggerLabel, errorMessage);
|
||||
settle(() => reject(new Error(errorMessage)));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Awaits worker shutdown so callers can sequence work (e.g. the next DB
|
||||
* access) after the thread has really exited. Termination failures are
|
||||
* logged and swallowed — there is nothing actionable left to do.
|
||||
*/
|
||||
async terminateWorker(worker: Worker, context: string): Promise<void> {
|
||||
try {
|
||||
await worker.terminate();
|
||||
} catch (error) {
|
||||
epgLogger.error(
|
||||
this.loggerLabel,
|
||||
`Failed to terminate ${context} worker:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
createEpgWorker(): Worker {
|
||||
const bootstrap = resolveWorkerRuntimeBootstrap({
|
||||
isPackaged: app.isPackaged,
|
||||
workerFilename: 'epg-parser.worker.js',
|
||||
developmentWorkerDir: path.join(__dirname, 'workers'),
|
||||
resourcesPath: (
|
||||
process as NodeJS.Process & { resourcesPath?: string }
|
||||
).resourcesPath,
|
||||
appPath: app.getAppPath(),
|
||||
});
|
||||
|
||||
const workerURL = pathToFileURL(bootstrap.workerPath);
|
||||
return new Worker(workerURL, {
|
||||
resourceLimits: {
|
||||
maxOldGenerationSizeMb: 4096,
|
||||
maxYoungGenerationSizeMb: 512,
|
||||
},
|
||||
workerData: {
|
||||
nativeModuleSearchPaths: bootstrap.nativeModuleSearchPaths,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,62 @@ describe('EpgWorkerService worker lifecycle', () => {
|
||||
consoleErrorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('keeps source credentials out of completion, reuse, cache and cleanup logs', async () => {
|
||||
const secretUrl =
|
||||
'https://secret-user:secret-password@example.com/private-key/guide.xml?custom=secret-query#secret-fragment';
|
||||
const first = service.fetchEpgFromUrl(secretUrl);
|
||||
const reused = service.fetchEpgFromUrl(secretUrl);
|
||||
mockWorkerInstances[0].emit('message', {
|
||||
type: 'EPG_COMPLETE',
|
||||
stats: { totalChannels: 1, totalPrograms: 2 },
|
||||
});
|
||||
await Promise.all([first, reused]);
|
||||
await service.fetchEpgFromUrl(secretUrl);
|
||||
const clear = service.clearEpgDataForSource(secretUrl);
|
||||
mockWorkerInstances[1].emit('message', { type: 'CLEAR_COMPLETE' });
|
||||
await clear;
|
||||
const output = JSON.stringify(consoleLogSpy.mock.calls);
|
||||
for (const secret of [
|
||||
'secret-user',
|
||||
'secret-password',
|
||||
'private-key',
|
||||
'secret-query',
|
||||
'secret-fragment',
|
||||
]) {
|
||||
expect(output).not.toContain(secret);
|
||||
}
|
||||
expect(output).toContain('EPG parsing complete');
|
||||
});
|
||||
|
||||
it('redacts worker errors without changing the rejected error', async () => {
|
||||
const secretUrl =
|
||||
'https://user:password@example.com/path-token/guide.xml?custom=query-token';
|
||||
const error = Object.assign(new Error(`Request failed: ${secretUrl}`), {
|
||||
config: {
|
||||
url: secretUrl,
|
||||
headers: { Authorization: 'Bearer header-secret' },
|
||||
},
|
||||
cause: new Error(
|
||||
'Redirect failed: https://example.org/redirect-token?custom=redirect-query'
|
||||
),
|
||||
});
|
||||
const pending = service.fetchEpgFromUrl(secretUrl);
|
||||
mockWorkerInstances[0].emit('error', error);
|
||||
await expect(pending).rejects.toBe(error);
|
||||
const output = JSON.stringify(consoleErrorSpy.mock.calls);
|
||||
for (const secret of [
|
||||
'password',
|
||||
'path-token',
|
||||
'query-token',
|
||||
'header-secret',
|
||||
'redirect-token',
|
||||
'redirect-query',
|
||||
]) {
|
||||
expect(output).not.toContain(secret);
|
||||
}
|
||||
expect(output).toContain('Request failed');
|
||||
});
|
||||
|
||||
describe('worker lifecycle', () => {
|
||||
it('spawns the worker with bootstrap paths and drives the FETCH_EPG flow', async () => {
|
||||
const options = { manuallyTrustedHosts: ['example.com'] } as any;
|
||||
|
||||
@@ -1,43 +1,40 @@
|
||||
import { app, BrowserWindow } from 'electron';
|
||||
import * as path from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { epgLogger } from '../util/epg-logger';
|
||||
import {
|
||||
epgSourceGeneration,
|
||||
requestEpgSource,
|
||||
retireEpgSource,
|
||||
} from './epg-source-generation';
|
||||
import { BrowserWindow } from 'electron';
|
||||
import { EpgWorkerRuntime } from './epg-worker-runtime';
|
||||
import { runEpgFetch } from './epg-fetch-operation';
|
||||
import { Worker } from 'worker_threads';
|
||||
import {
|
||||
ElectronBridgeSecurityErrorCode,
|
||||
ElectronBridgeTrustOptions,
|
||||
} from '@iptvnator/shared/interfaces';
|
||||
import { resolveWorkerRuntimeBootstrap } from '../workers/worker-runtime-paths';
|
||||
|
||||
export type EpgProgressStatus = 'queued' | 'loading' | 'complete' | 'error';
|
||||
export type EpgProgressStatus =
|
||||
'queued' | 'loading' | 'complete' | 'error' | 'cancelled';
|
||||
|
||||
export interface EpgProgressStats {
|
||||
totalChannels: number;
|
||||
totalPrograms: number;
|
||||
}
|
||||
|
||||
interface EpgWorkerMessage {
|
||||
type: string;
|
||||
error?: string;
|
||||
errorCode?: ElectronBridgeSecurityErrorCode;
|
||||
errorHost?: string;
|
||||
url?: string;
|
||||
stats?: EpgProgressStats;
|
||||
}
|
||||
|
||||
interface ClearWorkerMessage {
|
||||
type: 'CLEAR_EPG' | 'CLEAR_EPG_SOURCE';
|
||||
sourceUrl?: string;
|
||||
}
|
||||
|
||||
export class EpgWorkerService {
|
||||
private readonly fetchedUrls = new Set<string>();
|
||||
private readonly workers = new Map<string, Worker>();
|
||||
private readonly inFlightFetches = new Map<string, Promise<void>>();
|
||||
private readonly inFlightSourceClears = new Map<string, Promise<void>>();
|
||||
|
||||
private readonly runtime: EpgWorkerRuntime;
|
||||
|
||||
constructor(
|
||||
private readonly loggerLabel = '[EPG Events]',
|
||||
private readonly fetchTimeoutMs = 5 * 60 * 1000
|
||||
) {}
|
||||
) {
|
||||
this.runtime = new EpgWorkerRuntime(loggerLabel, fetchTimeoutMs);
|
||||
}
|
||||
|
||||
hasFetchedUrl(url: string): boolean {
|
||||
return this.fetchedUrls.has(url);
|
||||
@@ -58,12 +55,14 @@ export class EpgWorkerService {
|
||||
error?: string,
|
||||
queuePosition?: number,
|
||||
errorCode?: ElectronBridgeSecurityErrorCode,
|
||||
errorHost?: string
|
||||
errorHost?: string,
|
||||
generation = epgSourceGeneration(url)
|
||||
): void {
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
windows.forEach((win) => {
|
||||
win.webContents.send('EPG_PROGRESS_UPDATE', {
|
||||
url,
|
||||
generation,
|
||||
status,
|
||||
stats,
|
||||
error,
|
||||
@@ -78,6 +77,26 @@ export class EpgWorkerService {
|
||||
url: string,
|
||||
options: ElectronBridgeTrustOptions = {}
|
||||
): Promise<void> {
|
||||
url = url.trim();
|
||||
const generation = requestEpgSource(url);
|
||||
const clear = this.inFlightSourceClears.get(url);
|
||||
if (clear) {
|
||||
await clear.catch(() => undefined);
|
||||
if (generation !== epgSourceGeneration(url)) {
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'cancelled',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
generation
|
||||
);
|
||||
return;
|
||||
}
|
||||
return this.fetchEpgFromUrl(url, options);
|
||||
}
|
||||
// A second request for an URL that is already being fetched must not
|
||||
// spawn a competing worker: both would parse and write the same EPG
|
||||
// data, and the late one would overwrite the early one's entry in
|
||||
@@ -87,17 +106,14 @@ export class EpgWorkerService {
|
||||
// must keep awaiting that termination window.
|
||||
const inFlight = this.inFlightFetches.get(url);
|
||||
if (inFlight) {
|
||||
console.log(
|
||||
this.loggerLabel,
|
||||
`Reusing in-flight EPG fetch: ${url}`
|
||||
);
|
||||
epgLogger.log(this.loggerLabel, 'Reusing in-flight EPG fetch');
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
if (this.fetchedUrls.has(url)) {
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
this.loggerLabel,
|
||||
`Skipping already fetched URL: ${url}`
|
||||
'Skipping already fetched EPG source'
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -113,230 +129,18 @@ export class EpgWorkerService {
|
||||
url: string,
|
||||
options: ElectronBridgeTrustOptions
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let worker: Worker;
|
||||
try {
|
||||
worker = this.createEpgWorker();
|
||||
} catch (error) {
|
||||
console.error(
|
||||
this.loggerLabel,
|
||||
'Failed to create worker:',
|
||||
error
|
||||
);
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
this.workers.set(url, worker);
|
||||
|
||||
// Guards against double-settling and keeps the outer loop moving
|
||||
// when the worker dies or hangs without sending EPG_COMPLETE/EPG_ERROR.
|
||||
let settled = false;
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
let lastProgressStats: EpgProgressStats = {
|
||||
totalChannels: 0,
|
||||
totalPrograms: 0,
|
||||
};
|
||||
|
||||
const clearFetchTimeout = () => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
timeoutId = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const settle = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearFetchTimeout();
|
||||
fn();
|
||||
};
|
||||
|
||||
const scheduleFetchTimeout = () => {
|
||||
clearFetchTimeout();
|
||||
timeoutId = setTimeout(() => {
|
||||
handleFetchTimeout();
|
||||
}, this.fetchTimeoutMs);
|
||||
};
|
||||
|
||||
const hasProgressMoved = (stats: EpgProgressStats): boolean =>
|
||||
stats.totalChannels > lastProgressStats.totalChannels ||
|
||||
stats.totalPrograms > lastProgressStats.totalPrograms;
|
||||
|
||||
const recordProgress = (stats: EpgProgressStats): void => {
|
||||
lastProgressStats = {
|
||||
totalChannels: Math.max(
|
||||
lastProgressStats.totalChannels,
|
||||
stats.totalChannels
|
||||
),
|
||||
totalPrograms: Math.max(
|
||||
lastProgressStats.totalPrograms,
|
||||
stats.totalPrograms
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
const handleFetchTimeout = () => {
|
||||
const errorMessage = `EPG fetch timed out after ${
|
||||
this.fetchTimeoutMs / 1000
|
||||
}s without progress`;
|
||||
console.error(this.loggerLabel, `${errorMessage}: ${url}`);
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
errorMessage
|
||||
);
|
||||
this.workers.delete(url);
|
||||
// Settle only after the worker thread is really gone: a
|
||||
// terminated-but-still-running worker can keep holding the
|
||||
// SQLite lock and block the next EPG fetch.
|
||||
settle(() => {
|
||||
void this.terminateWorker(worker, 'timed out fetch').then(
|
||||
() => reject(new Error(errorMessage))
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
scheduleFetchTimeout();
|
||||
|
||||
worker.on('message', async (message: EpgWorkerMessage) => {
|
||||
try {
|
||||
switch (message.type) {
|
||||
case 'READY':
|
||||
scheduleFetchTimeout();
|
||||
this.sendProgressToRenderer(url, 'loading', {
|
||||
totalChannels: 0,
|
||||
totalPrograms: 0,
|
||||
});
|
||||
worker.postMessage({
|
||||
type: 'FETCH_EPG',
|
||||
url,
|
||||
options,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'EPG_PROGRESS':
|
||||
if (message.stats) {
|
||||
if (hasProgressMoved(message.stats)) {
|
||||
recordProgress(message.stats);
|
||||
scheduleFetchTimeout();
|
||||
}
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'loading',
|
||||
message.stats
|
||||
);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'EPG_COMPLETE':
|
||||
console.log(
|
||||
this.loggerLabel,
|
||||
`EPG parsing complete for ${url}:`,
|
||||
message.stats
|
||||
);
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'complete',
|
||||
message.stats
|
||||
);
|
||||
this.fetchedUrls.add(url);
|
||||
this.workers.delete(url);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
'completed fetch'
|
||||
).then(() => resolve());
|
||||
});
|
||||
break;
|
||||
|
||||
case 'EPG_ERROR':
|
||||
console.error(
|
||||
this.loggerLabel,
|
||||
'Worker error:',
|
||||
message.error
|
||||
);
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
message.error,
|
||||
undefined,
|
||||
message.errorCode,
|
||||
message.errorHost
|
||||
);
|
||||
this.workers.delete(url);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
'failed fetch'
|
||||
).then(() =>
|
||||
reject(
|
||||
new Error(
|
||||
message.error || 'Unknown error'
|
||||
)
|
||||
)
|
||||
);
|
||||
});
|
||||
break;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
this.loggerLabel,
|
||||
'Error handling message:',
|
||||
err
|
||||
);
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
err instanceof Error ? err.message : String(err)
|
||||
);
|
||||
this.workers.delete(url);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
'failed message handling'
|
||||
).then(() => reject(err));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
worker.on('error', (error) => {
|
||||
console.error(this.loggerLabel, 'Worker error event:', error);
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
error.message
|
||||
);
|
||||
this.workers.delete(url);
|
||||
settle(() => {
|
||||
void this.terminateWorker(worker, 'errored fetch').then(
|
||||
() => reject(error)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
if (settled) return;
|
||||
const errorMessage = `Worker exited unexpectedly (code ${code})`;
|
||||
console.error(this.loggerLabel, `${errorMessage}: ${url}`);
|
||||
this.sendProgressToRenderer(
|
||||
url,
|
||||
'error',
|
||||
undefined,
|
||||
errorMessage
|
||||
);
|
||||
this.workers.delete(url);
|
||||
settle(() => reject(new Error(errorMessage)));
|
||||
});
|
||||
return runEpgFetch(url, options, {
|
||||
runtime: this.runtime,
|
||||
workers: this.workers,
|
||||
fetchedUrls: this.fetchedUrls,
|
||||
loggerLabel: this.loggerLabel,
|
||||
fetchTimeoutMs: this.fetchTimeoutMs,
|
||||
sendProgressToRenderer: this.sendProgressToRenderer.bind(this),
|
||||
});
|
||||
}
|
||||
|
||||
async clearEpgData(): Promise<void> {
|
||||
return this.runClearWorker({
|
||||
return this.runtime.runClearWorker({
|
||||
timeoutLabel: 'EPG clear',
|
||||
exitLabel: 'Clear worker',
|
||||
readyMessage: { type: 'CLEAR_EPG' },
|
||||
@@ -344,21 +148,21 @@ export class EpgWorkerService {
|
||||
failedWorkerLabel: 'failed clear',
|
||||
erroredWorkerLabel: 'errored clear',
|
||||
onComplete: async (worker) => {
|
||||
console.log(this.loggerLabel, 'EPG data cleared via worker');
|
||||
epgLogger.log(this.loggerLabel, 'EPG data cleared via worker');
|
||||
this.fetchedUrls.clear();
|
||||
// Resolve only after every interrupted fetch worker has exited
|
||||
// too — they may still hold the SQLite lock the caller expects
|
||||
// to be free.
|
||||
const terminations = [...this.workers.values()].map(
|
||||
(runningWorker) =>
|
||||
this.terminateWorker(
|
||||
this.runtime.terminateWorker(
|
||||
runningWorker,
|
||||
'fetch during clear'
|
||||
)
|
||||
);
|
||||
this.workers.clear();
|
||||
terminations.push(
|
||||
this.terminateWorker(worker, 'completed clear')
|
||||
this.runtime.terminateWorker(worker, 'completed clear')
|
||||
);
|
||||
await Promise.all(terminations);
|
||||
},
|
||||
@@ -371,13 +175,34 @@ export class EpgWorkerService {
|
||||
return;
|
||||
}
|
||||
|
||||
retireEpgSource(normalizedSourceUrl);
|
||||
this.fetchedUrls.delete(normalizedSourceUrl);
|
||||
const previous = this.inFlightSourceClears.get(normalizedSourceUrl);
|
||||
const operation = previous
|
||||
? previous
|
||||
.catch(() => undefined)
|
||||
.then(() => this.startSourceClear(normalizedSourceUrl))
|
||||
: this.startSourceClear(normalizedSourceUrl);
|
||||
const clear = operation.finally(() => {
|
||||
if (this.inFlightSourceClears.get(normalizedSourceUrl) === clear)
|
||||
this.inFlightSourceClears.delete(normalizedSourceUrl);
|
||||
});
|
||||
this.inFlightSourceClears.set(normalizedSourceUrl, clear);
|
||||
return clear;
|
||||
}
|
||||
|
||||
private async startSourceClear(normalizedSourceUrl: string): Promise<void> {
|
||||
const inFlight = this.inFlightFetches.get(normalizedSourceUrl);
|
||||
const runningWorker = this.workers.get(normalizedSourceUrl);
|
||||
if (runningWorker) {
|
||||
this.workers.delete(normalizedSourceUrl);
|
||||
await this.terminateWorker(runningWorker, 'source clear');
|
||||
await this.runtime.terminateWorker(runningWorker, 'source clear');
|
||||
}
|
||||
// Error/timeout handlers may already have removed the worker from the
|
||||
// lookup, but its fetch promise still owns asynchronous termination.
|
||||
if (inFlight) await inFlight.catch(() => undefined);
|
||||
|
||||
return this.runClearWorker({
|
||||
return this.runtime.runClearWorker({
|
||||
timeoutLabel: 'EPG source clear',
|
||||
exitLabel: 'Source clear worker',
|
||||
readyMessage: {
|
||||
@@ -388,153 +213,15 @@ export class EpgWorkerService {
|
||||
failedWorkerLabel: 'failed source clear',
|
||||
erroredWorkerLabel: 'errored source clear',
|
||||
onComplete: async (worker) => {
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
this.loggerLabel,
|
||||
`EPG data cleared for source via worker: ${normalizedSourceUrl}`
|
||||
'EPG data cleared for source via worker'
|
||||
);
|
||||
this.fetchedUrls.delete(normalizedSourceUrl);
|
||||
await this.terminateWorker(worker, 'completed source clear');
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private runClearWorker(options: {
|
||||
timeoutLabel: string;
|
||||
exitLabel: string;
|
||||
readyMessage: ClearWorkerMessage;
|
||||
completeWorkerLabel: string;
|
||||
failedWorkerLabel: string;
|
||||
erroredWorkerLabel: string;
|
||||
onComplete: (worker: Worker) => Promise<void>;
|
||||
}): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let worker: Worker;
|
||||
try {
|
||||
worker = this.createEpgWorker();
|
||||
} catch (error) {
|
||||
console.error(
|
||||
this.loggerLabel,
|
||||
'Failed to create worker for clear:',
|
||||
error
|
||||
await this.runtime.terminateWorker(
|
||||
worker,
|
||||
'completed source clear'
|
||||
);
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
let settled = false;
|
||||
const settle = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeoutId);
|
||||
fn();
|
||||
};
|
||||
|
||||
const timeoutId = setTimeout(() => {
|
||||
const errorMessage = `${options.timeoutLabel} timed out after ${
|
||||
this.fetchTimeoutMs / 1000
|
||||
}s`;
|
||||
console.error(this.loggerLabel, errorMessage);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
`timed out ${options.timeoutLabel}`
|
||||
).then(() => reject(new Error(errorMessage)));
|
||||
});
|
||||
}, this.fetchTimeoutMs);
|
||||
|
||||
worker.on(
|
||||
'message',
|
||||
(message: { type: string; error?: string }) => {
|
||||
if (message.type === 'READY') {
|
||||
worker.postMessage(options.readyMessage);
|
||||
} else if (message.type === 'CLEAR_COMPLETE') {
|
||||
settle(() => {
|
||||
void options
|
||||
.onComplete(worker)
|
||||
.then(() => resolve(), reject);
|
||||
});
|
||||
} else if (message.type === 'EPG_ERROR') {
|
||||
console.error(
|
||||
this.loggerLabel,
|
||||
'Worker clear error:',
|
||||
message.error
|
||||
);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
options.failedWorkerLabel
|
||||
).then(() =>
|
||||
reject(
|
||||
new Error(message.error || 'Clear failed')
|
||||
)
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
worker.on('error', (error) => {
|
||||
console.error(
|
||||
this.loggerLabel,
|
||||
'Worker error during clear:',
|
||||
error
|
||||
);
|
||||
settle(() => {
|
||||
void this.terminateWorker(
|
||||
worker,
|
||||
options.erroredWorkerLabel
|
||||
).then(() => reject(error));
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
if (settled) return;
|
||||
const errorMessage = `${options.exitLabel} exited unexpectedly (code ${code})`;
|
||||
console.error(this.loggerLabel, errorMessage);
|
||||
settle(() => reject(new Error(errorMessage)));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Awaits worker shutdown so callers can sequence work (e.g. the next DB
|
||||
* access) after the thread has really exited. Termination failures are
|
||||
* logged and swallowed — there is nothing actionable left to do.
|
||||
*/
|
||||
private async terminateWorker(
|
||||
worker: Worker,
|
||||
context: string
|
||||
): Promise<void> {
|
||||
try {
|
||||
await worker.terminate();
|
||||
} catch (error) {
|
||||
console.error(
|
||||
this.loggerLabel,
|
||||
`Failed to terminate ${context} worker:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private createEpgWorker(): Worker {
|
||||
const bootstrap = resolveWorkerRuntimeBootstrap({
|
||||
isPackaged: app.isPackaged,
|
||||
workerFilename: 'epg-parser.worker.js',
|
||||
developmentWorkerDir: path.join(__dirname, 'workers'),
|
||||
resourcesPath: (
|
||||
process as NodeJS.Process & { resourcesPath?: string }
|
||||
).resourcesPath,
|
||||
appPath: app.getAppPath(),
|
||||
});
|
||||
|
||||
const workerURL = pathToFileURL(bootstrap.workerPath);
|
||||
return new Worker(workerURL, {
|
||||
resourceLimits: {
|
||||
maxOldGenerationSizeMb: 4096,
|
||||
maxYoungGenerationSizeMb: 512,
|
||||
},
|
||||
workerData: {
|
||||
nativeModuleSearchPaths: bootstrap.nativeModuleSearchPaths,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -78,6 +78,37 @@ describe('EpgEvents', () => {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
it.each(['missing', 'stale', 'fresh'])(
|
||||
'uses source-owned freshness for %s metadata',
|
||||
async (state) => {
|
||||
const { epgChannelSources } = await import('../database/schema');
|
||||
const { checkEpgFreshness } = await import('./epg-fetch.service');
|
||||
const fresh = [{ updatedAt: new Date().toISOString() }];
|
||||
const snapshots =
|
||||
state === 'missing'
|
||||
? []
|
||||
: state === 'fresh'
|
||||
? fresh
|
||||
: [{ updatedAt: '2000-01-01T00:00:00Z' }];
|
||||
getDatabase.mockResolvedValue({
|
||||
select: () => ({
|
||||
from: (table: unknown) => ({
|
||||
where: () => ({
|
||||
limit: async () =>
|
||||
table === epgChannelSources ? snapshots : fresh,
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
const result = await checkEpgFreshness(['shared-source'], 12);
|
||||
expect(result).toEqual(
|
||||
state === 'fresh'
|
||||
? { freshUrls: ['shared-source'], staleUrls: [] }
|
||||
: { freshUrls: [], staleUrls: ['shared-source'] }
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
it('uses the shared worker bootstrap and passes native module search paths to the EPG worker', async () => {
|
||||
const fetchPromise = (EpgEvents as unknown as Record<string, any>)[
|
||||
'fetchEpgFromUrl'
|
||||
@@ -287,6 +318,120 @@ describe('EpgEvents', () => {
|
||||
await flushPromises();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['exit', 1],
|
||||
['error', new Error('terminated worker')],
|
||||
] as const)(
|
||||
'cancels a retired source on %s without reviving it from late READY or COMPLETE messages',
|
||||
async (event, payload) => {
|
||||
const service = new EpgWorkerService('[Test EPG]', 1000);
|
||||
const url = 'https://removed.example/guide.xml';
|
||||
const progress = jest.spyOn(service, 'sendProgressToRenderer');
|
||||
const fetch = service.fetchEpgFromUrl(url).then(
|
||||
() => true,
|
||||
() => false
|
||||
);
|
||||
const worker = mockWorkerInstances[0];
|
||||
let finishTermination!: () => void;
|
||||
worker.terminate.mockReturnValue(
|
||||
new Promise<void>((resolve) => {
|
||||
finishTermination = resolve;
|
||||
})
|
||||
);
|
||||
const clear = service.clearEpgDataForSource(url);
|
||||
worker.emit('message', { type: 'READY' });
|
||||
worker.emit('message', { type: 'EPG_COMPLETE' });
|
||||
expect(worker.postMessage).not.toHaveBeenCalled();
|
||||
expect(service.hasFetchedUrl(url)).toBe(false);
|
||||
expect(mockWorkerInstances).toHaveLength(1);
|
||||
worker.emit(event, payload);
|
||||
finishTermination();
|
||||
await flushPromises();
|
||||
const clearWorker = mockWorkerInstances[1];
|
||||
clearWorker.emit('message', { type: 'READY' });
|
||||
clearWorker.emit('message', { type: 'CLEAR_COMPLETE' });
|
||||
await clear;
|
||||
expect(await fetch).toBe(true);
|
||||
expect(progress.mock.calls.map((call) => call[1])).toEqual([
|
||||
'cancelled',
|
||||
]);
|
||||
}
|
||||
);
|
||||
|
||||
it('does not start a queued source removed while an earlier source imports', async () => {
|
||||
getDatabase.mockRejectedValue(new Error('force stale for test'));
|
||||
const { handleFetchEpg } = await import('./epg-fetch.service');
|
||||
const { retireEpgSource } = await import('./epg-source-generation');
|
||||
const { epgWorkerService } = await import('./epg-worker.service');
|
||||
const progress = jest.spyOn(epgWorkerService, 'sendProgressToRenderer');
|
||||
let finishFirst!: () => void;
|
||||
const fetch = jest
|
||||
.spyOn(epgWorkerService, 'fetchEpgFromUrl')
|
||||
.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
finishFirst = resolve;
|
||||
})
|
||||
)
|
||||
.mockResolvedValue(undefined);
|
||||
const request = handleFetchEpg([
|
||||
'https://first.example/guide.xml',
|
||||
'https://queued.example/guide.xml',
|
||||
]);
|
||||
await flushPromises();
|
||||
retireEpgSource('https://queued.example/guide.xml');
|
||||
finishFirst();
|
||||
await request;
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(progress).toHaveBeenCalledWith(
|
||||
'https://queued.example/guide.xml',
|
||||
'cancelled',
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
expect.any(Number)
|
||||
);
|
||||
progress.mockRestore();
|
||||
fetch.mockRestore();
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'awaits a failing worker termination before source cleanup (already retired: %s)',
|
||||
async (alreadyRetired) => {
|
||||
const service = new EpgWorkerService('[Test EPG]', 1000);
|
||||
const url = `https://terminating-${alreadyRetired}.example/guide.xml`;
|
||||
const fetch = service.fetchEpgFromUrl(url).catch(() => undefined);
|
||||
const worker = mockWorkerInstances[0];
|
||||
let finishTermination!: () => void;
|
||||
worker.terminate.mockReturnValue(
|
||||
new Promise<void>((resolve) => {
|
||||
finishTermination = resolve;
|
||||
})
|
||||
);
|
||||
if (alreadyRetired) {
|
||||
const { retireEpgSource } =
|
||||
await import('./epg-source-generation');
|
||||
retireEpgSource(url);
|
||||
}
|
||||
worker.emit(
|
||||
'error',
|
||||
new Error('worker failed while another source clears')
|
||||
);
|
||||
const clear = service.clearEpgDataForSource(url);
|
||||
const workersBeforeTermination = mockWorkerInstances.length;
|
||||
finishTermination();
|
||||
await fetch;
|
||||
await flushPromises();
|
||||
const clearWorker = mockWorkerInstances[1];
|
||||
clearWorker.emit('message', { type: 'READY' });
|
||||
clearWorker.emit('message', { type: 'CLEAR_COMPLETE' });
|
||||
await clear;
|
||||
expect(workersBeforeTermination).toBe(1);
|
||||
}
|
||||
);
|
||||
|
||||
it('clears one EPG source through a worker and allows it to be fetched again', async () => {
|
||||
const workerService = new EpgWorkerService('[Test EPG]', 1000);
|
||||
const sourceUrl = 'https://playlist.example.com/guide.xml';
|
||||
@@ -336,6 +481,66 @@ describe('EpgEvents', () => {
|
||||
expect(terminated).toBe(true);
|
||||
});
|
||||
|
||||
it('waits for source cleanup before starting a replacement import of the same normalized URL', async () => {
|
||||
const service = new EpgWorkerService('[Test EPG]', 1000);
|
||||
const url = 'https://replacement.example/guide.xml';
|
||||
const clear = service.clearEpgDataForSource(url);
|
||||
const clearWorker = mockWorkerInstances[0];
|
||||
const replacement = service.fetchEpgFromUrl(` ${url} `);
|
||||
const workersBeforeCleanup = mockWorkerInstances.length;
|
||||
clearWorker.emit('message', { type: 'READY' });
|
||||
clearWorker.emit('message', { type: 'CLEAR_COMPLETE' });
|
||||
await clear;
|
||||
await flushPromises();
|
||||
const fetchWorker = mockWorkerInstances[1];
|
||||
fetchWorker.emit('message', { type: 'READY' });
|
||||
fetchWorker.emit('message', { type: 'EPG_COMPLETE' });
|
||||
await replacement;
|
||||
expect(workersBeforeCleanup).toBe(1);
|
||||
expect(service.hasFetchedUrl(url)).toBe(true);
|
||||
});
|
||||
|
||||
it('serializes repeated clears and retires a replacement waiting for the earlier clear', async () => {
|
||||
const service = new EpgWorkerService('[Test EPG]', 1000);
|
||||
const url = 'https://twice-removed.example/guide.xml';
|
||||
const firstClear = service.clearEpgDataForSource(url);
|
||||
const waitingFetch = service.fetchEpgFromUrl(url);
|
||||
const secondClear = service.clearEpgDataForSource(url);
|
||||
expect(mockWorkerInstances).toHaveLength(1);
|
||||
mockWorkerInstances[0].emit('message', { type: 'CLEAR_COMPLETE' });
|
||||
await firstClear;
|
||||
await waitingFetch;
|
||||
await flushPromises();
|
||||
expect(mockWorkerInstances).toHaveLength(2);
|
||||
mockWorkerInstances[1].emit('message', { type: 'READY' });
|
||||
expect(mockWorkerInstances[1].postMessage).toHaveBeenCalledWith({
|
||||
type: 'CLEAR_EPG_SOURCE',
|
||||
sourceUrl: url,
|
||||
});
|
||||
mockWorkerInstances[1].emit('message', { type: 'CLEAR_COMPLETE' });
|
||||
await secondClear;
|
||||
expect(service.hasFetchedUrl(url)).toBe(false);
|
||||
});
|
||||
|
||||
it('allows a replacement import after a failed source cleanup has terminated', async () => {
|
||||
const service = new EpgWorkerService('[Test EPG]', 1000);
|
||||
const url = 'https://retry-clear.example/guide.xml';
|
||||
const clear = service.clearEpgDataForSource(url);
|
||||
const outcome = clear.catch((error: Error) => error.message);
|
||||
const replacement = service.fetchEpgFromUrl(url);
|
||||
mockWorkerInstances[0].emit('message', {
|
||||
type: 'EPG_ERROR',
|
||||
error: 'clear failed',
|
||||
});
|
||||
expect(await outcome).toBe('clear failed');
|
||||
await flushPromises();
|
||||
expect(mockWorkerInstances[0].terminate).toHaveBeenCalled();
|
||||
mockWorkerInstances[1].emit('message', { type: 'READY' });
|
||||
mockWorkerInstances[1].emit('message', { type: 'EPG_COMPLETE' });
|
||||
await replacement;
|
||||
expect(service.hasFetchedUrl(url)).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps an active EPG fetch alive when worker progress keeps moving', async () => {
|
||||
jest.useFakeTimers();
|
||||
|
||||
@@ -381,7 +586,10 @@ describe('EpgEvents', () => {
|
||||
* inserts extra queries that the original test didn't anticipate.
|
||||
*/
|
||||
function queryChain<T>(data: T) {
|
||||
const chain: Record<string, jest.Mock> = {} as Record<string, jest.Mock>;
|
||||
const chain: Record<string, jest.Mock> = {} as Record<
|
||||
string,
|
||||
jest.Mock
|
||||
>;
|
||||
chain.where = jest.fn().mockReturnValue(chain);
|
||||
chain.innerJoin = jest.fn().mockReturnValue(chain);
|
||||
chain.groupBy = jest.fn().mockReturnValue(chain);
|
||||
@@ -393,25 +601,30 @@ describe('EpgEvents', () => {
|
||||
// getMapping queries (must come first — they return empty)
|
||||
const from = jest
|
||||
.fn()
|
||||
.mockReturnValueOnce(queryChain([])) // 1. getMapping: epgChannelMappings
|
||||
.mockReturnValueOnce(queryChain([])) // 2. getMapping: content table
|
||||
.mockReturnValueOnce(queryChain([])) // 1. getMapping: epgChannelMappings
|
||||
.mockReturnValueOnce(queryChain([])) // 2. getMapping: content table
|
||||
// Test expectations below
|
||||
.mockReturnValueOnce(queryChain([])) // 3. selectChannelPrograms (bbc.one.uk → empty)
|
||||
.mockReturnValueOnce(queryChain([{ id: 'BBC.ONE.UK', displayName: 'BBC One' }])) // 4. selectChannelById → channel found
|
||||
.mockReturnValueOnce(queryChain([ // 5. selectChannelPrograms (BBC.ONE.UK → programs)
|
||||
{
|
||||
id: 1,
|
||||
channelId: 'BBC.ONE.UK',
|
||||
start: '2026-04-14T10:00:00Z',
|
||||
stop: '2026-04-14T11:00:00Z',
|
||||
title: 'News',
|
||||
description: null,
|
||||
category: null,
|
||||
iconUrl: null,
|
||||
rating: null,
|
||||
episodeNum: null,
|
||||
},
|
||||
]));
|
||||
.mockReturnValueOnce(queryChain([])) // 3. selectChannelPrograms (bbc.one.uk → empty)
|
||||
.mockReturnValueOnce(
|
||||
queryChain([{ id: 'BBC.ONE.UK', displayName: 'BBC One' }])
|
||||
) // 4. selectChannelById → channel found
|
||||
.mockReturnValueOnce(
|
||||
queryChain([
|
||||
// 5. selectChannelPrograms (BBC.ONE.UK → programs)
|
||||
{
|
||||
id: 1,
|
||||
channelId: 'BBC.ONE.UK',
|
||||
start: '2026-04-14T10:00:00Z',
|
||||
stop: '2026-04-14T11:00:00Z',
|
||||
title: 'News',
|
||||
description: null,
|
||||
category: null,
|
||||
iconUrl: null,
|
||||
rating: null,
|
||||
episodeNum: null,
|
||||
},
|
||||
])
|
||||
);
|
||||
|
||||
select.mockImplementation(() => ({ from }));
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { reconcileEpgSources } from './epg-source-settings.service';
|
||||
import { ipcMain } from 'electron';
|
||||
import {
|
||||
ElectronBridgeCurrentProgramsOptions,
|
||||
@@ -29,6 +30,14 @@ export default class EpgEvents {
|
||||
* Bootstrap EPG events
|
||||
*/
|
||||
static bootstrapEpgEvents(): Electron.IpcMain {
|
||||
ipcMain.handle(
|
||||
'EPG_RECONCILE_SOURCES',
|
||||
async (_event, args: { urls: string[] }) => {
|
||||
await reconcileEpgSources(args.urls);
|
||||
return { success: true };
|
||||
}
|
||||
);
|
||||
|
||||
ipcMain.handle(
|
||||
'FETCH_EPG',
|
||||
async (
|
||||
@@ -179,10 +188,7 @@ export default class EpgEvents {
|
||||
|
||||
ipcMain.handle(
|
||||
'EPG_CHANNEL_SEARCH',
|
||||
async (
|
||||
_event,
|
||||
args: { searchTerm: string; limit?: number }
|
||||
) => {
|
||||
async (_event, args: { searchTerm: string; limit?: number }) => {
|
||||
return handleSearchEpgChannels(args.searchTerm, args.limit);
|
||||
}
|
||||
);
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { epgLogger } from './epg-logger';
|
||||
|
||||
describe('EPG diagnostics', () => {
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('omits URLs with arbitrary path/query secrets, redirects and malformed authorities', () => {
|
||||
const log = jest.spyOn(console, 'log').mockImplementation();
|
||||
epgLogger.log('[EPG Worker]', {
|
||||
message:
|
||||
'Redirect: https://example.com/path-secret?custom=query-secret -> https://user:password@[/malformed-secret',
|
||||
response: { url: new URL('https://example.org/redirect-secret') },
|
||||
authorization: 'Bearer header-secret',
|
||||
stats: { totalChannels: 12, totalPrograms: 34 },
|
||||
});
|
||||
const output = JSON.stringify(log.mock.calls);
|
||||
for (const secret of [
|
||||
'path-secret',
|
||||
'query-secret',
|
||||
'password',
|
||||
'malformed-secret',
|
||||
'redirect-secret',
|
||||
'header-secret',
|
||||
]) {
|
||||
expect(output).not.toContain(secret);
|
||||
}
|
||||
expect(output).toContain('Redirect:');
|
||||
expect(log.mock.calls[0][1]).toEqual(
|
||||
expect.objectContaining({
|
||||
stats: { totalChannels: 12, totalPrograms: 34 },
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('omits transport data and stacks while retaining safe error details without mutation', () => {
|
||||
const log = jest.spyOn(console, 'error').mockImplementation();
|
||||
const error = Object.assign(
|
||||
new Error('Failed https://example.org/path-secret'),
|
||||
{
|
||||
code: 'ECONNRESET',
|
||||
cause: new Error(
|
||||
'Redirect https://example.org/redirect-secret'
|
||||
),
|
||||
request: {
|
||||
path: '/feed/relative-path-secret?custom=relative-query-secret',
|
||||
_header:
|
||||
'GET /feed/header-path-secret?custom=header-query-secret HTTP/1.1',
|
||||
} as { self?: unknown; path: string; _header: string },
|
||||
}
|
||||
);
|
||||
error.request.self = error;
|
||||
epgLogger.error('[EPG Worker]', error);
|
||||
const output = JSON.stringify(log.mock.calls);
|
||||
expect(output).not.toContain('path-secret');
|
||||
expect(output).not.toContain('redirect-secret');
|
||||
expect(output).toContain('ECONNRESET');
|
||||
for (const secret of [
|
||||
'relative-path-secret',
|
||||
'relative-query-secret',
|
||||
'header-path-secret',
|
||||
'header-query-secret',
|
||||
]) {
|
||||
expect(output).not.toContain(secret);
|
||||
}
|
||||
expect(log.mock.calls[0][1]).not.toHaveProperty('request');
|
||||
expect(log.mock.calls[0][1]).not.toHaveProperty('stack');
|
||||
expect(error.message).toBe('Failed https://example.org/path-secret');
|
||||
expect(error.request.self).toBe(error);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { redactSensitiveData, REDACTED_VALUE } from '@iptvnator/shared/logging';
|
||||
|
||||
/** Shared redaction produces a bounded, acyclic plain-data tree. XMLTV feeds
|
||||
* can also put opaque secrets in arbitrary URL paths/query keys, so omit URLs
|
||||
* entirely from diagnostics (including redirects and malformed error URLs).
|
||||
* IPC progress and rejected errors keep their original values for callers.
|
||||
*/
|
||||
function omitUrls(value: unknown): unknown {
|
||||
if (typeof value === 'string') {
|
||||
return value.replace(
|
||||
/[a-z][a-z0-9+.-]*:\/\/[^\s"'<>]+/giu,
|
||||
REDACTED_VALUE
|
||||
);
|
||||
}
|
||||
if (Array.isArray(value)) return value.map(omitUrls);
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, entry]) => [
|
||||
omitUrls(key),
|
||||
omitUrls(entry),
|
||||
])
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
// Axios/Node transport objects also contain relative request paths and raw
|
||||
// HTTP headers without a URL scheme. Never forward them or arbitrary error
|
||||
// fields to the logger, even after generic redaction.
|
||||
function summarizeError(value: unknown): unknown {
|
||||
if (!value || typeof value !== 'object') return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value)
|
||||
.filter(([key]) =>
|
||||
[
|
||||
'name',
|
||||
'message',
|
||||
'code',
|
||||
'status',
|
||||
'statusCode',
|
||||
'cause',
|
||||
].includes(key)
|
||||
)
|
||||
.map(([key, entry]) => [key, summarizeError(entry)])
|
||||
);
|
||||
}
|
||||
|
||||
export const epgLogger = {
|
||||
log(...args: unknown[]): void {
|
||||
console.log(
|
||||
...args.map((value) => omitUrls(redactSensitiveData(value)))
|
||||
);
|
||||
},
|
||||
error(...args: unknown[]): void {
|
||||
console.error(
|
||||
...args.map((value) =>
|
||||
omitUrls(summarizeError(redactSensitiveData(value)))
|
||||
)
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -93,6 +93,10 @@ describe('EpgDatabase', () => {
|
||||
FROM epg_programs
|
||||
WHERE epg_programs.channel_id = epg_channels.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM epg_channel_sources
|
||||
WHERE epg_channel_sources.channel_id = epg_channels.id
|
||||
)
|
||||
`);
|
||||
|
||||
expect(preparedSql).toContain(deleteTodayAndFutureSql);
|
||||
@@ -230,6 +234,7 @@ describe('EpgDatabaseClearOperation', () => {
|
||||
expect(exec.mock.calls.map(([statement]) => statement)).toEqual([
|
||||
'BEGIN',
|
||||
'DELETE FROM epg_programs',
|
||||
'DELETE FROM epg_channel_sources',
|
||||
'DELETE FROM epg_channels',
|
||||
'COMMIT',
|
||||
]);
|
||||
@@ -250,6 +255,7 @@ describe('EpgDatabaseClearOperation', () => {
|
||||
expect(exec.mock.calls.map(([statement]) => statement)).toEqual([
|
||||
'BEGIN',
|
||||
'DELETE FROM epg_programs',
|
||||
'DELETE FROM epg_channel_sources',
|
||||
'DELETE FROM epg_channels',
|
||||
'ROLLBACK',
|
||||
]);
|
||||
@@ -276,6 +282,10 @@ describe('EpgDatabaseSourceClearOperation', () => {
|
||||
FROM epg_programs
|
||||
WHERE epg_programs.channel_id = epg_channels.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM epg_channel_sources
|
||||
WHERE epg_channel_sources.channel_id = epg_channels.id
|
||||
)
|
||||
`);
|
||||
|
||||
expect(preparedSql).toContain(deleteProgramsSql);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type BetterSqlite3 from 'better-sqlite3';
|
||||
import { getIptvnatorDatabasePath } from '@iptvnator/shared/database/path-utils';
|
||||
import type { ParsedChannel, ParsedProgram } from './epg-streaming-parser';
|
||||
import { restoreSurvivingChannelMetadata } from './epg-source-metadata';
|
||||
|
||||
/**
|
||||
* Database helper for worker-owned EPG operations.
|
||||
@@ -10,6 +11,8 @@ export class EpgDatabase {
|
||||
private readonly db: BetterSqlite3.Database;
|
||||
private readonly knownChannelIds = new Set<string>();
|
||||
private readonly insertChannelStmt: BetterSqlite3.Statement;
|
||||
private readonly insertChannelSourceStmt: BetterSqlite3.Statement;
|
||||
private readonly deleteChannelSourcesStmt: BetterSqlite3.Statement;
|
||||
private readonly insertProgramStmt: BetterSqlite3.Statement;
|
||||
private readonly deleteOrphanChannelsForSourceStmt: BetterSqlite3.Statement;
|
||||
private readonly deleteTodayAndFutureStmt: BetterSqlite3.Statement;
|
||||
@@ -30,6 +33,25 @@ export class EpgDatabase {
|
||||
updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
|
||||
`);
|
||||
|
||||
// Allocated under the same SQLite write transaction as the channel
|
||||
// upsert: ties and clock rollbacks cannot reorder surviving writers.
|
||||
this.insertChannelSourceStmt = this.db.prepare(`
|
||||
INSERT INTO epg_channel_sources
|
||||
(channel_id, display_name, icon_url, url, source_url, updated_at, write_order)
|
||||
VALUES (?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%SZ', 'now'),
|
||||
(SELECT COALESCE(MAX(write_order), 0) + 1
|
||||
FROM epg_channel_sources WHERE channel_id = ?))
|
||||
ON CONFLICT(channel_id, source_url) DO UPDATE SET
|
||||
display_name = excluded.display_name,
|
||||
icon_url = excluded.icon_url,
|
||||
url = excluded.url,
|
||||
updated_at = excluded.updated_at,
|
||||
write_order = excluded.write_order
|
||||
`);
|
||||
this.deleteChannelSourcesStmt = this.db.prepare(
|
||||
'DELETE FROM epg_channel_sources WHERE source_url = ?'
|
||||
);
|
||||
|
||||
// Guard against duplicate entries when the clearFirst logic misses old
|
||||
// rows. The same channel + start + title + source is treated as the
|
||||
// same programme — a later import with a corrected stop time simply
|
||||
@@ -66,6 +88,10 @@ export class EpgDatabase {
|
||||
FROM epg_programs
|
||||
WHERE epg_programs.channel_id = epg_channels.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM epg_channel_sources
|
||||
WHERE epg_channel_sources.channel_id = epg_channels.id
|
||||
)
|
||||
`);
|
||||
|
||||
this.deleteTodayAndFutureStmt = this.db.prepare(`
|
||||
@@ -88,7 +114,9 @@ export class EpgDatabase {
|
||||
): void {
|
||||
const insertMany = this.db.transaction((channels: ParsedChannel[]) => {
|
||||
if (clearTodayAndFuture) {
|
||||
restoreSurvivingChannelMetadata(this.db, sourceUrl);
|
||||
this.deleteTodayAndFutureStmt.run(sourceUrl);
|
||||
this.deleteChannelSourcesStmt.run(sourceUrl);
|
||||
this.deleteOrphanChannelsForSourceStmt.run(sourceUrl);
|
||||
this.knownChannelIds.clear();
|
||||
}
|
||||
@@ -106,6 +134,14 @@ export class EpgDatabase {
|
||||
url,
|
||||
sourceUrl
|
||||
);
|
||||
this.insertChannelSourceStmt.run(
|
||||
channel.id,
|
||||
displayName,
|
||||
iconUrl,
|
||||
url,
|
||||
sourceUrl,
|
||||
channel.id
|
||||
);
|
||||
this.knownChannelIds.add(channel.id);
|
||||
}
|
||||
});
|
||||
@@ -222,6 +258,7 @@ export class EpgDatabaseClearOperation {
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
this.db.exec('DELETE FROM epg_programs');
|
||||
this.db.exec('DELETE FROM epg_channel_sources');
|
||||
this.db.exec('DELETE FROM epg_channels');
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
@@ -256,6 +293,10 @@ export class EpgDatabaseSourceClearOperation {
|
||||
FROM epg_programs
|
||||
WHERE epg_programs.channel_id = epg_channels.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM epg_channel_sources
|
||||
WHERE epg_channel_sources.channel_id = epg_channels.id
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
@@ -266,7 +307,12 @@ export class EpgDatabaseSourceClearOperation {
|
||||
}
|
||||
|
||||
const clearSource = this.db.transaction((url: string) => {
|
||||
// Capture affected IDs while removed-source provenance still exists.
|
||||
restoreSurvivingChannelMetadata(this.db, url);
|
||||
this.deleteProgramsForSourceStmt.run(url);
|
||||
this.db
|
||||
.prepare('DELETE FROM epg_channel_sources WHERE source_url = ?')
|
||||
.run(url);
|
||||
this.deleteOrphanChannelsForSourceStmt.run(url);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { epgLogger } from '../util/epg-logger';
|
||||
import type BetterSqlite3 from 'better-sqlite3';
|
||||
import {
|
||||
ELECTRON_BRIDGE_SECURITY_ERROR_CODES,
|
||||
@@ -102,7 +103,7 @@ async function fetchAndParseEpgStreaming(
|
||||
url: string,
|
||||
options: ElectronBridgeTrustOptions = {}
|
||||
): Promise<void> {
|
||||
console.log(loggerLabel, `Fetching EPG from ${url}`);
|
||||
epgLogger.log(loggerLabel, 'Fetching EPG source');
|
||||
|
||||
// Create database connection in worker
|
||||
const epgDb = new EpgDatabase(Database);
|
||||
@@ -141,18 +142,15 @@ async function fetchAndParseEpgStreaming(
|
||||
const contentEncoding = getEpgResponseContentEncoding(response.headers);
|
||||
|
||||
if (responseUrl && responseUrl !== url) {
|
||||
console.log(
|
||||
loggerLabel,
|
||||
`Resolved EPG redirect: ${url} -> ${responseUrl}`
|
||||
);
|
||||
epgLogger.log(loggerLabel, 'Resolved EPG redirect');
|
||||
}
|
||||
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
loggerLabel,
|
||||
`EPG response detected as gzipped: ${isGzipped}`
|
||||
);
|
||||
if (contentEncoding) {
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
loggerLabel,
|
||||
`EPG response content-encoding: ${contentEncoding}`
|
||||
);
|
||||
@@ -200,7 +198,7 @@ async function fetchAndParseEpgStreaming(
|
||||
try {
|
||||
parser.write(chunk.toString('utf-8'));
|
||||
} catch (err) {
|
||||
console.error(loggerLabel, 'Parse error:', err);
|
||||
epgLogger.error(loggerLabel, 'Parse error:', err);
|
||||
epgDb.close();
|
||||
reject(err);
|
||||
}
|
||||
@@ -209,7 +207,7 @@ async function fetchAndParseEpgStreaming(
|
||||
dataStream.on('end', () => {
|
||||
try {
|
||||
const stats = parser.finish();
|
||||
console.log(
|
||||
epgLogger.log(
|
||||
loggerLabel,
|
||||
`Parsing complete: ${stats.totalChannels} channels, ${stats.totalPrograms} programs`
|
||||
);
|
||||
@@ -223,7 +221,7 @@ async function fetchAndParseEpgStreaming(
|
||||
// real problem (unreachable feed, SAX parse failure, etc.).
|
||||
if (stats.totalChannels === 0) {
|
||||
const errorMessage = `EPG parse produced 0 channels — feed may be unreachable or unsupported`;
|
||||
console.error(loggerLabel, `${errorMessage}: ${url}`);
|
||||
epgLogger.error(loggerLabel, errorMessage);
|
||||
const response: WorkerResponse = {
|
||||
type: 'EPG_ERROR',
|
||||
url,
|
||||
@@ -251,7 +249,7 @@ async function fetchAndParseEpgStreaming(
|
||||
});
|
||||
|
||||
dataStream.on('error', (err) => {
|
||||
console.error(loggerLabel, 'Stream error:', err);
|
||||
epgLogger.error(loggerLabel, 'Stream error:', err);
|
||||
epgDb.close();
|
||||
reject(err);
|
||||
});
|
||||
@@ -315,16 +313,16 @@ function clearAllEpgData(): void {
|
||||
const clearOperation = new EpgDatabaseClearOperation(Database);
|
||||
|
||||
try {
|
||||
console.log(loggerLabel, 'Clearing all EPG data...');
|
||||
epgLogger.log(loggerLabel, 'Clearing all EPG data...');
|
||||
|
||||
clearOperation.run();
|
||||
|
||||
console.log(loggerLabel, 'All EPG data cleared');
|
||||
epgLogger.log(loggerLabel, 'All EPG data cleared');
|
||||
|
||||
const response: WorkerResponse = { type: 'CLEAR_COMPLETE' };
|
||||
parentPort?.postMessage(response);
|
||||
} catch (error) {
|
||||
console.error(loggerLabel, 'Error clearing EPG data:', error);
|
||||
epgLogger.error(loggerLabel, 'Error clearing EPG data:', error);
|
||||
const errorResponse: WorkerResponse = {
|
||||
type: 'EPG_ERROR',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
@@ -339,19 +337,16 @@ function clearEpgDataForSource(sourceUrl: string): void {
|
||||
const clearOperation = new EpgDatabaseSourceClearOperation(Database);
|
||||
|
||||
try {
|
||||
console.log(
|
||||
loggerLabel,
|
||||
`Clearing EPG data for source ${sourceUrl}...`
|
||||
);
|
||||
epgLogger.log(loggerLabel, 'Clearing EPG data for source...');
|
||||
|
||||
clearOperation.run(sourceUrl);
|
||||
|
||||
console.log(loggerLabel, `EPG data cleared for source ${sourceUrl}`);
|
||||
epgLogger.log(loggerLabel, 'EPG data cleared for source');
|
||||
|
||||
const response: WorkerResponse = { type: 'CLEAR_COMPLETE' };
|
||||
parentPort?.postMessage(response);
|
||||
} catch (error) {
|
||||
console.error(loggerLabel, 'Error clearing EPG source data:', error);
|
||||
epgLogger.error(loggerLabel, 'Error clearing EPG source data:', error);
|
||||
const errorResponse: WorkerResponse = {
|
||||
type: 'EPG_ERROR',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
@@ -379,7 +374,7 @@ if (parentPort) {
|
||||
clearEpgDataForSource(message.sourceUrl ?? '');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(loggerLabel, 'Worker error:', error);
|
||||
epgLogger.error(loggerLabel, 'Worker error:', error);
|
||||
const typedError = error as {
|
||||
code?: ElectronBridgeSecurityErrorCode;
|
||||
host?: string;
|
||||
@@ -398,5 +393,5 @@ if (parentPort) {
|
||||
// Notify parent that worker is ready
|
||||
parentPort.postMessage({ type: 'READY' });
|
||||
} else {
|
||||
console.error(loggerLabel, 'parentPort is not available!');
|
||||
epgLogger.error(loggerLabel, 'parentPort is not available!');
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createRequire } from 'node:module';
|
||||
import { resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
/** Exercise worker SQL with Electron's native SQLite binding, including upgrades. */
|
||||
function runDatabaseScenario(scenario: string): unknown {
|
||||
const moduleUrl = pathToFileURL(resolve(__dirname, 'epg-database.ts')).href;
|
||||
const connectionUrl = pathToFileURL(
|
||||
resolve(process.cwd(), 'libs/shared/database/src/lib/connection.ts')
|
||||
).href;
|
||||
const script = `
|
||||
const { default: Database } = await import('better-sqlite3');
|
||||
const { EpgDatabase, EpgDatabaseSourceClearOperation, EpgDatabaseClearOperation }
|
||||
= await import(${JSON.stringify(moduleUrl)});
|
||||
const { __databaseConnectionTestHooks } = await import(${JSON.stringify(connectionUrl)});
|
||||
const sqlite = new Database(':memory:');
|
||||
const statements = __databaseConnectionTestHooks.createTableStatements
|
||||
.filter(sql => /(?:TABLE|INDEX|TRIGGER) IF NOT EXISTS (?:idx_)?epg_/.test(sql));
|
||||
for (const sql of statements) sqlite.exec(sql);
|
||||
class SharedDatabase { constructor() { return sqlite; } }
|
||||
const worker = new EpgDatabase(SharedDatabase);
|
||||
const clear = new EpgDatabaseSourceClearOperation(SharedDatabase);
|
||||
const importSource = (source, programs = true, clearFirst = false) => {
|
||||
worker.insertChannels([{
|
||||
id: 'shared', displayName: [{ value: source + ' News' }],
|
||||
icon: [{ src: source + '.png' }], url: [source + '.website']
|
||||
}], source, clearFirst);
|
||||
if (programs) worker.insertPrograms([{
|
||||
channel: 'shared', start: '2099-01-01T00:00:00Z',
|
||||
stop: '2099-01-01T01:00:00Z', title: [{value: source}]
|
||||
}], source);
|
||||
};
|
||||
const channel = () => sqlite.prepare('SELECT * FROM epg_channels WHERE id = ?').get('shared');
|
||||
${scenario}
|
||||
sqlite.close();
|
||||
`;
|
||||
return JSON.parse(
|
||||
execFileSync(
|
||||
createRequire(__filename)('electron'),
|
||||
['--import', 'tsx', '--eval', script],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
TSX_TSCONFIG_PATH: resolve(
|
||||
process.cwd(),
|
||||
'tsconfig.base.json'
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
describe('source-owned XMLTV channel metadata', () => {
|
||||
it.each(['A', 'B'])(
|
||||
'restores the survivor when removing %s, then removes the final owner',
|
||||
(removed) => {
|
||||
const survivor = removed === 'A' ? 'B' : 'A';
|
||||
const result = runDatabaseScenario(`
|
||||
importSource('A'); importSource('B');
|
||||
clear.run('${removed}');
|
||||
clear.run('${removed}');
|
||||
const surviving = channel();
|
||||
const programs = sqlite.prepare('SELECT source_url FROM epg_programs').all();
|
||||
clear.run('${survivor}');
|
||||
process.stdout.write(JSON.stringify({ surviving, programs, remaining: channel() ?? null }));
|
||||
`);
|
||||
expect(result).toEqual({
|
||||
surviving: expect.objectContaining({
|
||||
display_name: `${survivor} News`,
|
||||
icon_url: `${survivor}.png`,
|
||||
url: `${survivor}.website`,
|
||||
source_url: survivor,
|
||||
}),
|
||||
programs: [{ source_url: survivor }],
|
||||
remaining: null,
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
it.each(['same tick', 'clock rollback'])(
|
||||
'restores the latest surviving write across %s and source upserts',
|
||||
(clockMode) => {
|
||||
expect(
|
||||
runDatabaseScenario(`
|
||||
let timestamp = '2026-09-06T10:00:00.000Z';
|
||||
sqlite.function('strftime', { varargs: true }, () => timestamp);
|
||||
importSource('A');
|
||||
if ('${clockMode}' === 'clock rollback') timestamp = '2026-09-06T09:00:00.000Z';
|
||||
importSource('Z');
|
||||
importSource('M');
|
||||
clear.run('M');
|
||||
const first = channel();
|
||||
importSource('A');
|
||||
importSource('M');
|
||||
clear.run('M');
|
||||
const updated = channel();
|
||||
importSource('Z', true, true);
|
||||
importSource('M');
|
||||
clear.run('M');
|
||||
process.stdout.write(JSON.stringify({first, updated, refreshed: channel()}));
|
||||
`)
|
||||
).toEqual({
|
||||
first: expect.objectContaining({
|
||||
display_name: 'Z News',
|
||||
icon_url: 'Z.png',
|
||||
url: 'Z.website',
|
||||
source_url: 'Z',
|
||||
}),
|
||||
updated: expect.objectContaining({
|
||||
display_name: 'A News',
|
||||
icon_url: 'A.png',
|
||||
url: 'A.website',
|
||||
source_url: 'A',
|
||||
}),
|
||||
refreshed: expect.objectContaining({
|
||||
display_name: 'Z News',
|
||||
icon_url: 'Z.png',
|
||||
url: 'Z.website',
|
||||
source_url: 'Z',
|
||||
}),
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
it('retains metadata-only owners during another source refresh and removal', () => {
|
||||
expect(
|
||||
runDatabaseScenario(`
|
||||
importSource('A', false); importSource('B', false);
|
||||
worker.insertChannels([], 'A', true);
|
||||
clear.run('A');
|
||||
const surviving = channel();
|
||||
clear.run('B');
|
||||
process.stdout.write(JSON.stringify({surviving, remaining: channel() ?? null}));
|
||||
`)
|
||||
).toEqual({
|
||||
surviving: expect.objectContaining({
|
||||
display_name: 'B News',
|
||||
source_url: 'B',
|
||||
}),
|
||||
remaining: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('upgrades an existing ledger without losing snapshots and orders subsequent imports', () => {
|
||||
expect(
|
||||
runDatabaseScenario(`
|
||||
importSource('A'); importSource('Z');
|
||||
sqlite.exec('ALTER TABLE epg_channel_sources DROP COLUMN write_order');
|
||||
const migration = __databaseConnectionTestHooks.columnMigrationStatements
|
||||
.find(sql => sql.startsWith('ALTER TABLE epg_channel_sources'));
|
||||
sqlite.exec(migration);
|
||||
const snapshots = sqlite.prepare('SELECT source_url, write_order FROM epg_channel_sources ORDER BY source_url').all();
|
||||
importSource('Z'); importSource('M');
|
||||
clear.run('M');
|
||||
process.stdout.write(JSON.stringify({ snapshots, surviving: channel() }));
|
||||
`)
|
||||
).toEqual({
|
||||
snapshots: [
|
||||
{ source_url: 'A', write_order: 0 },
|
||||
{ source_url: 'Z', write_order: 0 },
|
||||
],
|
||||
surviving: expect.objectContaining({
|
||||
display_name: 'Z News',
|
||||
source_url: 'Z',
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it('removes an orphan whose original owner disappeared during refresh', () => {
|
||||
expect(
|
||||
runDatabaseScenario(`
|
||||
importSource('A', false); importSource('B', false);
|
||||
worker.insertChannels([], 'A', true);
|
||||
clear.run('B');
|
||||
process.stdout.write(JSON.stringify(channel() ?? null));
|
||||
`)
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('restores metadata before a refresh removes the last writer provenance', () => {
|
||||
expect(
|
||||
runDatabaseScenario(`
|
||||
importSource('A'); importSource('B');
|
||||
worker.insertChannels([], 'B', true);
|
||||
clear.run('B');
|
||||
process.stdout.write(JSON.stringify(channel()));
|
||||
`)
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
display_name: 'A News',
|
||||
icon_url: 'A.png',
|
||||
source_url: 'A',
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('neutralizes ambiguous legacy metadata while preserving surviving programmes', () => {
|
||||
expect(
|
||||
runDatabaseScenario(`
|
||||
importSource('A'); importSource('B');
|
||||
sqlite.exec('DROP TABLE IF EXISTS epg_channel_sources');
|
||||
for (const sql of statements) sqlite.exec(sql);
|
||||
clear.run('B');
|
||||
process.stdout.write(JSON.stringify(channel()));
|
||||
`)
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
display_name: 'shared',
|
||||
icon_url: null,
|
||||
url: null,
|
||||
source_url: 'A',
|
||||
updated_at: null,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('clears all metadata and permits a clean reimport', () => {
|
||||
expect(
|
||||
runDatabaseScenario(`
|
||||
importSource('A'); importSource('B');
|
||||
new EpgDatabaseClearOperation(SharedDatabase).run();
|
||||
const count = sqlite.prepare('SELECT count(*) AS count FROM epg_channel_sources').get().count;
|
||||
importSource('C');
|
||||
clear.run('C');
|
||||
process.stdout.write(JSON.stringify({ count, remaining: channel() ?? null }));
|
||||
`)
|
||||
).toEqual({ count: 0, remaining: null });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
import type BetterSqlite3 from 'better-sqlite3';
|
||||
|
||||
/** Called inside the source-clear transaction, before deleting its provenance. */
|
||||
export function restoreSurvivingChannelMetadata(
|
||||
db: BetterSqlite3.Database,
|
||||
sourceUrl: string
|
||||
): void {
|
||||
// Existing global rows cannot be backfilled: their last metadata writer may
|
||||
// differ from source_url. Only imports recorded in the ledger prove origin.
|
||||
const survivingValue = (column: string) => `(
|
||||
SELECT ${column} FROM epg_channel_sources
|
||||
WHERE channel_id = epg_channels.id AND source_url != @sourceUrl
|
||||
ORDER BY write_order DESC, updated_at DESC, source_url LIMIT 1
|
||||
)`;
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE epg_channels SET
|
||||
display_name = COALESCE(${survivingValue('display_name')}, id),
|
||||
icon_url = ${survivingValue('icon_url')},
|
||||
url = ${survivingValue('url')},
|
||||
updated_at = ${survivingValue('updated_at')},
|
||||
source_url = COALESCE(${survivingValue('source_url')}, (
|
||||
SELECT source_url FROM epg_programs
|
||||
WHERE channel_id = epg_channels.id AND source_url != @sourceUrl
|
||||
ORDER BY source_url LIMIT 1
|
||||
), @sourceUrl)
|
||||
WHERE source_url = @sourceUrl
|
||||
OR id IN (SELECT channel_id FROM epg_channel_sources WHERE source_url = @sourceUrl)
|
||||
OR id IN (SELECT channel_id FROM epg_programs WHERE source_url = @sourceUrl)
|
||||
`
|
||||
).run({ sourceUrl });
|
||||
}
|
||||
Reference in new issue
Block a user