mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-15 05:36:17 -08:00
deduplicateXtreamCache no longer scans the whole catalog on every launch once both unique indexes exist (7.8 s cold on a 3.9M-title library). The playlists table keeps payload as its last column (playlists-table.ts); ensurePlaylistsPayloadLast rebuilds an existing table once, keeping its indexes and triggers, foreign keys off, committing only if references referencing playlists gain no violations. On a full copy of that profile initDatabase went from 5.2-8.1 s to 39 ms and the playlist metadata read from 1.5-2 s to 22-34 ms; the one-time rebuild costs about 4 s. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
203 lines
9.1 KiB
TypeScript
203 lines
9.1 KiB
TypeScript
import { execFileSync } from 'node:child_process';
|
|
import { createRequire } from 'node:module';
|
|
import { resolve } from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
|
|
interface RebuildOutcome {
|
|
warnings: string[];
|
|
columns: string[];
|
|
tableSql: string;
|
|
playlists: Record<string, unknown>[];
|
|
favorites: Record<string, unknown>[];
|
|
leftoverRebuildTable: boolean;
|
|
foreignKeys: number;
|
|
integrity: string;
|
|
dependents: { type: string; name: string }[];
|
|
triggerFires: boolean;
|
|
}
|
|
|
|
/**
|
|
* Runs ensurePlaylistsPayloadLast against a real in-memory SQLite database
|
|
* holding a pre-0.19 playlists table (payload appended before the EPG URL
|
|
* columns), a favorites table referencing it, and an orphaned favorite a
|
|
* legacy profile can carry. better-sqlite3 is built for Electron, so the
|
|
* script runs under Electron as Node.
|
|
*/
|
|
function rebuildLegacyPlaylists(options: {
|
|
blockingView: boolean;
|
|
}): RebuildOutcome {
|
|
const electronPath = createRequire(__filename)('electron') as string;
|
|
const moduleUrl = pathToFileURL(
|
|
resolve(__dirname, 'playlists-table.ts')
|
|
).href;
|
|
const script = `
|
|
const { default: Database } = await import('better-sqlite3');
|
|
const { ensurePlaylistsPayloadLast } = await import(${JSON.stringify(moduleUrl)});
|
|
const sqlite = new Database(':memory:');
|
|
sqlite.pragma('foreign_keys = ON');
|
|
sqlite.exec(\`
|
|
CREATE TABLE playlists (
|
|
id TEXT PRIMARY KEY, name TEXT NOT NULL, serverUrl TEXT,
|
|
username TEXT, password TEXT,
|
|
date_created TEXT DEFAULT (datetime('now')), last_updated TEXT,
|
|
type TEXT NOT NULL CHECK (type IN ('xtream', 'stalker', 'm3u-file', 'm3u-text', 'm3u-url')),
|
|
userAgent TEXT, origin TEXT, referrer TEXT, filePath TEXT,
|
|
autoRefresh INTEGER DEFAULT 0, macAddress TEXT, url TEXT,
|
|
last_usage TEXT
|
|
);
|
|
ALTER TABLE playlists ADD COLUMN portal_url TEXT;
|
|
ALTER TABLE playlists ADD COLUMN count INTEGER;
|
|
ALTER TABLE playlists ADD COLUMN import_date TEXT;
|
|
ALTER TABLE playlists ADD COLUMN update_date INTEGER;
|
|
ALTER TABLE playlists ADD COLUMN position INTEGER;
|
|
ALTER TABLE playlists ADD COLUMN favorites TEXT;
|
|
ALTER TABLE playlists ADD COLUMN recently_viewed TEXT;
|
|
ALTER TABLE playlists ADD COLUMN payload TEXT;
|
|
ALTER TABLE playlists ADD COLUMN epg_urls TEXT;
|
|
ALTER TABLE playlists ADD COLUMN detected_epg_urls TEXT;
|
|
ALTER TABLE playlists ADD COLUMN manual_epg_urls TEXT;
|
|
ALTER TABLE playlists ADD COLUMN disabled_epg_urls TEXT;
|
|
CREATE TABLE favorites (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
content_id INTEGER NOT NULL,
|
|
playlist_id TEXT NOT NULL,
|
|
FOREIGN KEY (playlist_id) REFERENCES playlists (id) ON DELETE CASCADE
|
|
);
|
|
INSERT INTO playlists (id, name, type, url, last_usage, count, payload,
|
|
epg_urls, detected_epg_urls, manual_epg_urls, disabled_epg_urls)
|
|
VALUES ('m3u', 'Saved list', 'm3u-url', 'https://list.invalid/a.m3u',
|
|
'2026-01-01T00:00:00.000Z', 2, '{"channels":[{"name":"One"},{"name":"Two"}]}',
|
|
'["https://epg.invalid/a.xml"]', '["https://epg.invalid/b.xml"]',
|
|
'["https://epg.invalid/c.xml"]', '[]');
|
|
INSERT INTO playlists (id, name, type, serverUrl, username, password)
|
|
VALUES ('xtream', 'Saved source', 'xtream', 'https://source.invalid', 'user', 'secret');
|
|
INSERT INTO favorites (content_id, playlist_id) VALUES (1, 'm3u'), (2, 'xtream');
|
|
\`);
|
|
sqlite.pragma('foreign_keys = OFF');
|
|
sqlite.exec("INSERT INTO favorites (content_id, playlist_id) VALUES (3, 'deleted')");
|
|
sqlite.pragma('foreign_keys = ON');
|
|
sqlite.exec(\`
|
|
CREATE INDEX playlists_type_idx ON playlists(type);
|
|
CREATE TRIGGER playlists_reject_blocked BEFORE INSERT ON playlists
|
|
WHEN NEW.id = 'blocked' BEGIN SELECT RAISE(ABORT, 'blocked id'); END;
|
|
\`);
|
|
if (${options.blockingView}) {
|
|
sqlite.exec('CREATE VIEW playlist_names AS SELECT name FROM playlists');
|
|
}
|
|
const warnings = [];
|
|
console.warn = (...args) => warnings.push(args.map(String).join(' '));
|
|
console.log = () => undefined;
|
|
ensurePlaylistsPayloadLast(sqlite);
|
|
let triggerFires = false;
|
|
try {
|
|
sqlite.prepare("INSERT INTO playlists (id, name, type) VALUES ('blocked', 'B', 'm3u-url')").run();
|
|
} catch (error) {
|
|
triggerFires = String(error).includes('blocked id');
|
|
}
|
|
const byName = (rows) => rows.map((row) =>
|
|
Object.fromEntries(Object.entries(row).sort(([a], [b]) => a.localeCompare(b))));
|
|
process.stdout.write(JSON.stringify({
|
|
warnings,
|
|
columns: sqlite.prepare("SELECT name FROM pragma_table_info('playlists')").all().map((r) => r.name),
|
|
tableSql: sqlite.prepare("SELECT sql FROM sqlite_master WHERE name = 'playlists'").get().sql,
|
|
playlists: byName(sqlite.prepare('SELECT * FROM playlists ORDER BY id').all()),
|
|
favorites: sqlite.prepare('SELECT * FROM favorites ORDER BY id').all(),
|
|
leftoverRebuildTable: Boolean(sqlite.prepare(
|
|
"SELECT 1 FROM sqlite_master WHERE name = 'playlists_payload_rebuild'").get()),
|
|
foreignKeys: sqlite.pragma('foreign_keys', { simple: true }),
|
|
dependents: sqlite.prepare(
|
|
"SELECT type, name FROM sqlite_master WHERE tbl_name = 'playlists' AND sql IS NOT NULL AND type IN ('index', 'trigger') ORDER BY name").all(),
|
|
triggerFires,
|
|
integrity: sqlite.pragma('integrity_check', { simple: true }),
|
|
}));
|
|
sqlite.close();
|
|
`;
|
|
const output = execFileSync(
|
|
electronPath,
|
|
['--import', 'tsx', '--eval', script],
|
|
{
|
|
cwd: process.cwd(),
|
|
encoding: 'utf8',
|
|
timeout: 30_000,
|
|
env: {
|
|
...process.env,
|
|
ELECTRON_RUN_AS_NODE: '1',
|
|
TSX_TSCONFIG_PATH: resolve(process.cwd(), 'tsconfig.base.json'),
|
|
},
|
|
}
|
|
);
|
|
return JSON.parse(output) as RebuildOutcome;
|
|
}
|
|
|
|
const expectedPlaylists = [
|
|
expect.objectContaining({
|
|
id: 'm3u',
|
|
name: 'Saved list',
|
|
url: 'https://list.invalid/a.m3u',
|
|
last_usage: '2026-01-01T00:00:00.000Z',
|
|
count: 2,
|
|
payload: '{"channels":[{"name":"One"},{"name":"Two"}]}',
|
|
epg_urls: '["https://epg.invalid/a.xml"]',
|
|
detected_epg_urls: '["https://epg.invalid/b.xml"]',
|
|
manual_epg_urls: '["https://epg.invalid/c.xml"]',
|
|
disabled_epg_urls: '[]',
|
|
}),
|
|
expect.objectContaining({
|
|
id: 'xtream',
|
|
serverUrl: 'https://source.invalid',
|
|
username: 'user',
|
|
password: 'secret',
|
|
payload: null,
|
|
}),
|
|
];
|
|
|
|
const expectedDependents = [
|
|
{ type: 'trigger', name: 'playlists_reject_blocked' },
|
|
{ type: 'index', name: 'playlists_type_idx' },
|
|
];
|
|
|
|
const expectedFavorites = [
|
|
{ id: 1, content_id: 1, playlist_id: 'm3u' },
|
|
{ id: 2, content_id: 2, playlist_id: 'xtream' },
|
|
{ id: 3, content_id: 3, playlist_id: 'deleted' },
|
|
];
|
|
|
|
describe('ensurePlaylistsPayloadLast on real SQLite', () => {
|
|
it('moves payload last and keeps every row, favorite, pre-existing orphan, index and trigger', () => {
|
|
const outcome = rebuildLegacyPlaylists({ blockingView: false });
|
|
|
|
expect(outcome.warnings).toEqual([]);
|
|
expect(outcome.columns.at(-1)).toBe('payload');
|
|
expect(outcome.columns).toHaveLength(28);
|
|
expect(outcome.playlists).toEqual(expectedPlaylists);
|
|
// Not cascaded away by the drop, and the orphan it found does not
|
|
// block the rebuild.
|
|
expect(outcome.favorites).toEqual(expectedFavorites);
|
|
expect(outcome.leftoverRebuildTable).toBe(false);
|
|
expect(outcome.foreignKeys).toBe(1);
|
|
expect(outcome.integrity).toBe('ok');
|
|
expect(outcome.dependents).toEqual(expectedDependents);
|
|
expect(outcome.triggerFires).toBe(true);
|
|
});
|
|
|
|
it('rolls everything back when a step fails, and turns foreign keys back on', () => {
|
|
// A view on playlists makes the rename fail after the drop.
|
|
const outcome = rebuildLegacyPlaylists({ blockingView: true });
|
|
|
|
expect(outcome.warnings).toEqual([
|
|
expect.stringContaining(
|
|
'[DB] playlists payload-last rebuild failed:'
|
|
),
|
|
]);
|
|
expect(outcome.columns.at(-1)).toBe('disabled_epg_urls');
|
|
expect(outcome.tableSql).toContain('payload TEXT, epg_urls TEXT');
|
|
expect(outcome.playlists).toEqual(expectedPlaylists);
|
|
expect(outcome.favorites).toEqual(expectedFavorites);
|
|
expect(outcome.leftoverRebuildTable).toBe(false);
|
|
expect(outcome.foreignKeys).toBe(1);
|
|
expect(outcome.integrity).toBe('ok');
|
|
expect(outcome.dependents).toEqual(expectedDependents);
|
|
expect(outcome.triggerFires).toBe(true);
|
|
});
|
|
});
|