mirror of
https://github.com/4gray/iptvnator.git
synced 2026-10-08 17:06:15 -08:00
* perf(database): commit catalog writes in row-budgeted transactions Refreshing or deleting a large Xtream playlist spent most of its time in "Removing cached content": every 100 rows were deleted in their own transaction, so a 300k-row catalog cost ~3,000 commits, each flushing an FTS5 segment, re-appending dirty index pages to the WAL and, about every 4 MB, running an fsync-ing auto-checkpoint. Measured on a 900k-row copy of a real database the delete took 13 s where a single set-based statement takes 3 s, with 2 GB of WAL traffic instead of 140 MB. The re-import wrote its rows the same way. Deletes now read content row counts per category from the covering indexes, pack categories into groups of about 5,000 rows and issue one set-based DELETE per group, then drop the categories (and, for playlist removal, the user-data tables) with one scoped statement each. Inserts keep 100-row statements but commit fifty of them at a time. Cancellation still lands between commits and progress still reports after each one; the worker additionally throttles progress events to one per 100 ms with summed increments, so a large operation no longer floods the renderer. Same subset, same machine: 13.0 s -> 5.6 s for the delete, 16.7 s -> 8.6 s for the insert; the fsync-bound share is larger on Windows and spinning disks. Closes #1292 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * test(database): pin the row budget and the worker-side progress flush Review follow-up: the default 5,000-row budget and the 50-statement insert commit were only exercised with explicit overrides or sub-budget inputs, and nothing covered the worker controller flushing a coalesced progress report before its terminal event. Both are now pinned, and the docs no longer claim the insert path reports SQLite `changes` or binds 1,600 parameters per statement (it binds the eleven columns a value supplies). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * test(e2e): keep the stress suites on 100-row commits via a test-only budget knob The Xtream responsiveness and playlist-switcher suites slow the database worker down with IPTVNATOR_DB_WORKER_BATCH_DELAY_MS so they can observe an import mid-flight; the stress catalog is 1,920 rows per type, which at the production budget of 5,000 rows per commit is a single commit per type and too few progress events for their assertions. The new companion knob IPTVNATOR_DB_WORKER_ROWS_PER_TRANSACTION restores 100-row commits for those runs only; unset or invalid it leaves the default untouched. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * fix(database): scope refresh and cache-clear deletes to the captured category ids The row-budgeted rewrite deleted a refreshed playlist's categories with a playlist-wide predicate. The worker serves other requests between commits, so a newer import of the same playlist could create categories in that window and lose them to the older refresh, after which its content inserts fail their foreign keys. Count and delete now use the ids the collection step read, as the chunked code did; playlist removal keeps its playlist scope because its final playlist-row delete cascades the same set. deleteCategoriesWhere runs every filter through requireScopedFilter. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
308 lines
10 KiB
TypeScript
308 lines
10 KiB
TypeScript
import type { SQL } from 'drizzle-orm';
|
|
import { SQLiteSyncDialect } from 'drizzle-orm/sqlite-core';
|
|
import * as schema from '@iptvnator/shared/database/schema';
|
|
import { XTREAM_DATABASE_PERFORMANCE_PHASE } from '@iptvnator/shared/interfaces';
|
|
import type { AppDatabase } from '../database.types';
|
|
import { createRecordingOperationPhaseCapture } from './performance-phase-capture.test-helpers';
|
|
import { deleteXtreamContent } from './xtream.operations';
|
|
|
|
/** Renders a drizzle filter the way the driver would, for asserting its scope. */
|
|
function renderFilter(filter: SQL): { sql: string; params: unknown[] } {
|
|
const query = new SQLiteSyncDialect().sqlToQuery(filter);
|
|
return { sql: query.sql, params: query.params };
|
|
}
|
|
|
|
function createSelectStep(
|
|
rows: readonly unknown[],
|
|
label: string,
|
|
timeline: string[]
|
|
) {
|
|
const where = jest.fn(() => {
|
|
timeline.push(label);
|
|
return Promise.resolve(rows);
|
|
});
|
|
const innerJoin = jest.fn(() => ({ where }));
|
|
const from = jest.fn(() => ({ innerJoin, where }));
|
|
return { from };
|
|
}
|
|
|
|
/** The grouped per-category content count query. */
|
|
function createCountStep(
|
|
rows: readonly unknown[],
|
|
label: string,
|
|
timeline: string[],
|
|
filters: SQL[]
|
|
) {
|
|
const groupBy = jest.fn(() => {
|
|
timeline.push(label);
|
|
return Promise.resolve(rows);
|
|
});
|
|
const where = jest.fn((filter: SQL) => {
|
|
filters.push(filter);
|
|
return { groupBy };
|
|
});
|
|
const innerJoin = jest.fn(() => ({ where }));
|
|
const from = jest.fn(() => ({ innerJoin }));
|
|
return { from };
|
|
}
|
|
|
|
function createDeleteHarness() {
|
|
const timeline: string[] = [];
|
|
const categories = [
|
|
{ hidden: true, id: 11, type: 'live', xtreamId: 101 },
|
|
{ hidden: false, id: 12, type: 'movies', xtreamId: 102 },
|
|
];
|
|
const favorites = [
|
|
{
|
|
addedAt: '2026-01-01T00:00:00.000Z',
|
|
contentType: 'live',
|
|
position: 3,
|
|
xtreamId: 201,
|
|
},
|
|
];
|
|
const recentlyViewed = [
|
|
{
|
|
contentType: 'movie',
|
|
viewedAt: '2026-01-02T00:00:00.000Z',
|
|
xtreamId: 202,
|
|
},
|
|
];
|
|
// 205 content rows spread over the two categories.
|
|
const contentRowCounts = [
|
|
{ categoryId: 11, rowCount: 120 },
|
|
{ categoryId: 12, rowCount: 85 },
|
|
];
|
|
const countFilters: SQL[] = [];
|
|
const deleteFilters: Array<{ table: unknown; filter: SQL }> = [];
|
|
const select = jest
|
|
.fn()
|
|
.mockReturnValueOnce(
|
|
createSelectStep(categories, 'select:categories', timeline)
|
|
)
|
|
.mockReturnValueOnce(
|
|
createSelectStep(favorites, 'select:favorites', timeline)
|
|
)
|
|
.mockReturnValueOnce(
|
|
createSelectStep(recentlyViewed, 'select:recently-viewed', timeline)
|
|
)
|
|
.mockReturnValueOnce(
|
|
createCountStep(
|
|
contentRowCounts,
|
|
'select:content',
|
|
timeline,
|
|
countFilters
|
|
)
|
|
);
|
|
const deleteRows = jest.fn((table: unknown) => ({
|
|
where: jest.fn((filter: SQL) => {
|
|
deleteFilters.push({ table, filter });
|
|
return {
|
|
run: jest.fn(() => ({
|
|
changes: table === schema.content ? 205 : 2,
|
|
})),
|
|
};
|
|
}),
|
|
}));
|
|
const transaction = jest.fn((execute: (tx: unknown) => unknown) => {
|
|
timeline.push('transaction');
|
|
return execute({ delete: deleteRows });
|
|
});
|
|
|
|
return {
|
|
countFilters,
|
|
db: { select, transaction } as unknown as AppDatabase,
|
|
deleteFilters,
|
|
timeline,
|
|
transaction,
|
|
};
|
|
}
|
|
|
|
describe('Xtream delete performance phases', () => {
|
|
it('times all user-data collection but counts only content/category deletion candidates', async () => {
|
|
const harness = createDeleteHarness();
|
|
const recording = createRecordingOperationPhaseCapture((event) => {
|
|
harness.timeline.push(`${event.phase}:${event.boundary}`);
|
|
});
|
|
const checkpoint = jest.fn(() => {
|
|
harness.timeline.push('checkpoint');
|
|
});
|
|
const onProgress = jest.fn(
|
|
(progress: {
|
|
phase: string;
|
|
current?: number;
|
|
total?: number;
|
|
increment?: number;
|
|
}) => {
|
|
harness.timeline.push(`progress:${progress.current}`);
|
|
}
|
|
);
|
|
|
|
await expect(
|
|
deleteXtreamContent(
|
|
harness.db,
|
|
'playlist-1',
|
|
{ checkpoint, onProgress },
|
|
recording.capture
|
|
)
|
|
).resolves.toEqual({
|
|
favorites: [
|
|
{
|
|
addedAt: '2026-01-01T00:00:00.000Z',
|
|
contentType: 'live',
|
|
position: 3,
|
|
xtreamId: 201,
|
|
},
|
|
],
|
|
hiddenCategories: [{ categoryType: 'live', xtreamId: 101 }],
|
|
recentlyViewed: [
|
|
{
|
|
contentType: 'movie',
|
|
viewedAt: '2026-01-02T00:00:00.000Z',
|
|
xtreamId: 202,
|
|
},
|
|
],
|
|
success: true,
|
|
});
|
|
|
|
// 205 rows fit one row-budgeted category group, then the categories
|
|
// go in a single statement.
|
|
expect(harness.transaction).toHaveBeenCalledTimes(2);
|
|
// Progress reporting performs a second cooperative checkpoint after
|
|
// each committed batch.
|
|
expect(checkpoint).toHaveBeenCalledTimes(4);
|
|
expect(onProgress.mock.calls.map(([value]) => value)).toEqual([
|
|
{
|
|
phase: 'deleting-content',
|
|
current: 205,
|
|
total: 205,
|
|
increment: 205,
|
|
},
|
|
{
|
|
phase: 'deleting-categories',
|
|
current: 2,
|
|
total: 2,
|
|
increment: 2,
|
|
},
|
|
]);
|
|
expect(harness.timeline.slice(1, 5)).toEqual([
|
|
'select:categories',
|
|
'select:favorites',
|
|
'select:recently-viewed',
|
|
'select:content',
|
|
]);
|
|
// The count and both deletes stay scoped to the category ids the
|
|
// collection read, never to the playlist: a newer import of the same
|
|
// playlist may create categories between the read and the delete.
|
|
expect(harness.countFilters.map(renderFilter)).toEqual([
|
|
{ sql: '"categories"."id" in (?, ?)', params: [11, 12] },
|
|
]);
|
|
expect(
|
|
harness.deleteFilters.map(({ table, filter }) => ({
|
|
table: table === schema.content ? 'content' : 'categories',
|
|
...renderFilter(filter),
|
|
}))
|
|
).toEqual([
|
|
{
|
|
table: 'content',
|
|
sql: '"content"."category_id" in (?, ?)',
|
|
params: [11, 12],
|
|
},
|
|
{
|
|
table: 'categories',
|
|
sql: '"categories"."id" in (?, ?)',
|
|
params: [11, 12],
|
|
},
|
|
]);
|
|
const writeStart = harness.timeline.indexOf(
|
|
'sqlite.xtream-delete.write-transactions:start'
|
|
);
|
|
const writeEnd = harness.timeline.indexOf(
|
|
'sqlite.xtream-delete.write-transactions:end'
|
|
);
|
|
expect(writeEnd).toBeGreaterThan(
|
|
harness.timeline.lastIndexOf('progress:2')
|
|
);
|
|
expect(
|
|
harness.timeline
|
|
.map((entry, index) => ({ entry, index }))
|
|
.filter(({ entry }) => entry === 'transaction')
|
|
.every(({ index }) => index > writeStart && index < writeEnd)
|
|
).toBe(true);
|
|
expect(recording.events).toEqual([
|
|
{
|
|
boundary: 'start',
|
|
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
|
|
},
|
|
{
|
|
boundary: 'end',
|
|
metadata: { itemCount: 207 },
|
|
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
|
|
},
|
|
{
|
|
boundary: 'start',
|
|
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_WRITE_TRANSACTIONS,
|
|
},
|
|
{
|
|
boundary: 'end',
|
|
metadata: { itemCount: 207 },
|
|
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_WRITE_TRANSACTIONS,
|
|
},
|
|
]);
|
|
});
|
|
|
|
it('closes the write phase on cancellation and preserves error identity', async () => {
|
|
const harness = createDeleteHarness();
|
|
const recording = createRecordingOperationPhaseCapture();
|
|
const cancellation = new Error('synthetic cancellation');
|
|
cancellation.name = 'AbortError';
|
|
let checkpointCount = 0;
|
|
|
|
await expect(
|
|
deleteXtreamContent(
|
|
harness.db,
|
|
'playlist-1',
|
|
{
|
|
checkpoint: () => {
|
|
checkpointCount += 1;
|
|
if (checkpointCount === 2) {
|
|
throw cancellation;
|
|
}
|
|
},
|
|
},
|
|
recording.capture
|
|
)
|
|
).rejects.toBe(cancellation);
|
|
|
|
expect(harness.transaction).toHaveBeenCalledTimes(1);
|
|
expect(recording.events.at(-1)).toEqual({
|
|
boundary: 'end',
|
|
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_WRITE_TRANSACTIONS,
|
|
});
|
|
});
|
|
|
|
it('closes collection on query failure and preserves error identity', async () => {
|
|
const failure = new Error('synthetic select failure');
|
|
const where = jest.fn().mockRejectedValue(failure);
|
|
const from = jest.fn(() => ({ where }));
|
|
const db = {
|
|
select: jest.fn(() => ({ from })),
|
|
} as unknown as AppDatabase;
|
|
const recording = createRecordingOperationPhaseCapture();
|
|
|
|
await expect(
|
|
deleteXtreamContent(db, 'playlist-1', undefined, recording.capture)
|
|
).rejects.toBe(failure);
|
|
|
|
expect(recording.events).toEqual([
|
|
{
|
|
boundary: 'start',
|
|
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
|
|
},
|
|
{
|
|
boundary: 'end',
|
|
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
|
|
},
|
|
]);
|
|
});
|
|
});
|