Files
iptvnator/apps/electron-backend/src/app/database/operations/xtream-delete-performance.spec.ts
T
4grayandClaude Fable 5.1 82295356ed perf(database): commit catalog writes in row-budgeted transactions (#1511)
* perf(database): commit catalog writes in row-budgeted transactions

Refreshing or deleting a large Xtream playlist spent most of its time in
"Removing cached content": every 100 rows were deleted in their own
transaction, so a 300k-row catalog cost ~3,000 commits, each flushing an
FTS5 segment, re-appending dirty index pages to the WAL and, about every
4 MB, running an fsync-ing auto-checkpoint. Measured on a 900k-row copy of
a real database the delete took 13 s where a single set-based statement
takes 3 s, with 2 GB of WAL traffic instead of 140 MB. The re-import wrote
its rows the same way.

Deletes now read content row counts per category from the covering
indexes, pack categories into groups of about 5,000 rows and issue one
set-based DELETE per group, then drop the categories (and, for playlist
removal, the user-data tables) with one scoped statement each. Inserts keep
100-row statements but commit fifty of them at a time. Cancellation still
lands between commits and progress still reports after each one; the
worker additionally throttles progress events to one per 100 ms with
summed increments, so a large operation no longer floods the renderer.

Same subset, same machine: 13.0 s -> 5.6 s for the delete, 16.7 s -> 8.6 s
for the insert; the fsync-bound share is larger on Windows and spinning
disks.

Closes #1292

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* test(database): pin the row budget and the worker-side progress flush

Review follow-up: the default 5,000-row budget and the 50-statement insert
commit were only exercised with explicit overrides or sub-budget inputs, and
nothing covered the worker controller flushing a coalesced progress report
before its terminal event. Both are now pinned, and the docs no longer claim
the insert path reports SQLite `changes` or binds 1,600 parameters per
statement (it binds the eleven columns a value supplies).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* test(e2e): keep the stress suites on 100-row commits via a test-only budget knob

The Xtream responsiveness and playlist-switcher suites slow the database
worker down with IPTVNATOR_DB_WORKER_BATCH_DELAY_MS so they can observe an
import mid-flight; the stress catalog is 1,920 rows per type, which at the
production budget of 5,000 rows per commit is a single commit per type and
too few progress events for their assertions. The new companion knob
IPTVNATOR_DB_WORKER_ROWS_PER_TRANSACTION restores 100-row commits for those
runs only; unset or invalid it leaves the default untouched.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* fix(database): scope refresh and cache-clear deletes to the captured category ids

The row-budgeted rewrite deleted a refreshed playlist's categories with a
playlist-wide predicate. The worker serves other requests between commits,
so a newer import of the same playlist could create categories in that
window and lose them to the older refresh, after which its content inserts
fail their foreign keys. Count and delete now use the ids the collection
step read, as the chunked code did; playlist removal keeps its playlist
scope because its final playlist-row delete cascades the same set.
deleteCategoriesWhere runs every filter through requireScopedFilter.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-03 10:31:31 +02:00

308 lines
10 KiB
TypeScript

import type { SQL } from 'drizzle-orm';
import { SQLiteSyncDialect } from 'drizzle-orm/sqlite-core';
import * as schema from '@iptvnator/shared/database/schema';
import { XTREAM_DATABASE_PERFORMANCE_PHASE } from '@iptvnator/shared/interfaces';
import type { AppDatabase } from '../database.types';
import { createRecordingOperationPhaseCapture } from './performance-phase-capture.test-helpers';
import { deleteXtreamContent } from './xtream.operations';
/** Renders a drizzle filter the way the driver would, for asserting its scope. */
function renderFilter(filter: SQL): { sql: string; params: unknown[] } {
const query = new SQLiteSyncDialect().sqlToQuery(filter);
return { sql: query.sql, params: query.params };
}
function createSelectStep(
rows: readonly unknown[],
label: string,
timeline: string[]
) {
const where = jest.fn(() => {
timeline.push(label);
return Promise.resolve(rows);
});
const innerJoin = jest.fn(() => ({ where }));
const from = jest.fn(() => ({ innerJoin, where }));
return { from };
}
/** The grouped per-category content count query. */
function createCountStep(
rows: readonly unknown[],
label: string,
timeline: string[],
filters: SQL[]
) {
const groupBy = jest.fn(() => {
timeline.push(label);
return Promise.resolve(rows);
});
const where = jest.fn((filter: SQL) => {
filters.push(filter);
return { groupBy };
});
const innerJoin = jest.fn(() => ({ where }));
const from = jest.fn(() => ({ innerJoin }));
return { from };
}
function createDeleteHarness() {
const timeline: string[] = [];
const categories = [
{ hidden: true, id: 11, type: 'live', xtreamId: 101 },
{ hidden: false, id: 12, type: 'movies', xtreamId: 102 },
];
const favorites = [
{
addedAt: '2026-01-01T00:00:00.000Z',
contentType: 'live',
position: 3,
xtreamId: 201,
},
];
const recentlyViewed = [
{
contentType: 'movie',
viewedAt: '2026-01-02T00:00:00.000Z',
xtreamId: 202,
},
];
// 205 content rows spread over the two categories.
const contentRowCounts = [
{ categoryId: 11, rowCount: 120 },
{ categoryId: 12, rowCount: 85 },
];
const countFilters: SQL[] = [];
const deleteFilters: Array<{ table: unknown; filter: SQL }> = [];
const select = jest
.fn()
.mockReturnValueOnce(
createSelectStep(categories, 'select:categories', timeline)
)
.mockReturnValueOnce(
createSelectStep(favorites, 'select:favorites', timeline)
)
.mockReturnValueOnce(
createSelectStep(recentlyViewed, 'select:recently-viewed', timeline)
)
.mockReturnValueOnce(
createCountStep(
contentRowCounts,
'select:content',
timeline,
countFilters
)
);
const deleteRows = jest.fn((table: unknown) => ({
where: jest.fn((filter: SQL) => {
deleteFilters.push({ table, filter });
return {
run: jest.fn(() => ({
changes: table === schema.content ? 205 : 2,
})),
};
}),
}));
const transaction = jest.fn((execute: (tx: unknown) => unknown) => {
timeline.push('transaction');
return execute({ delete: deleteRows });
});
return {
countFilters,
db: { select, transaction } as unknown as AppDatabase,
deleteFilters,
timeline,
transaction,
};
}
describe('Xtream delete performance phases', () => {
it('times all user-data collection but counts only content/category deletion candidates', async () => {
const harness = createDeleteHarness();
const recording = createRecordingOperationPhaseCapture((event) => {
harness.timeline.push(`${event.phase}:${event.boundary}`);
});
const checkpoint = jest.fn(() => {
harness.timeline.push('checkpoint');
});
const onProgress = jest.fn(
(progress: {
phase: string;
current?: number;
total?: number;
increment?: number;
}) => {
harness.timeline.push(`progress:${progress.current}`);
}
);
await expect(
deleteXtreamContent(
harness.db,
'playlist-1',
{ checkpoint, onProgress },
recording.capture
)
).resolves.toEqual({
favorites: [
{
addedAt: '2026-01-01T00:00:00.000Z',
contentType: 'live',
position: 3,
xtreamId: 201,
},
],
hiddenCategories: [{ categoryType: 'live', xtreamId: 101 }],
recentlyViewed: [
{
contentType: 'movie',
viewedAt: '2026-01-02T00:00:00.000Z',
xtreamId: 202,
},
],
success: true,
});
// 205 rows fit one row-budgeted category group, then the categories
// go in a single statement.
expect(harness.transaction).toHaveBeenCalledTimes(2);
// Progress reporting performs a second cooperative checkpoint after
// each committed batch.
expect(checkpoint).toHaveBeenCalledTimes(4);
expect(onProgress.mock.calls.map(([value]) => value)).toEqual([
{
phase: 'deleting-content',
current: 205,
total: 205,
increment: 205,
},
{
phase: 'deleting-categories',
current: 2,
total: 2,
increment: 2,
},
]);
expect(harness.timeline.slice(1, 5)).toEqual([
'select:categories',
'select:favorites',
'select:recently-viewed',
'select:content',
]);
// The count and both deletes stay scoped to the category ids the
// collection read, never to the playlist: a newer import of the same
// playlist may create categories between the read and the delete.
expect(harness.countFilters.map(renderFilter)).toEqual([
{ sql: '"categories"."id" in (?, ?)', params: [11, 12] },
]);
expect(
harness.deleteFilters.map(({ table, filter }) => ({
table: table === schema.content ? 'content' : 'categories',
...renderFilter(filter),
}))
).toEqual([
{
table: 'content',
sql: '"content"."category_id" in (?, ?)',
params: [11, 12],
},
{
table: 'categories',
sql: '"categories"."id" in (?, ?)',
params: [11, 12],
},
]);
const writeStart = harness.timeline.indexOf(
'sqlite.xtream-delete.write-transactions:start'
);
const writeEnd = harness.timeline.indexOf(
'sqlite.xtream-delete.write-transactions:end'
);
expect(writeEnd).toBeGreaterThan(
harness.timeline.lastIndexOf('progress:2')
);
expect(
harness.timeline
.map((entry, index) => ({ entry, index }))
.filter(({ entry }) => entry === 'transaction')
.every(({ index }) => index > writeStart && index < writeEnd)
).toBe(true);
expect(recording.events).toEqual([
{
boundary: 'start',
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
},
{
boundary: 'end',
metadata: { itemCount: 207 },
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
},
{
boundary: 'start',
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_WRITE_TRANSACTIONS,
},
{
boundary: 'end',
metadata: { itemCount: 207 },
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_WRITE_TRANSACTIONS,
},
]);
});
it('closes the write phase on cancellation and preserves error identity', async () => {
const harness = createDeleteHarness();
const recording = createRecordingOperationPhaseCapture();
const cancellation = new Error('synthetic cancellation');
cancellation.name = 'AbortError';
let checkpointCount = 0;
await expect(
deleteXtreamContent(
harness.db,
'playlist-1',
{
checkpoint: () => {
checkpointCount += 1;
if (checkpointCount === 2) {
throw cancellation;
}
},
},
recording.capture
)
).rejects.toBe(cancellation);
expect(harness.transaction).toHaveBeenCalledTimes(1);
expect(recording.events.at(-1)).toEqual({
boundary: 'end',
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_WRITE_TRANSACTIONS,
});
});
it('closes collection on query failure and preserves error identity', async () => {
const failure = new Error('synthetic select failure');
const where = jest.fn().mockRejectedValue(failure);
const from = jest.fn(() => ({ where }));
const db = {
select: jest.fn(() => ({ from })),
} as unknown as AppDatabase;
const recording = createRecordingOperationPhaseCapture();
await expect(
deleteXtreamContent(db, 'playlist-1', undefined, recording.capture)
).rejects.toBe(failure);
expect(recording.events).toEqual([
{
boundary: 'start',
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
},
{
boundary: 'end',
phase: XTREAM_DATABASE_PERFORMANCE_PHASE.SQLITE_XTREAM_DELETE_COLLECT_USER_DATA,
},
]);
});
});