diff --git a/.changes/host-health-live-trial.md b/.changes/host-health-live-trial.md new file mode 100644 index 000000000..482435dff --- /dev/null +++ b/.changes/host-health-live-trial.md @@ -0,0 +1,7 @@ +--- +type: fix +area: host-health +issues: [1439] +--- + +Portal recovery now waits for an active probe to finish before sending another request, even when redirects or a slow response take longer than 45 seconds. Completed and cancelled requests release the probe slot reliably. diff --git a/AGENTS.md b/AGENTS.md index 3c042bfea..fadea455e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,6 +176,12 @@ categories by provider ID and type. See `docs/architecture/category-management.m ## Portal Connectivity Preference +- Half-open trial slots follow the complete request lifetime with no elapsed-time + expiry. All four Electron/web-backend portal handlers release in `finally`, + independently of outcome reporting; cleanup preserves trial/epoch ownership + and works while the environment override is disabled. Contract: + `docs/architecture/host-connectivity-guard.md` ("Trial ownership follows the + request lifetime"). - Desktop Settings > General > Portal connections exposes default-on `Settings.portalConnectivityGuard`. Only explicit false opts out. Save mirrors the value to Electron `PORTAL_CONNECTIVITY_GUARD` and applies it without restart; diff --git a/CLAUDE.md b/CLAUDE.md index d9d114606..f588108e8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1661,6 +1661,12 @@ No formal migration system yet. Schema changes are applied via raw SQL in the `c ## Portal Connectivity Preference +- Half-open trial slots follow the complete request lifetime with no elapsed-time + expiry. All four Electron/web-backend portal handlers release in `finally`, + independently of outcome reporting; cleanup preserves trial/epoch ownership + and works while the environment override is disabled. Contract: + `docs/architecture/host-connectivity-guard.md` ("Trial ownership follows the + request lifetime"). - Desktop Settings > General > Portal connections exposes default-on `Settings.portalConnectivityGuard`. Only explicit false opts out. Save mirrors the value to Electron `PORTAL_CONNECTIVITY_GUARD` and applies it without restart; diff --git a/apps/electron-backend/src/app/events/stalker.events.spec.ts b/apps/electron-backend/src/app/events/stalker.events.spec.ts index 10fe5a800..4cdc2321d 100644 --- a/apps/electron-backend/src/app/events/stalker.events.spec.ts +++ b/apps/electron-backend/src/app/events/stalker.events.spec.ts @@ -83,6 +83,75 @@ describe('StalkerEvents host connectivity guard', () => { consoleWarnSpy.mockRestore(); }); + it('holds a live trial beyond 45 seconds and releases a cancelled request', async () => { + let now = 1_000; + const clock = jest.spyOn(Date, 'now').mockImplementation(() => now); + let cancel!: (error: Error) => void; + let arrived!: () => void; + const pending = new Promise((_, reject) => { + cancel = reject; + }); + const started = new Promise((resolve) => { + arrived = resolve; + }); + let trial: Promise | undefined; + try { + axiosMock.mockRejectedValue(connectionRefused()); + await expect(request()).rejects.toBeDefined(); + await expect(request()).rejects.toBeDefined(); + now += 30_001; + axiosMock.mockImplementationOnce(() => { + arrived(); + return pending; + }); + trial = request().catch((error) => error); + await started; + now += 45_001; + await expect(request()).rejects.toThrow( + buildHostConnectivityFastFailMessage(PORTAL_ENDPOINT) + ); + expect(axiosMock).toHaveBeenCalledTimes(3); + cancel( + Object.assign(new Error('cancelled'), { code: 'ERR_CANCELED' }) + ); + await trial; + axiosMock.mockResolvedValue({ status: 200, data: [], headers: {} }); + await expect(request()).resolves.toBeDefined(); + expect(axiosMock).toHaveBeenCalledTimes(4); + } finally { + cancel(new Error('test cleanup')); + await trial; + clock.mockRestore(); + } + }); + + it('releases the trial in finally when debug reporting throws before the outcome report', async () => { + let now = 1_000; + const clock = jest.spyOn(Date, 'now').mockImplementation(() => now); + const debug = await import('./portal-debug.events'); + const reportingError = new Error('debug reporting failed'); + try { + axiosMock.mockRejectedValue(connectionRefused()); + await expect(request()).rejects.toBeDefined(); + await expect(request()).rejects.toBeDefined(); + now += 30_001; + jest.mocked(debug.emitPortalDebugEvent).mockImplementationOnce( + () => { + throw reportingError; + } + ); + await expect(request({ requestId: 'debug-trial' })).rejects.toBe( + reportingError + ); + axiosMock.mockResolvedValue({ status: 200, data: [], headers: {} }); + await expect(request()).resolves.toBeDefined(); + expect(axiosMock).toHaveBeenCalledTimes(4); + } finally { + jest.mocked(debug.emitPortalDebugEvent).mockReset(); + clock.mockRestore(); + } + }); + it('stops contacting a portal host that refused twice in a row', async () => { axiosMock.mockRejectedValue(connectionRefused()); diff --git a/apps/electron-backend/src/app/events/stalker.events.ts b/apps/electron-backend/src/app/events/stalker.events.ts index 86bcf2a2d..6e7528e78 100644 --- a/apps/electron-backend/src/app/events/stalker.events.ts +++ b/apps/electron-backend/src/app/events/stalker.events.ts @@ -26,6 +26,7 @@ import { observeGuardedHostRequest, reportGuardedHostFailure, reportGuardedHostSuccess, + releaseGuardedHostRequest, } from '../util/host-connectivity-guard'; export default class StalkerEvents { @@ -264,6 +265,8 @@ ipcMain.handle( status: 500, }; } + } finally { + releaseGuardedHostRequest(guardToken); } } ); diff --git a/apps/electron-backend/src/app/events/xtream.events.spec.ts b/apps/electron-backend/src/app/events/xtream.events.spec.ts index 0246dff3f..11b7e581e 100644 --- a/apps/electron-backend/src/app/events/xtream.events.spec.ts +++ b/apps/electron-backend/src/app/events/xtream.events.spec.ts @@ -341,6 +341,174 @@ describe('XtreamEvents host connectivity guard', () => { delete process.env[GUARD_DISABLED_ENV]; }); + it.each(['success', 'failure', 'cancel', 'redirect-failure'])( + 'holds a long request or redirect-chain trial until %s settles', + async (outcome) => { + let now = 1_000; + const clock = jest.spyOn(Date, 'now').mockImplementation(() => now); + const finalHop = createDeferred(); + const arrived = createDeferred(); + let trial: Promise | undefined; + const expectedCalls = outcome === 'failure' ? 3 : 5; + try { + axiosMock.mockRejectedValue(timedOut()); + await expect(request()).rejects.toBeDefined(); + await expect(request()).rejects.toBeDefined(); + now += 30_001; + // Direct failure uses a slow pending transfer. Other outcomes + // traverse the real validated-axios loop with two 25 s hops. + if (outcome !== 'failure') { + axiosMock + .mockImplementationOnce(async () => { + now += 25_000; + return { + status: 302, + headers: { location: '/second' }, + }; + }) + .mockImplementationOnce(async () => { + now += 25_000; + return { + status: 302, + headers: { location: '/third' }, + }; + }); + } + axiosMock.mockImplementationOnce(() => { + if (outcome === 'failure') now += 50_000; + arrived.resolve(); + return finalHop.promise; + }); + trial = request().then( + (value) => ({ value }), + (error) => ({ error }) + ); + await arrived.promise; + await expect(request()).rejects.toThrow( + buildHostConnectivityFastFailMessage(SERVER_ENDPOINT) + ); + expect(axiosMock).toHaveBeenCalledTimes(expectedCalls); + if (outcome === 'success') { + finalHop.resolve({ status: 200, data: [], headers: {} }); + } else { + finalHop.reject( + Object.assign(timedOut(), { + code: + outcome === 'cancel' + ? 'ERR_CANCELED' + : 'ETIMEDOUT', + config: { + url: + outcome === 'redirect-failure' + ? `${SERVER_URL}/third` + : `${SERVER_URL}/player_api.php`, + }, + }) + ); + } + await trial; + if (outcome === 'failure') { + await expect(request()).rejects.toThrow( + buildHostConnectivityFastFailMessage(SERVER_ENDPOINT) + ); + now += 30_001; + } + axiosMock.mockResolvedValue({ + status: 200, + data: [], + headers: {}, + }); + await expect(request()).resolves.toMatchObject({ payload: [] }); + expect(axiosMock).toHaveBeenCalledTimes(expectedCalls + 1); + } finally { + finalHop.resolve({ status: 200, data: [], headers: {} }); + await trial; + clock.mockRestore(); + } + } + ); + + it('ignores an old pending trial failure after reset while a replacement is active', async () => { + let now = 1_000; + const clock = jest.spyOn(Date, 'now').mockImplementation(() => now); + const old = createDeferred(); + const replacement = createDeferred(); + const oldStarted = createDeferred(); + const replacementStarted = createDeferred(); + let oldRequest: Promise | undefined; + let newRequest: Promise | undefined; + try { + axiosMock.mockRejectedValue(timedOut()); + await expect(request()).rejects.toBeDefined(); + await expect(request()).rejects.toBeDefined(); + now += 30_001; + axiosMock.mockImplementationOnce(() => { + oldStarted.resolve(); + return old.promise; + }); + oldRequest = request().catch((error) => error); + await oldStarted.promise; + await resetHandler({}, { url: SERVER_URL }); + await expect(request()).rejects.toBeDefined(); + await expect(request()).rejects.toBeDefined(); + now += 30_001; + axiosMock.mockImplementationOnce(() => { + replacementStarted.resolve(); + return replacement.promise; + }); + newRequest = request(); + await replacementStarted.promise; + old.reject(timedOut()); + await oldRequest; + now += 45_001; + await expect(request()).rejects.toThrow( + buildHostConnectivityFastFailMessage(SERVER_ENDPOINT) + ); + expect(axiosMock).toHaveBeenCalledTimes(6); + replacement.resolve({ status: 200, data: [], headers: {} }); + await expect(newRequest).resolves.toMatchObject({ payload: [] }); + } finally { + old.resolve({ status: 200, data: [], headers: {} }); + replacement.resolve({ status: 200, data: [], headers: {} }); + await Promise.all([oldRequest, newRequest]); + clock.mockRestore(); + } + }); + + it('releases the trial in finally when debug reporting throws before the outcome report', async () => { + let now = 1_000; + const clock = jest.spyOn(Date, 'now').mockImplementation(() => now); + const debug = await import('./portal-debug.events'); + const reportingError = new Error('debug reporting failed'); + try { + axiosMock.mockRejectedValue(timedOut()); + await expect(request()).rejects.toBeDefined(); + await expect(request()).rejects.toBeDefined(); + now += 30_001; + jest.mocked(debug.emitPortalDebugEvent).mockImplementationOnce( + () => { + throw reportingError; + } + ); + await expect( + requestHandler( + {}, + { + url: SERVER_URL, + params: { action: 'get_live_categories' }, + requestId: 'debug-trial', + } + ) + ).rejects.toBe(reportingError); + axiosMock.mockResolvedValue({ status: 200, data: [], headers: {} }); + await expect(request()).resolves.toBeDefined(); + expect(axiosMock).toHaveBeenCalledTimes(4); + } finally { + jest.mocked(debug.emitPortalDebugEvent).mockReset(); + clock.mockRestore(); + } + }); + it('stops contacting a panel that timed out twice in a row', async () => { axiosMock.mockRejectedValue(timedOut()); diff --git a/apps/electron-backend/src/app/events/xtream.events.ts b/apps/electron-backend/src/app/events/xtream.events.ts index 0a0771ecd..be51a509a 100644 --- a/apps/electron-backend/src/app/events/xtream.events.ts +++ b/apps/electron-backend/src/app/events/xtream.events.ts @@ -22,6 +22,7 @@ import { beginGuardedHostRequest, reportGuardedHostFailure, reportGuardedHostSuccess, + releaseGuardedHostRequest, } from '../util/host-connectivity-guard'; import { createXtreamMainPerformanceCaptureForRequest, @@ -265,6 +266,7 @@ ipcMain.handle( }; } } finally { + releaseGuardedHostRequest(guardToken); if (activeRequestKey) { activeXtreamRequests.delete(activeRequestKey); } diff --git a/apps/electron-backend/src/app/util/host-connectivity-guard.ts b/apps/electron-backend/src/app/util/host-connectivity-guard.ts index 2b0289a7b..55a36d9da 100644 --- a/apps/electron-backend/src/app/util/host-connectivity-guard.ts +++ b/apps/electron-backend/src/app/util/host-connectivity-guard.ts @@ -140,6 +140,15 @@ export function reportGuardedHostFailure( } } +/** Finally cleanup, even if reporting or debug/error formatting threw. */ +export function releaseGuardedHostRequest( + token: HostRequestToken | null +): void { + if (token && currentTokens.delete(token)) { + getHostConnectivityGuard().reportInconclusive(token); + } +} + /** Forgets the recorded failures for the endpoint `url` points at. */ export function resetGuardedHost(url: string): boolean { const endpoint = portalEndpointKeyOf(url); diff --git a/apps/web-backend/src/app/host-guard.ts b/apps/web-backend/src/app/host-guard.ts index 645295e7a..d4d710656 100644 --- a/apps/web-backend/src/app/host-guard.ts +++ b/apps/web-backend/src/app/host-guard.ts @@ -101,8 +101,8 @@ export function admitProviderRequest( * For a request that was admitted and then abandoned before it went out — a URL * the SSRF policy refused, for instance. That says nothing about reachability, * so it must not count as a failure; but the slot a half-open trial reserved - * has to be released, or the breaker waits out the full trial timeout for a - * request that never happened. + * has to be released. Every admitted route also calls this in finally, even + * if outcome reporting throws; cleanup is idempotent and records no failure. */ export function releaseProviderRequest( guard: HostConnectivityGuard, diff --git a/apps/web-backend/src/app/web-backend-app.host-guard.spec.ts b/apps/web-backend/src/app/web-backend-app.host-guard.spec.ts index 0bb7e8af6..b01b3529b 100644 --- a/apps/web-backend/src/app/web-backend-app.host-guard.spec.ts +++ b/apps/web-backend/src/app/web-backend-app.host-guard.spec.ts @@ -13,6 +13,7 @@ import { } from '@iptvnator/shared/host-health'; import { isHostConnectivityFastFailMessage } from '@iptvnator/shared/interfaces'; import { createWebBackendApp } from './web-backend-app'; +import express, { Response } from 'express'; import { registerProviderTarget, resolvePublicHost, @@ -538,6 +539,226 @@ describe('web backend host connectivity guard', () => { ); }); + it('keeps ownership through a real axios redirect and unfinished streaming body', async () => { + const provider = express(); + let stream!: Response; + let arrived!: () => void; + const started = new Promise((resolve) => { + arrived = resolve; + }); + provider.get('/player_api.php', (_req, res) => res.redirect('/body')); + provider.get('/body', (_req, res) => { + if (stream) { + res.json({ duplicate: true }); + return; + } + stream = res; + res.type('json').write('{"items":['); + arrived(); + }); + const { guard, advance } = createTestGuard(); + await withServer(provider, async (providerUrl) => { + for (let attempt = 0; attempt < 2; attempt++) { + const admission = guard.check(providerUrl); + if (!admission.allowed) + throw new Error('Expected initial admission'); + guard.reportFailure(admission.token); + } + advance(OPEN_DURATION_MS + 1); + await withServer( + createWebBackendApp({ + hostGuard: guard, + allowPrivateNetworkTargets: true, + }), + async (baseUrl) => { + const targetId = await registerProviderTarget( + baseUrl, + providerUrl + ); + const call = () => + fetch(`${baseUrl}/xtream?targetId=${targetId}`); + const trial = call(); + try { + await started; + advance(25_000); + stream.write('1,'); + advance(25_000); + const response = await call(); + const body = (await response.json()) as { + message: string; + }; + expect( + isHostConnectivityFastFailMessage(body.message) + ).toBe(true); + } finally { + stream?.end('2]}'); + await trial; + } + await expect((await trial).json()).resolves.toMatchObject({ + payload: { items: [1, 2] }, + }); + expect(guard.check(providerUrl)).toMatchObject({ + allowed: true, + token: { trial: false }, + }); + } + ); + }); + }); + + it.each(['xtream', 'stalker'])( + '%s releases its trial if outcome reporting throws', + async (route) => { + const httpClient = new StubHttpClient(); + const { advance, guard } = createTestGuard(); + httpClient.queueNetworkError(hostLevelFailure()); + httpClient.queueNetworkError(hostLevelFailure()); + httpClient.queueNetworkError(hostLevelFailure()); + httpClient.queueResponse([]); + await withServer( + createWebBackendApp({ + hostGuard: guard, + httpClient, + resolveHostname: resolvePublicHost, + }), + async (baseUrl) => { + const targetId = await registerProviderTarget( + baseUrl, + 'http://portal.example' + ); + const call = () => + fetch(`${baseUrl}/${route}?targetId=${targetId}`); + await call(); + await call(); + advance(OPEN_DURATION_MS + 1); + const report = jest + .spyOn(guard, 'reportFailure') + .mockImplementationOnce(() => { + throw new Error('outcome reporting failed'); + }); + try { + expect((await call()).status).toBe(500); + } finally { + report.mockRestore(); + } + await expect((await call()).json()).resolves.toMatchObject({ + payload: [], + }); + expect(httpClient.requests).toHaveLength(4); + } + ); + } + ); + + it.each(['xtream', 'stalker'])( + '%s holds a live trial beyond 45 seconds until the transport settles', + async (route) => { + for (const outcome of [ + 'success', + 'failure', + 'cancel', + 'redirect-failure', + ]) { + const httpClient = new StubHttpClient(); + const { advance, guard } = createTestGuard(); + httpClient.queueNetworkError(hostLevelFailure()); + httpClient.queueNetworkError(hostLevelFailure()); + await withServer( + createWebBackendApp({ + hostGuard: guard, + httpClient, + resolveHostname: resolvePublicHost, + }), + async (baseUrl) => { + const targetId = await registerProviderTarget( + baseUrl, + 'http://portal.example' + ); + const call = () => + fetch( + `${baseUrl}/${route}?targetId=${targetId}&action=get_genres` + ); + await call(); + await call(); + advance(OPEN_DURATION_MS + 1); + let settle!: () => void; + let arrived!: () => void; + const pending = new Promise((resolve) => { + settle = resolve; + }); + const started = new Promise((resolve) => { + arrived = resolve; + }); + const transport = jest + .spyOn(httpClient, 'get') + .mockImplementationOnce(async () => { + arrived(); + await pending; + if (outcome !== 'success') { + throw Object.assign( + hostLevelFailure( + outcome === 'cancel' + ? 'ERR_CANCELED' + : 'ETIMEDOUT' + ), + { + request: { + _currentUrl: + outcome === + 'redirect-failure' + ? 'http://cdn.example/slow' + : `http://portal.example/${route === 'xtream' ? 'player_api.php' : ''}`, + }, + } + ); + } + return { data: [] as never }; + }); + const trial = call(); + try { + await started; + // Advance only the guard clock; the HTTP route remains genuinely pending. + advance(45_001); + const blocked = await call(); + expect( + isHostConnectivityFastFailMessage( + ( + (await blocked.json()) as { + message: string; + } + ).message + ) + ).toBe(true); + expect(transport).toHaveBeenCalledTimes(1); + } finally { + settle(); + await trial; + transport.mockRestore(); + } + if (outcome === 'failure') { + const blocked = await call(); + expect( + isHostConnectivityFastFailMessage( + ( + (await blocked.json()) as { + message: string; + } + ).message + ) + ).toBe(true); + advance(OPEN_DURATION_MS + 1); + } + httpClient.queueResponse([]); + await expect( + (await call()).json() + ).resolves.toMatchObject({ payload: [] }); + expect(httpClient.requests).toHaveLength(3); + } + ); + } + } + ); + it('lets exactly one request through once the open window elapses', async () => { const httpClient = new StubHttpClient(); httpClient.queueNetworkError(hostLevelFailure()); diff --git a/apps/web-backend/src/app/web-backend-app.ts b/apps/web-backend/src/app/web-backend-app.ts index e30144e24..4ff17922a 100644 --- a/apps/web-backend/src/app/web-backend-app.ts +++ b/apps/web-backend/src/app/web-backend-app.ts @@ -352,6 +352,8 @@ export function createWebBackendApp( }); logProviderRequestFailure({ error, route: '/xtream', url }); res.json(normalizeProviderError(error)); + } finally { + releaseProviderRequest(hostGuard, guardToken); } }); @@ -459,6 +461,8 @@ export function createWebBackendApp( }); logProviderRequestFailure({ error, route: '/stalker', url }); res.json(normalizeProviderError(error)); + } finally { + releaseProviderRequest(hostGuard, guardToken); } }); diff --git a/docs/architecture/host-connectivity-guard.md b/docs/architecture/host-connectivity-guard.md index 24272136c..ecf2bd86b 100644 --- a/docs/architecture/host-connectivity-guard.md +++ b/docs/architecture/host-connectivity-guard.md @@ -65,9 +65,6 @@ timeouts but not the breaker, deliberately: add-playlist dialog sends `PLAYLIST_PARSE_BY_URL`). Refusing an immediate retry is a regression, not a protection, and there is no natural reset site on that path the way portal Retry has one. -- A large XMLTV transfer can legitimately run for minutes — the timeout is - idle-based, see above — which outlives the 45 s half-open trial expiry and - would let a second trial in behind the first. ## Desktop preference and account feedback @@ -95,8 +92,8 @@ show a localized **Requests temporarily paused** explanation with **Retry now**. Stalker keeps cached account data visible and offers the same retry beside the paused refresh notice. Retry resets before requesting again; actual network failures retain the generic unavailable state. These notices describe the last -request outcome, not a live countdown. The preference does not fix the separate -same-millisecond sibling-counting issue #1438. +request outcome, not a live countdown. Same-millisecond sibling counting is +handled by monotonic admission ids (#1438). ## Rules @@ -180,7 +177,7 @@ Two more rules exist because of specific failure modes: are never reused when an endpoint's state is evicted. A recreated record has no failure streak, so the first old in-flight failure can still count once; its remaining siblings cannot add further links to that streak. Timestamps - still determine streak expiry, cooldown and trial timeout. Only a failure that + still determine streak expiry and cooldown. Only a failure that was actually counted may trip the threshold: a sibling settling after the open window elapsed would otherwise start a fresh one off the existing count and push the half-open trial past the intended cooldown. @@ -190,14 +187,38 @@ Two more rules exist because of specific failure modes: behind settle right after they press Retry and re-open the breaker underneath the very retry that cleared it. -A half-open trial that never reports back expires after 45 s, so a leaked token -cannot leave the breaker open forever. That expiry is why the slot has an -identity: a trial can genuinely outlive its window — `requestWithValidatedRedirects` -gives each of up to five redirect hops its own 30 s budget — and once a -replacement has been admitted, the abandoned request's late report must not free -the replacement's slot and let a third request through. `trial: true` alone -cannot tell the two apart, so the token carries the slot id it owns; an -abandoned trial's failure is still counted as ordinary evidence. +### Trial ownership follows the request lifetime + +A half-open slot has no wall-clock expiry (#1439). A redirect chain gives each +hop a separate timeout, and a body that keeps delivering bytes can exceed any +fixed deadline without reaching its inactivity timeout. Neither may admit a +second trial while the first request remains pending. + +The four request owners (Electron Xtream/Stalker IPC and web-backend +`/xtream`/`/stalker`) acquire inside `try`, await the whole transport operation, +report its outcome, and unconditionally release any remaining token in `finally`. +`reportInconclusive` is the idempotent cleanup operation: it releases only the +matching `trialId` and epoch and records no reachability evidence. This is the +leak backstop even when debug logging, classification, or response formatting +throws before an outcome report. Cleanup also runs while the environment kill +switch is on, so switching it back off cannot revive a completed trial's slot. +Desktop preference transitions invalidate the old guard and tokens as before. + +Cancellation releases after the awaited transport rejects, when it actually +settles, rather than merely when an abort is requested. Xtream's AbortSignal +continues through all Electron redirect hops. Stalker has no IPC cancellation +signal, and closing a PWA client connection does not cancel the backend's outbound +request; those slots remain owned until that transport settles. The transport's +existing timeout handles inactivity. No heartbeat, polling timer, or larger +trial deadline is needed. A future caller must preserve the `try`/`finally` +contract; a custom transport that never settles needs cancellation at its own +layer and must not be silently overlapped by a second trial. + +The slot still has an identity: delayed cleanup from a released owner must not +free its replacement. A late failure still follows the ordinary streak and epoch +rules; a real HTTP response still proves reachability and clears the record. +Explicit reset, discovery success, preference transitions and bounded state +eviction retain their existing rules for forgetting evidence. ## The fast-fail error is a renderer contract @@ -347,7 +368,8 @@ module's contract that unreachable ≠ contacted-and-refused. The ## Tests - `libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts` — the state - machine, with an injected clock. + machine, with an injected clock, long-lived trial ownership, late cleanup, + and settlement while the environment kill switch is on. - `apps/electron-backend/src/app/util/host-connectivity-guard.spec.ts` — the main-process singleton and redirect attribution through it. - `apps/web-backend/src/app/web-backend-app.host-guard.spec.ts` — the proxy @@ -362,6 +384,7 @@ module's contract that unreachable ≠ contacted-and-refused. The that the discovery bypass is forwarded, and that a reset calls the backend. - `apps/electron-backend/src/app/events/stalker.events.spec.ts` and `xtream.events.spec.ts` — trip, fast-fail without contacting axios, reset, - exemption, and the absence of per-request log spam. + exemption, long pending requests and redirect chains, cancellation, cleanup + when debug reporting throws, and the absence of per-request log spam. - `libs/portal/stalker/data-access/src/lib/stalker-portal-discovery.utils.spec.ts` and `stalker-portal-repair.service.spec.ts` — the message contract. diff --git a/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts b/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts index d8c5b0e91..a35c30ee7 100644 --- a/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts +++ b/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts @@ -15,7 +15,6 @@ const HOST = 'http://portal.example.com:8080'; const GUARD_DISABLED_ENV = 'IPTVNATOR_DISABLE_CONNECTIVITY_GUARD'; const OPEN_DURATION_MS = 30_000; const FAILURE_WINDOW_MS = 120_000; -const TRIAL_TIMEOUT_MS = 45_000; function timeoutError(code = 'ETIMEDOUT') { return Object.assign(new Error('timeout of 30000ms exceeded'), { code }); @@ -431,6 +430,32 @@ describe('HostConnectivityGuard', () => { expect(guard.check(HOST).allowed).toBe(false); }); + it('keeps a live trial exclusive beyond 45 seconds and the idle TTL', () => { + const trial = expectAllowed(); + advance(45_001); + expectBlocked(); + advance(600_001); + guard.check('http://another.example'); + expectBlocked(); + guard.reportSuccess(trial); + expect(expectAllowed().trial).toBe(false); + }); + + it.each([ + 'reportSuccess', + 'reportFailure', + 'reportInconclusive', + ] as const)( + 'releases a completed trial via %s while the environment override is disabled', + (report) => { + const trial = expectAllowed(); + process.env[GUARD_DISABLED_ENV] = '1'; + guard[report](trial); + delete process.env[GUARD_DISABLED_ENV]; + expect(expectAllowed().trial).toBe(true); + } + ); + it('closes the breaker when the trial succeeds', () => { const trial = expectAllowed(); guard.reportSuccess(trial); @@ -461,12 +486,9 @@ describe('HostConnectivityGuard', () => { }); it('lets only the request holding the slot release it', () => { - // A trial can outlive its own 45 s window: the validated-redirect - // transport gives each of up to five hops its own 30 s budget. Once - // a replacement has been admitted, the abandoned request's late - // report must not hand the slot to a third request. + // Cleanup may run again after a replacement has acquired the slot. const abandoned = expectAllowed(); - advance(TRIAL_TIMEOUT_MS); + guard.reportInconclusive(abandoned); const replacement = expectAllowed(); expect(replacement.trial).toBe(true); @@ -475,11 +497,24 @@ describe('HostConnectivityGuard', () => { expectBlocked(); }); - it('recovers when a trial never reports back', () => { - expectAllowed(); + it('keeps the replacement slot when the old owner reports a late failure', () => { + const old = expectAllowed(); + guard.reportInconclusive(old); + const replacement = expectAllowed(); + guard.reportFailure(old); + advance(OPEN_DURATION_MS + 1); expectBlocked(); + guard.reportInconclusive(old); + expectBlocked(); + guard.reportSuccess(replacement); + expect(expectAllowed().trial).toBe(false); + }); - advance(TRIAL_TIMEOUT_MS); + it('recovers when the owner releases a trial without an outcome report', () => { + const trial = expectAllowed(); + expectBlocked(); + // The request owner's finally is independent of outcome reporting. + guard.reportInconclusive(trial); const replacement = expectAllowed(); expect(replacement.trial).toBe(true); diff --git a/libs/shared/host-health/src/lib/host-connectivity-guard.ts b/libs/shared/host-health/src/lib/host-connectivity-guard.ts index fd264d0e3..2661ac64f 100644 --- a/libs/shared/host-health/src/lib/host-connectivity-guard.ts +++ b/libs/shared/host-health/src/lib/host-connectivity-guard.ts @@ -46,12 +46,6 @@ export const OPEN_DURATION_MS = 30_000; * 30 s timeouts must fit inside it, hence comfortably above 60 s. */ const FAILURE_WINDOW_MS = 120_000; -/** - * Safety net for a half-open trial that never reports back. Above the longest - * request timeout (30 s) plus margin, so it only fires if a caller leaked the - * token — without it a lost report would keep the breaker open forever. - */ -const TRIAL_TIMEOUT_MS = 45_000; /** Hosts tracked at once; portals per user are few, this is just a bound. */ const MAX_TRACKED_HOSTS = 256; /** Idle records are forgotten; they hold nothing worth remembering. */ @@ -111,11 +105,8 @@ export interface HostRequestToken { /** * Which half-open slot this request holds, when it holds one. * - * A trial can outlive its own window — `requestWithValidatedRedirects` - * gives each of up to five redirect hops its own 30 s budget — and once a - * replacement has been admitted, the abandoned request must not be able to - * free the replacement's slot. `trial: true` alone cannot tell the two - * apart, so the owner is identified explicitly. + * Cleanup from a released owner can arrive after another trial has been + * admitted. Only this identity (plus epoch) may release the current slot. */ readonly trialId: number; } @@ -133,7 +124,7 @@ interface HostState { /** Latest guard-wide admission id when this host's last failure counted. */ lastFailureAdmissionId: number; openUntil: number; - trialStartedAt: number | null; + trialInFlight: boolean; /** Monotonic id of the half-open slot; see `HostRequestToken.trialId`. */ trialId: number; epoch: number; @@ -247,13 +238,10 @@ export class HostConnectivityGuard { // let one request through and hold the rest back until it settles. let trial = false; if (state.openUntil > 0) { - const trialStale = - state.trialStartedAt !== null && - now - state.trialStartedAt >= TRIAL_TIMEOUT_MS; - if (state.trialStartedAt !== null && !trialStale) { + if (state.trialInFlight) { return { allowed: false, retryAfterMs: 0 }; } - state.trialStartedAt = now; + state.trialInFlight = true; state.trialId += 1; trial = true; } @@ -276,21 +264,29 @@ export class HostConnectivityGuard { */ reportSuccess(token: HostRequestToken): void { const state = this.states.get(token.endpoint); - if (!state || isHostConnectivityGuardDisabled()) { + if (!state) { + return; + } + if (isHostConnectivityGuardDisabled()) { + this.releaseTrial(state, token); return; } state.consecutiveFailures = 0; state.lastFailureAt = 0; state.openUntil = 0; - state.trialStartedAt = null; + state.trialInFlight = false; state.lastTouchedAt = this.now(); } /** The host did not answer at all. */ reportFailure(token: HostRequestToken): void { const state = this.states.get(token.endpoint); - if (!state || isHostConnectivityGuardDisabled()) { + if (!state) { + return; + } + if (isHostConnectivityGuardDisabled()) { + this.releaseTrial(state, token); return; } @@ -307,7 +303,7 @@ export class HostConnectivityGuard { // through the streak rules below and leaves the replacement alone. const wasTrial = this.ownsTrial(state, token); if (wasTrial) { - state.trialStartedAt = null; + state.trialInFlight = false; } state.lastTouchedAt = now; @@ -354,10 +350,13 @@ export class HostConnectivityGuard { /** * The request failed for a reason that says nothing about the host. Only * releases the half-open slot, so the next request can be the trial. + * Owners MUST also call this in finally, independently of outcome reporting. + * It is idempotent and still releases while the guard is disabled by the environment; + * no elapsed-time expiry can distinguish a leak from an active transfer. */ reportInconclusive(token: HostRequestToken): void { const state = this.states.get(token.endpoint); - if (!state || isHostConnectivityGuardDisabled()) { + if (!state) { return; } @@ -376,7 +375,7 @@ export class HostConnectivityGuard { state.consecutiveFailures = 0; state.lastFailureAt = 0; state.openUntil = 0; - state.trialStartedAt = null; + state.trialInFlight = false; state.epoch += 1; state.lastTouchedAt = now; } @@ -409,7 +408,7 @@ export class HostConnectivityGuard { private ownsTrial(state: HostState, token: HostRequestToken): boolean { return ( token.trial && - state.trialStartedAt !== null && + state.trialInFlight && state.epoch === token.epoch && state.trialId === token.trialId ); @@ -417,7 +416,7 @@ export class HostConnectivityGuard { private releaseTrial(state: HostState, token: HostRequestToken): void { if (this.ownsTrial(state, token)) { - state.trialStartedAt = null; + state.trialInFlight = false; } } @@ -433,7 +432,7 @@ export class HostConnectivityGuard { lastFailureAt: 0, lastFailureAdmissionId: 0, openUntil: 0, - trialStartedAt: null, + trialInFlight: false, trialId: 0, epoch: 0, lastTouchedAt: now, @@ -447,7 +446,7 @@ export class HostConnectivityGuard { if ( now - state.lastTouchedAt > IDLE_TTL_MS && state.openUntil <= now && - state.trialStartedAt === null + !state.trialInFlight ) { this.states.delete(endpoint); }