diff --git a/.changes/portals-concurrent-connectivity-failures.md b/.changes/portals-concurrent-connectivity-failures.md new file mode 100644 index 000000000..8b4324169 --- /dev/null +++ b/.changes/portals-concurrent-connectivity-failures.md @@ -0,0 +1,7 @@ +--- +type: fix +area: portals +issues: [1438] +--- + +Xtream and Stalker portals no longer enter a connection cooldown merely because several parallel requests fail in the same millisecond. This fixes false unavailability in both the desktop app and the self-hosted web version. diff --git a/CLAUDE.md b/CLAUDE.md index 75b4f3520..126d1743c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -345,7 +345,7 @@ This is an Nx monorepo with the following structure: - **services** - Abstract DataService contract and shared app services (incl. the TMDB metadata enrichment module in `lib/tmdb/`) - **shared/interfaces** - TypeScript interfaces and types (incl. `ElectronBridgeApi`) - **shared/logging** - Dependency-free structured redaction for diagnostic logs - - **shared/host-health** - Per-host circuit breaker for portal requests (`HostConnectivityGuard`), shared by the Electron main process and the web backend; transport-free, the owning app supplies the clock and owns the instance + - **shared/host-health** - Per-host circuit breaker for portal requests (`HostConnectivityGuard`), shared by the Electron main process and the web backend; transport-free, the owning app supplies the clock and owns the instance. Monotonic admission ids with per-endpoint failure boundaries distinguish parallel failures from later attempts even within one clock tick (#1438) - **shared/database** - Canonical Drizzle schema and DB connection (used by the Electron backend) - **shared/m3u-utils** - M3U playlist utilities - **shared/marketing-fixtures** - Provider-neutral fictional movie metadata, live channel list and the generated channel-logo SVG renderer shared by the Xtream and Stalker marketing mocks (both serve `/assets/marketing/logo/.svg`) diff --git a/docs/architecture/host-connectivity-guard.md b/docs/architecture/host-connectivity-guard.md index 63f9a0190..24272136c 100644 --- a/docs/architecture/host-connectivity-guard.md +++ b/docs/architecture/host-connectivity-guard.md @@ -170,10 +170,17 @@ Two more rules exist because of specific failure modes: - **Siblings are not a streak.** Catalog initialization fans out three category requests at once; one network hiccup failing all three is one piece of - evidence, not a trip. A failure counts only if its request started at or after - the moment the previous failure was recorded. Timestamps are millisecond - coarse, so this only separates siblings once a request actually took time — - which is precisely the expensive case worth protecting. Only a failure that + evidence, not a trip. Each admitted request receives a guard-wide monotonic + admission id. A counted failure saves the highest id admitted so far in that + endpoint's failure record, not just the id of the request that failed. During + that streak, only a request admitted beyond this boundary can add another + failure. This distinguishes + siblings from later attempts even if admission and completion all happen in + the same millisecond (#1438), regardless of completion order. Admission ids + are never reused when an endpoint's state is evicted. A recreated record has + no failure streak, so the first old in-flight failure can still count once; + its remaining siblings cannot add further links to that streak. Timestamps + still determine streak expiry, cooldown and trial timeout. Only a failure that was actually counted may trip the threshold: a sibling settling after the open window elapsed would otherwise start a fresh one off the existing count and push the half-open trial past the intended cooldown. diff --git a/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts b/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts index cd06f2786..d8c5b0e91 100644 --- a/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts +++ b/libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts @@ -80,7 +80,7 @@ describe('failedAfterRedirect', () => { const token: HostRequestToken = { endpoint: ENDPOINT, epoch: 0, - startedAt: 0, + admissionId: 0, trial: false, trialId: 0, }; @@ -276,6 +276,62 @@ describe('HostConnectivityGuard', () => { expect(opened).toEqual([]); }); + it.each([ + [0, 1, 2], + [2, 0, 1], + [1, 2, 0], + ])( + 'counts same-millisecond siblings once in completion order %s, %s, %s', + (...order) => { + const tokens = [expectAllowed(), expectAllowed(), expectAllowed()]; + + for (const index of order) { + guard.reportFailure(tokens[index]); + } + + expectAllowed(); + expect(opened).toEqual([]); + } + ); + + it('keeps old siblings together after their endpoint state is evicted', () => { + const first = expectAllowed(); + const sibling = expectAllowed(); + for (let index = 0; index < 256; index += 1) { + guard.check(`http://other-${index}.example`); + } + expectAllowed(); + advance(30_000); + + guard.reportFailure(first); + guard.reportFailure(sibling); + + expectAllowed(); + expect(opened).toEqual([]); + }); + + it('counts a new attempt after a failure even in the same millisecond', () => { + failRequest(); + failRequest(); + + expectBlocked(); + expect(opened).toEqual([HOST]); + }); + + it('does not extend the cooldown for same-millisecond siblings', () => { + const first = expectAllowed(); + const sibling = expectAllowed(); + guard.reportFailure(first); + failRequest(); + expectBlocked(); + + advance(OPEN_DURATION_MS); + guard.reportFailure(sibling); + + expect(expectAllowed().trial).toBe(true); + expect(opened).toEqual([HOST]); + }); + it('opens when a second fan-out fails after the first one did', () => { const first = expectAllowed(); const second = expectAllowed(); @@ -537,4 +593,3 @@ describe('HostConnectivityGuard', () => { }); }); }); - diff --git a/libs/shared/host-health/src/lib/host-connectivity-guard.ts b/libs/shared/host-health/src/lib/host-connectivity-guard.ts index c2c85dae8..fd264d0e3 100644 --- a/libs/shared/host-health/src/lib/host-connectivity-guard.ts +++ b/libs/shared/host-health/src/lib/host-connectivity-guard.ts @@ -104,7 +104,8 @@ export interface HostRequestToken { /** Scheme, host and port — see {@link portalEndpointKeyOf}. */ readonly endpoint: string; readonly epoch: number; - readonly startedAt: number; + /** Admission order within this guard, independent of clock precision. */ + readonly admissionId: number; /** Whether this request is the single probe allowed while half-open. */ readonly trial: boolean; /** @@ -129,6 +130,8 @@ interface HostState { consecutiveFailures: number; /** When the last COUNTED failure was recorded. */ lastFailureAt: number; + /** Latest guard-wide admission id when this host's last failure counted. */ + lastFailureAdmissionId: number; openUntil: number; trialStartedAt: number | null; /** Monotonic id of the half-open slot; see `HostRequestToken.trialId`. */ @@ -201,6 +204,8 @@ export function portalEndpointKeyOf(url: string): string | null { export class HostConnectivityGuard { private readonly states = new Map(); + /** Never reused when an endpoint is evicted while requests are in flight. */ + private admissionId = 0; private readonly now: () => number; private readonly onOpen?: (endpoint: string) => void; @@ -224,7 +229,7 @@ export class HostConnectivityGuard { token: { endpoint, epoch: 0, - startedAt: now, + admissionId: 0, trial: false, trialId: 0, }, @@ -258,7 +263,7 @@ export class HostConnectivityGuard { token: { endpoint, epoch: state.epoch, - startedAt: now, + admissionId: ++this.admissionId, trial, trialId: trial ? state.trialId : 0, }, @@ -316,15 +321,16 @@ export class HostConnectivityGuard { // A request already in flight when the previous failure was recorded is // not the next link in a streak — it is a sibling of it. Catalog // loading fans out several requests at once, and one hiccup failing all - // of them is one piece of evidence, not a trip. A request that started - // at or after that moment is a genuine new attempt. + // of them is one piece of evidence, not a trip. Admission order makes + // that boundary exact even when all events share one clock tick. let counted = false; if ( state.consecutiveFailures === 0 || - token.startedAt >= state.lastFailureAt + token.admissionId > state.lastFailureAdmissionId ) { state.consecutiveFailures += 1; state.lastFailureAt = now; + state.lastFailureAdmissionId = this.admissionId; counted = true; } @@ -388,7 +394,7 @@ export class HostConnectivityGuard { return { endpoint, epoch: this.states.get(endpoint)?.epoch ?? 0, - startedAt: this.now(), + admissionId: 0, trial: false, trialId: 0, }; @@ -425,6 +431,7 @@ export class HostConnectivityGuard { const state: HostState = { consecutiveFailures: 0, lastFailureAt: 0, + lastFailureAdmissionId: 0, openUntil: 0, trialStartedAt: null, trialId: 0,