fix(portals): count concurrent connection failures once (#1537)

* fix(portals): count concurrent connection failures once

* docs(portals): clarify failure counting after guard eviction

.changes/portals-concurrent-connectivity-failures.md

@@ -0,0 +1,7 @@
+---
+type: fix
+area: portals
+issues: [1438]
+---
+
+Xtream and Stalker portals no longer enter a connection cooldown merely because several parallel requests fail in the same millisecond. This fixes false unavailability in both the desktop app and the self-hosted web version.

CLAUDE.md

@@ -345,7 +345,7 @@ This is an Nx monorepo with the following structure:
     - **services** - Abstract DataService contract and shared app services (incl. the TMDB metadata enrichment module in `lib/tmdb/`)
     - **shared/interfaces** - TypeScript interfaces and types (incl. `ElectronBridgeApi`)
     - **shared/logging** - Dependency-free structured redaction for diagnostic logs
-    - **shared/host-health** - Per-host circuit breaker for portal requests (`HostConnectivityGuard`), shared by the Electron main process and the web backend; transport-free, the owning app supplies the clock and owns the instance
+    - **shared/host-health** - Per-host circuit breaker for portal requests (`HostConnectivityGuard`), shared by the Electron main process and the web backend; transport-free, the owning app supplies the clock and owns the instance. Monotonic admission ids with per-endpoint failure boundaries distinguish parallel failures from later attempts even within one clock tick (#1438)
     - **shared/database** - Canonical Drizzle schema and DB connection (used by the Electron backend)
     - **shared/m3u-utils** - M3U playlist utilities
     - **shared/marketing-fixtures** - Provider-neutral fictional movie metadata, live channel list and the generated channel-logo SVG renderer shared by the Xtream and Stalker marketing mocks (both serve `/assets/marketing/logo/<slug>.svg`)

docs/architecture/host-connectivity-guard.md

@@ -170,10 +170,17 @@ Two more rules exist because of specific failure modes:
 
 - **Siblings are not a streak.** Catalog initialization fans out three category
   requests at once; one network hiccup failing all three is one piece of
-  evidence, not a trip. A failure counts only if its request started at or after
-  the moment the previous failure was recorded. Timestamps are millisecond
-  coarse, so this only separates siblings once a request actually took time —
-  which is precisely the expensive case worth protecting. Only a failure that
+  evidence, not a trip. Each admitted request receives a guard-wide monotonic
+  admission id. A counted failure saves the highest id admitted so far in that
+  endpoint's failure record, not just the id of the request that failed. During
+  that streak, only a request admitted beyond this boundary can add another
+  failure. This distinguishes
+  siblings from later attempts even if admission and completion all happen in
+  the same millisecond (#1438), regardless of completion order. Admission ids
+  are never reused when an endpoint's state is evicted. A recreated record has
+  no failure streak, so the first old in-flight failure can still count once;
+  its remaining siblings cannot add further links to that streak. Timestamps
+  still determine streak expiry, cooldown and trial timeout. Only a failure that
   was actually counted may trip the threshold: a sibling settling after the open
   window elapsed would otherwise start a fresh one off the existing count and
   push the half-open trial past the intended cooldown.

libs/shared/host-health/src/lib/host-connectivity-guard.spec.ts

@@ -80,7 +80,7 @@ describe('failedAfterRedirect', () => {
     const token: HostRequestToken = {
         endpoint: ENDPOINT,
         epoch: 0,
-        startedAt: 0,
+        admissionId: 0,
         trial: false,
         trialId: 0,
     };
@@ -276,6 +276,62 @@ describe('HostConnectivityGuard', () => {
         expect(opened).toEqual([]);
     });
 
+    it.each([
+        [0, 1, 2],
+        [2, 0, 1],
+        [1, 2, 0],
+    ])(
+        'counts same-millisecond siblings once in completion order %s, %s, %s',
+        (...order) => {
+            const tokens = [expectAllowed(), expectAllowed(), expectAllowed()];
+
+            for (const index of order) {
+                guard.reportFailure(tokens[index]);
+            }
+
+            expectAllowed();
+            expect(opened).toEqual([]);
+        }
+    );
+
+    it('keeps old siblings together after their endpoint state is evicted', () => {
+        const first = expectAllowed();
+        const sibling = expectAllowed();
+        for (let index = 0; index < 256; index += 1) {
+            guard.check(`http://other-${index}.example`);
+        }
+        expectAllowed();
+        advance(30_000);
+
+        guard.reportFailure(first);
+        guard.reportFailure(sibling);
+
+        expectAllowed();
+        expect(opened).toEqual([]);
+    });
+
+    it('counts a new attempt after a failure even in the same millisecond', () => {
+        failRequest();
+        failRequest();
+
+        expectBlocked();
+        expect(opened).toEqual([HOST]);
+    });
+
+    it('does not extend the cooldown for same-millisecond siblings', () => {
+        const first = expectAllowed();
+        const sibling = expectAllowed();
+        guard.reportFailure(first);
+        failRequest();
+        expectBlocked();
+
+        advance(OPEN_DURATION_MS);
+        guard.reportFailure(sibling);
+
+        expect(expectAllowed().trial).toBe(true);
+        expect(opened).toEqual([HOST]);
+    });
+
     it('opens when a second fan-out fails after the first one did', () => {
         const first = expectAllowed();
         const second = expectAllowed();
@@ -537,4 +593,3 @@ describe('HostConnectivityGuard', () => {
         });
     });
 });
-

libs/shared/host-health/src/lib/host-connectivity-guard.ts

@@ -104,7 +104,8 @@ export interface HostRequestToken {
     /** Scheme, host and port — see {@link portalEndpointKeyOf}. */
     readonly endpoint: string;
     readonly epoch: number;
-    readonly startedAt: number;
+    /** Admission order within this guard, independent of clock precision. */
+    readonly admissionId: number;
     /** Whether this request is the single probe allowed while half-open. */
     readonly trial: boolean;
     /**
@@ -129,6 +130,8 @@ interface HostState {
     consecutiveFailures: number;
     /** When the last COUNTED failure was recorded. */
     lastFailureAt: number;
+    /** Latest guard-wide admission id when this host's last failure counted. */
+    lastFailureAdmissionId: number;
     openUntil: number;
     trialStartedAt: number | null;
     /** Monotonic id of the half-open slot; see `HostRequestToken.trialId`. */
@@ -201,6 +204,8 @@ export function portalEndpointKeyOf(url: string): string | null {
 
 export class HostConnectivityGuard {
     private readonly states = new Map<string, HostState>();
+    /** Never reused when an endpoint is evicted while requests are in flight. */
+    private admissionId = 0;
     private readonly now: () => number;
     private readonly onOpen?: (endpoint: string) => void;
 
@@ -224,7 +229,7 @@ export class HostConnectivityGuard {
                 token: {
                     endpoint,
                     epoch: 0,
-                    startedAt: now,
+                    admissionId: 0,
                     trial: false,
                     trialId: 0,
                 },
@@ -258,7 +263,7 @@ export class HostConnectivityGuard {
             token: {
                 endpoint,
                 epoch: state.epoch,
-                startedAt: now,
+                admissionId: ++this.admissionId,
                 trial,
                 trialId: trial ? state.trialId : 0,
             },
@@ -316,15 +321,16 @@ export class HostConnectivityGuard {
         // A request already in flight when the previous failure was recorded is
         // not the next link in a streak — it is a sibling of it. Catalog
         // loading fans out several requests at once, and one hiccup failing all
-        // of them is one piece of evidence, not a trip. A request that started
-        // at or after that moment is a genuine new attempt.
+        // of them is one piece of evidence, not a trip. Admission order makes
+        // that boundary exact even when all events share one clock tick.
         let counted = false;
         if (
             state.consecutiveFailures === 0 ||
-            token.startedAt >= state.lastFailureAt
+            token.admissionId > state.lastFailureAdmissionId
         ) {
             state.consecutiveFailures += 1;
             state.lastFailureAt = now;
+            state.lastFailureAdmissionId = this.admissionId;
             counted = true;
         }
 
@@ -388,7 +394,7 @@ export class HostConnectivityGuard {
         return {
             endpoint,
             epoch: this.states.get(endpoint)?.epoch ?? 0,
-            startedAt: this.now(),
+            admissionId: 0,
             trial: false,
             trialId: 0,
         };
@@ -425,6 +431,7 @@ export class HostConnectivityGuard {
         const state: HostState = {
             consecutiveFailures: 0,
             lastFailureAt: 0,
+            lastFailureAdmissionId: 0,
             openUntil: 0,
             trialStartedAt: null,
             trialId: 0,