ci: wait for nightly builds before publishing the release (#2695)

GITHUB_TOKEN-dispatched builds do not trigger downstream workflow_run
jobs. The nightly release check ran before builds finished and then
exited successfully without publishing anything.

Track dispatch run IDs and wait for all missing components to succeed.
Retry failed jobs once, account for delayed rerun state, and fail on
terminal errors. Bound the build and wait job to three hours.

Pass the built commit to publication so later main commits do not
change the release target. Retain component deduplication and manual
build completion handlers, and document the nightly lifecycle.

Add regression coverage for pending builds, retries, terminal failures,
dispatch races, and main advancing before publication. All 34 Node tests
and actionlint for the three nightly workflows pass.

.github/scripts/builds.test.js

@@ -7,8 +7,8 @@ const context = { repo: { owner: 'EasyTier', repo: 'EasyTier' } };
 const core = { info() {} };
 const workflows = ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml'];
 
-async function nightly(branches, successfulRuns = {}) {
-  const dispatched = [];
+function nightly(branches, successfulRuns = {}) {
+  const f = { dispatched: [], retried: [], responses: {}, inspected: [], sleeps: 0 };
   const github = {
     paginate: async () => branches,
     rest: {
@@ -21,47 +21,118 @@ async function nightly(branches, successfulRuns = {}) {
           return { data: { workflow_runs: sha ? [{ head_sha: sha }] : [] } };
         },
         async createWorkflowDispatch(request) {
-          dispatched.push(request);
+          assert.equal(request.headers['X-GitHub-Api-Version'], '2026-03-10');
+          f.dispatched.push(request);
+          return { data: { workflow_run_id: f.dispatched.length, html_url: `run/${f.dispatched.length}` } };
+        },
+        async getWorkflowRun({ run_id }) {
+          const workflow = f.dispatched[run_id - 1].workflow_id;
+          f.inspected.push({ workflow, dispatched: f.dispatched.length });
+          return { data: {
+            head_sha: branches.find(branch => branch.name === 'main').commit.sha,
+            status: 'completed', conclusion: 'success', run_attempt: 1,
+            ...f.responses[workflow]?.shift(),
+          } };
+        },
+        async reRunWorkflowFailedJobs({ run_id }) {
+          f.retried.push(run_id);
         },
       },
     },
   };
-  await dispatch({ github, context, core });
-  return dispatched;
+  f.run = () => dispatch({ github, context, core, sleep: async delay => {
+    assert.equal(delay, 60_000);
+    assert.ok(++f.sleeps < 10, 'nightly should eventually finish');
+  } });
+  return f;
 }
 
 test('first nightly builds every workflow on an existing main branch', async () => {
-  const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }]);
-  assert.deepEqual(runs.map(run => run.workflow_id), workflows);
-  assert.ok(runs.every(run => run.ref === 'main'));
-  assert.deepEqual(runs[0].inputs, { sign_windows: false });
-  assert.ok(runs.slice(1).every(run => !run.inputs));
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  assert.equal(await f.run(), 'new');
+  assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows);
+  assert.ok(f.dispatched.every(run => run.ref === 'main'));
+  assert.deepEqual(f.dispatched[0].inputs, { sign_windows: false });
+  assert.ok(f.dispatched.slice(1).every(run => !run.inputs));
+  assert.deepEqual(f.inspected.map(run => run.workflow), workflows);
+  assert.ok(f.inspected.every(run => run.dispatched === 4));
 });
 
 test('unchanged branches skip builds and unmonitored branches are ignored', async () => {
   const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'same']));
-  const runs = await nightly([
+  const f = nightly([
     { name: 'main', commit: { sha: 'same' } },
     { name: 'feature', commit: { sha: 'new' } },
   ], successfulRuns);
-  assert.deepEqual(runs, []);
+  assert.equal(await f.run(), 'same');
+  assert.deepEqual(f.dispatched, []);
+  assert.equal(f.sleeps, 0);
 });
 
 test('nightly retries a failed component using its last successful commit', async () => {
   const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'new']));
   successfulRuns['main/gui.yml'] = 'old';
-  const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns);
-  assert.deepEqual(runs.map(run => run.workflow_id), ['gui.yml']);
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns);
+  assert.equal(await f.run(), 'new');
+  assert.deepEqual(f.dispatched.map(run => run.workflow_id), ['gui.yml']);
 });
 
 test('nightly only builds main even when other branches exist', async () => {
-  const runs = await nightly([
+  const f = nightly([
     { name: 'main', commit: { sha: 'new' } },
     { name: 'develop', commit: { sha: 'new' } },
     { name: 'feature', commit: { sha: 'new' } },
   ]);
-  assert.deepEqual(runs.map(run => run.workflow_id), workflows);
-  assert.ok(runs.every(run => run.ref === 'main'));
+  await f.run();
+  assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows);
+  assert.ok(f.dispatched.every(run => run.ref === 'main'));
+});
+
+test('nightly waits for every dispatched build before allowing publication', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['core.yml'] = [{ status: 'queued' }, { status: 'in_progress' }, {}];
+  f.responses['test.yml'] = [{ status: 'in_progress' }, {}];
+  assert.equal(await f.run(), 'new');
+  assert.equal(f.sleeps, 2);
+  assert.deepEqual(f.inspected.map(run => run.workflow), [
+    ...workflows, 'core.yml', 'test.yml', 'core.yml',
+  ]);
+});
+
+test('nightly retries failed jobs once and waits for the new attempt', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['gui.yml'] = [
+    { conclusion: 'failure' },
+    { conclusion: 'failure' }, // The API still returns the first attempt.
+    { status: 'in_progress', run_attempt: 2 },
+    { run_attempt: 2 },
+  ];
+  assert.equal(await f.run(), 'new');
+  assert.deepEqual(f.retried, [2]);
+  assert.equal(f.sleeps, 3);
+});
+
+test('nightly fails instead of publishing if a retried build fails again', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['mobile.yml'] = [{ conclusion: 'failure' }, { conclusion: 'failure', run_attempt: 2 }];
+  await assert.rejects(f.run(), /mobile.yml: run 3 finished with failure/);
+  assert.deepEqual(f.retried, [3]);
+});
+
+test('nightly does not retry cancelled or timed-out builds', async () => {
+  for (const conclusion of ['cancelled', 'timed_out']) {
+    const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+    f.responses['test.yml'] = [{ conclusion }];
+    await assert.rejects(f.run(), new RegExp(`test.yml: run 4 finished with ${conclusion}`));
+    assert.deepEqual(f.retried, []);
+  }
+});
+
+test('nightly rejects a dispatch that races with a new main commit', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['gui.yml'] = [{ head_sha: 'newer', status: 'queued' }];
+  await assert.rejects(f.run(), /main changed while dispatching/);
+  assert.deepEqual(f.retried, []);
 });
 
 process.env.CORE_RUN_ID = '1';

.github/scripts/nightly-build.js

@@ -1,4 +1,6 @@
-module.exports = async ({ github, context, core }) => {
+const { setTimeout } = require('node:timers/promises');
+
+module.exports = async ({ github, context, core, sleep = setTimeout }) => {
   const repository = context.repo;
   const branches = await github.paginate(github.rest.repos.listBranches, {
     ...repository,
@@ -6,6 +8,7 @@ module.exports = async ({ github, context, core }) => {
   });
 
   for (const branch of branches.filter(({ name }) => name === 'main')) {
+    const pending = new Map();
     for (const workflow of ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml']) {
       const { data } = await github.rest.actions.listWorkflowRuns({
         ...repository,
@@ -20,13 +23,48 @@ module.exports = async ({ github, context, core }) => {
         continue;
       }
 
-      await github.rest.actions.createWorkflowDispatch({
+      const { data: run } = await github.rest.actions.createWorkflowDispatch({
         ...repository,
         workflow_id: workflow,
         ref: branch.name,
+        // This API version returns the dispatched run ID.
+        headers: { 'X-GitHub-Api-Version': '2026-03-10' },
         ...(workflow === 'core.yml' ? { inputs: { sign_windows: false } } : {}),
       });
-      core.info(`${workflow}: dispatched a full build for ${branch.name}.`);
+      pending.set(workflow, { id: run.workflow_run_id, attempt: 1 });
+      core.info(`${workflow}: dispatched ${run.html_url} for ${branch.name}.`);
     }
+
+    // GITHUB_TOKEN dispatches do not trigger downstream workflow_run jobs.
+    // Wait here so publication and retries do not depend on those events.
+    while (pending.size) {
+      for (const [workflow, build] of pending) {
+        const { data: run } = await github.rest.actions.getWorkflowRun({
+          ...repository, run_id: build.id,
+        });
+        if (run.head_sha !== branch.commit.sha) {
+          throw new Error(`${workflow}: main changed while dispatching; run Nightly again.`);
+        }
+        // A rerun request can return before the new attempt becomes visible.
+        if (run.run_attempt < build.attempt || run.status !== 'completed') continue;
+        if (run.conclusion === 'success') {
+          core.info(`${workflow}: run ${build.id} succeeded.`);
+          pending.delete(workflow);
+        } else if (run.conclusion === 'failure' && run.run_attempt === 1) {
+          await github.rest.actions.reRunWorkflowFailedJobs({
+            ...repository, run_id: build.id,
+          });
+          build.attempt = 2;
+          core.info(`${workflow}: retrying failed jobs in run ${build.id}.`);
+        } else {
+          throw new Error(`${workflow}: run ${build.id} finished with ${run.conclusion}.`);
+        }
+      }
+      if (pending.size) {
+        core.info(`Waiting for ${[...pending.keys()].join(', ')} at ${branch.commit.sha}.`);
+        await sleep(60_000);
+      }
+    }
+    return branch.commit.sha;
   }
 };

.github/scripts/nightly-release.js

@@ -32,10 +32,10 @@ function releaseNotes(repository, baseline, sha, commits, runs) {
   ].join('\n');
 }
 
-async function prepare({ github, context, core }) {
+async function prepare({ github, context, core, sha }) {
   const repository = context.repo;
   const { data: branch } = await github.rest.repos.getBranch({ ...repository, branch: 'main' });
-  const sha = context.payload.workflow_run?.head_sha || branch.commit.sha;
+  sha ||= context.payload.workflow_run?.head_sha || branch.commit.sha;
   const { data: ancestry } = await github.rest.repos.compareCommitsWithBasehead({
     ...repository, basehead: `${sha}...${branch.commit.sha}`, per_page: 1,
   });

.github/scripts/nightly-release.test.js

@@ -178,6 +178,15 @@ test('manual publication checks the current main commit', async () => {
   assert.equal((await prepare(f)).sha, 'main-head');
 });
 
+test('scheduled publication uses the completed commit even after main advances', async () => {
+  const f = fixture();
+  f.context.payload = {};
+  const build = await prepare({ ...f, sha: 'built' });
+  assert.equal(build.sha, 'built');
+  assert.ok(f.calls.some(call => call[0] === 'compare' && call[1] === 'built...main-head'));
+  assert.deepEqual(build.runs, { core: 1, gui: 2, mobile: 3, test: 4 });
+});
+
 test('nightly requires a published version baseline and a mutable release', async () => {
   const f = fixture();
   f.state.releases = [];