ci: wait for nightly builds before publishing the release (#2695)

GITHUB_TOKEN-dispatched builds do not trigger downstream workflow_run
jobs. The nightly release check ran before builds finished and then
exited successfully without publishing anything.

Track dispatch run IDs and wait for all missing components to succeed.
Retry failed jobs once, account for delayed rerun state, and fail on
terminal errors. Bound the build and wait job to three hours.

Pass the built commit to publication so later main commits do not
change the release target. Retain component deduplication and manual
build completion handlers, and document the nightly lifecycle.

Add regression coverage for pending builds, retries, terminal failures,
dispatch races, and main advancing before publication. All 34 Node tests
and actionlint for the three nightly workflows pass.

.github/CI.md

@@ -34,14 +34,25 @@ Compilation outputs are cached. Ordinary pushes do not cancel full builds.
 
 ## Nightly
 
-Scheduled at 03:17 Asia/Shanghai. Check Core, GUI, Mobile, and Test separately
-on each existing `main`/`develop` branch. Compare HEAD with that workflow's
-latest successful `workflow_dispatch` run: skip matching commits; dispatch
-otherwise. Commits without a successful run are retried on the next check.
-
-Nightly only dispatches jobs; results appear in the corresponding workflows.
+Scheduled at 03:17 Asia/Shanghai, or started manually on `main`. Check Core,
+GUI, Mobile, and Test separately. Compare main HEAD with that workflow's latest
+successful `workflow_dispatch` run: skip matching commits; dispatch otherwise.
 Core's `sign_windows` is set to `false` for nightly builds.
 
+Nightly waits for the dispatched runs, retrying failed jobs once. A second
+failure or a cancelled run fails Nightly and prevents publication. The build
+and wait job has a three-hour timeout. Once all four components succeed at the
+same commit, Nightly updates the rolling `nightly` prerelease using that commit,
+even if main advances while builds are running. If main changes between the
+dispatch requests and the builds target different commits, Nightly fails and
+must be started again.
+
+Builds dispatched with `GITHUB_TOKEN` do not trigger downstream `workflow_run`
+jobs, so scheduled publication and retries run directly in Nightly. Completion
+events still handle full main builds dispatched manually by a maintainer.
+Nightly Release can also be started manually to publish existing main HEAD
+builds without rebuilding them.
+
 ## Manual builds and releases
 
 Repository Write access is required. In Actions, run Core, GUI, and Mobile

.github/CI.zh-CN.md

@@ -31,13 +31,20 @@ Core 二进制、GUI 安装包、APK 和 Magisk 产物仅在全量模式上传
 
 ## 每日构建
 
-定时为北京时间 03:17。对实际存在的 `main`/`develop` 分支，分别检查 Core、
-GUI、Mobile、Test：比较分支 HEAD 与该 workflow 最近一次成功的
+定时为北京时间 03:17，也可以在 `main` 上手动运行。分别检查 Core、GUI、
+Mobile、Test：比较 main HEAD 与该 workflow 最近一次成功的
 `workflow_dispatch` 提交，相同则跳过，不同或无成功记录则触发该 workflow。
-尚未成功构建的提交在下一轮继续触发。
+每日 Core 构建设置 `sign_windows: false`。
 
-Nightly 只负责触发，结果见对应 workflow。每日 Core 构建设置
-`sign_windows: false`。
+Nightly 等待已触发的构建结束，失败任务自动重试一次。再次失败或取消会使
+Nightly 失败并阻止发布，构建和等待任务的超时为三小时。四个组件在同一提交
+上全部成功后，Nightly 使用该提交更新固定的 `nightly` 预发布版本，即使等待
+期间 main 已有新提交。如果 main 在各次触发请求之间变化，导致构建提交不
+一致，Nightly 会失败，需要重新运行。
+
+`GITHUB_TOKEN` 触发的构建不会触发下游 `workflow_run` 任务，因此定时构建的
+发布和重试直接由 Nightly 执行。维护者手动触发的 main 全量构建仍由完成事件
+处理。也可手动运行 Nightly Release，使用 main HEAD 的现有成功构建直接发布。
 
 ## 手动构建与发布
 

.github/scripts/builds.test.js

@@ -7,8 +7,8 @@ const context = { repo: { owner: 'EasyTier', repo: 'EasyTier' } };
 const core = { info() {} };
 const workflows = ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml'];
 
-async function nightly(branches, successfulRuns = {}) {
-  const dispatched = [];
+function nightly(branches, successfulRuns = {}) {
+  const f = { dispatched: [], retried: [], responses: {}, inspected: [], sleeps: 0 };
   const github = {
     paginate: async () => branches,
     rest: {
@@ -21,47 +21,118 @@ async function nightly(branches, successfulRuns = {}) {
           return { data: { workflow_runs: sha ? [{ head_sha: sha }] : [] } };
         },
         async createWorkflowDispatch(request) {
-          dispatched.push(request);
+          assert.equal(request.headers['X-GitHub-Api-Version'], '2026-03-10');
+          f.dispatched.push(request);
+          return { data: { workflow_run_id: f.dispatched.length, html_url: `run/${f.dispatched.length}` } };
+        },
+        async getWorkflowRun({ run_id }) {
+          const workflow = f.dispatched[run_id - 1].workflow_id;
+          f.inspected.push({ workflow, dispatched: f.dispatched.length });
+          return { data: {
+            head_sha: branches.find(branch => branch.name === 'main').commit.sha,
+            status: 'completed', conclusion: 'success', run_attempt: 1,
+            ...f.responses[workflow]?.shift(),
+          } };
+        },
+        async reRunWorkflowFailedJobs({ run_id }) {
+          f.retried.push(run_id);
         },
       },
     },
   };
-  await dispatch({ github, context, core });
-  return dispatched;
+  f.run = () => dispatch({ github, context, core, sleep: async delay => {
+    assert.equal(delay, 60_000);
+    assert.ok(++f.sleeps < 10, 'nightly should eventually finish');
+  } });
+  return f;
 }
 
 test('first nightly builds every workflow on an existing main branch', async () => {
-  const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }]);
-  assert.deepEqual(runs.map(run => run.workflow_id), workflows);
-  assert.ok(runs.every(run => run.ref === 'main'));
-  assert.deepEqual(runs[0].inputs, { sign_windows: false });
-  assert.ok(runs.slice(1).every(run => !run.inputs));
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  assert.equal(await f.run(), 'new');
+  assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows);
+  assert.ok(f.dispatched.every(run => run.ref === 'main'));
+  assert.deepEqual(f.dispatched[0].inputs, { sign_windows: false });
+  assert.ok(f.dispatched.slice(1).every(run => !run.inputs));
+  assert.deepEqual(f.inspected.map(run => run.workflow), workflows);
+  assert.ok(f.inspected.every(run => run.dispatched === 4));
 });
 
 test('unchanged branches skip builds and unmonitored branches are ignored', async () => {
   const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'same']));
-  const runs = await nightly([
+  const f = nightly([
     { name: 'main', commit: { sha: 'same' } },
     { name: 'feature', commit: { sha: 'new' } },
   ], successfulRuns);
-  assert.deepEqual(runs, []);
+  assert.equal(await f.run(), 'same');
+  assert.deepEqual(f.dispatched, []);
+  assert.equal(f.sleeps, 0);
 });
 
 test('nightly retries a failed component using its last successful commit', async () => {
   const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'new']));
   successfulRuns['main/gui.yml'] = 'old';
-  const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns);
-  assert.deepEqual(runs.map(run => run.workflow_id), ['gui.yml']);
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns);
+  assert.equal(await f.run(), 'new');
+  assert.deepEqual(f.dispatched.map(run => run.workflow_id), ['gui.yml']);
 });
 
 test('nightly only builds main even when other branches exist', async () => {
-  const runs = await nightly([
+  const f = nightly([
     { name: 'main', commit: { sha: 'new' } },
     { name: 'develop', commit: { sha: 'new' } },
     { name: 'feature', commit: { sha: 'new' } },
   ]);
-  assert.deepEqual(runs.map(run => run.workflow_id), workflows);
-  assert.ok(runs.every(run => run.ref === 'main'));
+  await f.run();
+  assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows);
+  assert.ok(f.dispatched.every(run => run.ref === 'main'));
+});
+
+test('nightly waits for every dispatched build before allowing publication', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['core.yml'] = [{ status: 'queued' }, { status: 'in_progress' }, {}];
+  f.responses['test.yml'] = [{ status: 'in_progress' }, {}];
+  assert.equal(await f.run(), 'new');
+  assert.equal(f.sleeps, 2);
+  assert.deepEqual(f.inspected.map(run => run.workflow), [
+    ...workflows, 'core.yml', 'test.yml', 'core.yml',
+  ]);
+});
+
+test('nightly retries failed jobs once and waits for the new attempt', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['gui.yml'] = [
+    { conclusion: 'failure' },
+    { conclusion: 'failure' }, // The API still returns the first attempt.
+    { status: 'in_progress', run_attempt: 2 },
+    { run_attempt: 2 },
+  ];
+  assert.equal(await f.run(), 'new');
+  assert.deepEqual(f.retried, [2]);
+  assert.equal(f.sleeps, 3);
+});
+
+test('nightly fails instead of publishing if a retried build fails again', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['mobile.yml'] = [{ conclusion: 'failure' }, { conclusion: 'failure', run_attempt: 2 }];
+  await assert.rejects(f.run(), /mobile.yml: run 3 finished with failure/);
+  assert.deepEqual(f.retried, [3]);
+});
+
+test('nightly does not retry cancelled or timed-out builds', async () => {
+  for (const conclusion of ['cancelled', 'timed_out']) {
+    const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+    f.responses['test.yml'] = [{ conclusion }];
+    await assert.rejects(f.run(), new RegExp(`test.yml: run 4 finished with ${conclusion}`));
+    assert.deepEqual(f.retried, []);
+  }
+});
+
+test('nightly rejects a dispatch that races with a new main commit', async () => {
+  const f = nightly([{ name: 'main', commit: { sha: 'new' } }]);
+  f.responses['gui.yml'] = [{ head_sha: 'newer', status: 'queued' }];
+  await assert.rejects(f.run(), /main changed while dispatching/);
+  assert.deepEqual(f.retried, []);
 });
 
 process.env.CORE_RUN_ID = '1';

.github/scripts/nightly-build.js

@@ -1,4 +1,6 @@
-module.exports = async ({ github, context, core }) => {
+const { setTimeout } = require('node:timers/promises');
+
+module.exports = async ({ github, context, core, sleep = setTimeout }) => {
   const repository = context.repo;
   const branches = await github.paginate(github.rest.repos.listBranches, {
     ...repository,
@@ -6,6 +8,7 @@ module.exports = async ({ github, context, core }) => {
   });
 
   for (const branch of branches.filter(({ name }) => name === 'main')) {
+    const pending = new Map();
     for (const workflow of ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml']) {
       const { data } = await github.rest.actions.listWorkflowRuns({
         ...repository,
@@ -20,13 +23,48 @@ module.exports = async ({ github, context, core }) => {
         continue;
       }
 
-      await github.rest.actions.createWorkflowDispatch({
+      const { data: run } = await github.rest.actions.createWorkflowDispatch({
         ...repository,
         workflow_id: workflow,
         ref: branch.name,
+        // This API version returns the dispatched run ID.
+        headers: { 'X-GitHub-Api-Version': '2026-03-10' },
         ...(workflow === 'core.yml' ? { inputs: { sign_windows: false } } : {}),
       });
-      core.info(`${workflow}: dispatched a full build for ${branch.name}.`);
+      pending.set(workflow, { id: run.workflow_run_id, attempt: 1 });
+      core.info(`${workflow}: dispatched ${run.html_url} for ${branch.name}.`);
     }
+
+    // GITHUB_TOKEN dispatches do not trigger downstream workflow_run jobs.
+    // Wait here so publication and retries do not depend on those events.
+    while (pending.size) {
+      for (const [workflow, build] of pending) {
+        const { data: run } = await github.rest.actions.getWorkflowRun({
+          ...repository, run_id: build.id,
+        });
+        if (run.head_sha !== branch.commit.sha) {
+          throw new Error(`${workflow}: main changed while dispatching; run Nightly again.`);
+        }
+        // A rerun request can return before the new attempt becomes visible.
+        if (run.run_attempt < build.attempt || run.status !== 'completed') continue;
+        if (run.conclusion === 'success') {
+          core.info(`${workflow}: run ${build.id} succeeded.`);
+          pending.delete(workflow);
+        } else if (run.conclusion === 'failure' && run.run_attempt === 1) {
+          await github.rest.actions.reRunWorkflowFailedJobs({
+            ...repository, run_id: build.id,
+          });
+          build.attempt = 2;
+          core.info(`${workflow}: retrying failed jobs in run ${build.id}.`);
+        } else {
+          throw new Error(`${workflow}: run ${build.id} finished with ${run.conclusion}.`);
+        }
+      }
+      if (pending.size) {
+        core.info(`Waiting for ${[...pending.keys()].join(', ')} at ${branch.commit.sha}.`);
+        await sleep(60_000);
+      }
+    }
+    return branch.commit.sha;
   }
 };

.github/scripts/nightly-release.js

@@ -32,10 +32,10 @@ function releaseNotes(repository, baseline, sha, commits, runs) {
   ].join('\n');
 }
 
-async function prepare({ github, context, core }) {
+async function prepare({ github, context, core, sha }) {
   const repository = context.repo;
   const { data: branch } = await github.rest.repos.getBranch({ ...repository, branch: 'main' });
-  const sha = context.payload.workflow_run?.head_sha || branch.commit.sha;
+  sha ||= context.payload.workflow_run?.head_sha || branch.commit.sha;
   const { data: ancestry } = await github.rest.repos.compareCommitsWithBasehead({
     ...repository, basehead: `${sha}...${branch.commit.sha}`, per_page: 1,
   });

.github/scripts/nightly-release.test.js

@@ -178,6 +178,15 @@ test('manual publication checks the current main commit', async () => {
   assert.equal((await prepare(f)).sha, 'main-head');
 });
 
+test('scheduled publication uses the completed commit even after main advances', async () => {
+  const f = fixture();
+  f.context.payload = {};
+  const build = await prepare({ ...f, sha: 'built' });
+  assert.equal(build.sha, 'built');
+  assert.ok(f.calls.some(call => call[0] === 'compare' && call[1] === 'built...main-head'));
+  assert.deepEqual(build.runs, { core: 1, gui: 2, mobile: 3, test: 4 });
+});
+
 test('nightly requires a published version baseline and a mutable release', async () => {
   const f = fixture();
   f.state.releases = [];

.github/workflows/nightly-release.yml

@@ -2,6 +2,11 @@ name: EasyTier Nightly Release
 
 on:
   workflow_call:
+    inputs:
+      sha:
+        description: Commit whose full builds the caller has waited for
+        required: true
+        type: string
   workflow_run:
     workflows: [EasyTier Core, EasyTier GUI, EasyTier Mobile, EasyTier Test]
     branches: [main]
@@ -33,11 +38,13 @@ jobs:
       - name: Select complete main builds and prepare release notes
         id: prepare
         uses: actions/github-script@v8
+        env:
+          BUILD_SHA: ${{ inputs.sha }}
         with:
           script: |
             const fs = require('node:fs');
             const { prepare } = require('./.github/scripts/nightly-release.js');
-            const build = await prepare({ github, context, core });
+            const build = await prepare({ github, context, core, sha: process.env.BUILD_SHA });
             if (!build) return;
             fs.writeFileSync('nightly-release.json', JSON.stringify(build));
             for (const workflow of ['core', 'gui', 'mobile']) {

.github/workflows/nightly.yml

@@ -18,15 +18,19 @@ jobs:
   dispatch:
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
+    timeout-minutes: 180
+    outputs:
+      sha: ${{ steps.build.outputs.sha }}
     steps:
       - uses: actions/checkout@v5
 
-      - name: Build branches with new commits
+      - name: Build main and wait for successful results
+        id: build
         uses: actions/github-script@v8
         with:
           script: |
             const dispatch = require('./.github/scripts/nightly-build.js');
-            await dispatch({ github, context, core });
+            core.setOutput('sha', await dispatch({ github, context, core }));
 
   release:
     needs: dispatch
@@ -34,3 +38,5 @@ jobs:
       actions: read
       contents: write
     uses: ./.github/workflows/nightly-release.yml
+    with:
+      sha: ${{ needs.dispatch.outputs.sha }}