From 15a3d4303994254d20557101b7f5cc1712defa47 Mon Sep 17 00:00:00 2001 From: KKRainbow <5665404+KKRainbow@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:57:54 +0800 Subject: [PATCH] ci: wait for nightly builds before publishing the release (#2695) GITHUB_TOKEN-dispatched builds do not trigger downstream workflow_run jobs. The nightly release check ran before builds finished and then exited successfully without publishing anything. Track dispatch run IDs and wait for all missing components to succeed. Retry failed jobs once, account for delayed rerun state, and fail on terminal errors. Bound the build and wait job to three hours. Pass the built commit to publication so later main commits do not change the release target. Retain component deduplication and manual build completion handlers, and document the nightly lifecycle. Add regression coverage for pending builds, retries, terminal failures, dispatch races, and main advancing before publication. All 34 Node tests and actionlint for the three nightly workflows pass. --- .github/CI.md | 23 ++++-- .github/CI.zh-CN.md | 17 ++-- .github/scripts/builds.test.js | 105 ++++++++++++++++++++---- .github/scripts/nightly-build.js | 44 +++++++++- .github/scripts/nightly-release.js | 4 +- .github/scripts/nightly-release.test.js | 9 ++ .github/workflows/nightly-release.yml | 9 +- .github/workflows/nightly.yml | 10 ++- 8 files changed, 185 insertions(+), 36 deletions(-) diff --git a/.github/CI.md b/.github/CI.md index bdc564f6..1f9448a4 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -34,14 +34,25 @@ Compilation outputs are cached. Ordinary pushes do not cancel full builds. ## Nightly -Scheduled at 03:17 Asia/Shanghai. Check Core, GUI, Mobile, and Test separately -on each existing `main`/`develop` branch. Compare HEAD with that workflow's -latest successful `workflow_dispatch` run: skip matching commits; dispatch -otherwise. Commits without a successful run are retried on the next check. - -Nightly only dispatches jobs; results appear in the corresponding workflows. +Scheduled at 03:17 Asia/Shanghai, or started manually on `main`. Check Core, +GUI, Mobile, and Test separately. Compare main HEAD with that workflow's latest +successful `workflow_dispatch` run: skip matching commits; dispatch otherwise. Core's `sign_windows` is set to `false` for nightly builds. +Nightly waits for the dispatched runs, retrying failed jobs once. A second +failure or a cancelled run fails Nightly and prevents publication. The build +and wait job has a three-hour timeout. Once all four components succeed at the +same commit, Nightly updates the rolling `nightly` prerelease using that commit, +even if main advances while builds are running. If main changes between the +dispatch requests and the builds target different commits, Nightly fails and +must be started again. + +Builds dispatched with `GITHUB_TOKEN` do not trigger downstream `workflow_run` +jobs, so scheduled publication and retries run directly in Nightly. Completion +events still handle full main builds dispatched manually by a maintainer. +Nightly Release can also be started manually to publish existing main HEAD +builds without rebuilding them. + ## Manual builds and releases Repository Write access is required. In Actions, run Core, GUI, and Mobile diff --git a/.github/CI.zh-CN.md b/.github/CI.zh-CN.md index 4809f7ef..f44fdf2f 100644 --- a/.github/CI.zh-CN.md +++ b/.github/CI.zh-CN.md @@ -31,13 +31,20 @@ Core 二进制、GUI 安装包、APK 和 Magisk 产物仅在全量模式上传 ## 每日构建 -定时为北京时间 03:17。对实际存在的 `main`/`develop` 分支，分别检查 Core、 -GUI、Mobile、Test：比较分支 HEAD 与该 workflow 最近一次成功的 +定时为北京时间 03:17，也可以在 `main` 上手动运行。分别检查 Core、GUI、 +Mobile、Test：比较 main HEAD 与该 workflow 最近一次成功的 `workflow_dispatch` 提交，相同则跳过，不同或无成功记录则触发该 workflow。 -尚未成功构建的提交在下一轮继续触发。 +每日 Core 构建设置 `sign_windows: false`。 -Nightly 只负责触发，结果见对应 workflow。每日 Core 构建设置 -`sign_windows: false`。 +Nightly 等待已触发的构建结束，失败任务自动重试一次。再次失败或取消会使 +Nightly 失败并阻止发布，构建和等待任务的超时为三小时。四个组件在同一提交 +上全部成功后，Nightly 使用该提交更新固定的 `nightly` 预发布版本，即使等待 +期间 main 已有新提交。如果 main 在各次触发请求之间变化，导致构建提交不 +一致，Nightly 会失败，需要重新运行。 + +`GITHUB_TOKEN` 触发的构建不会触发下游 `workflow_run` 任务，因此定时构建的 +发布和重试直接由 Nightly 执行。维护者手动触发的 main 全量构建仍由完成事件 +处理。也可手动运行 Nightly Release，使用 main HEAD 的现有成功构建直接发布。 ## 手动构建与发布 diff --git a/.github/scripts/builds.test.js b/.github/scripts/builds.test.js index bcb37cd8..d8795528 100644 --- a/.github/scripts/builds.test.js +++ b/.github/scripts/builds.test.js @@ -7,8 +7,8 @@ const context = { repo: { owner: 'EasyTier', repo: 'EasyTier' } }; const core = { info() {} }; const workflows = ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml']; -async function nightly(branches, successfulRuns = {}) { - const dispatched = []; +function nightly(branches, successfulRuns = {}) { + const f = { dispatched: [], retried: [], responses: {}, inspected: [], sleeps: 0 }; const github = { paginate: async () => branches, rest: { @@ -21,47 +21,118 @@ async function nightly(branches, successfulRuns = {}) { return { data: { workflow_runs: sha ? [{ head_sha: sha }] : [] } }; }, async createWorkflowDispatch(request) { - dispatched.push(request); + assert.equal(request.headers['X-GitHub-Api-Version'], '2026-03-10'); + f.dispatched.push(request); + return { data: { workflow_run_id: f.dispatched.length, html_url: `run/${f.dispatched.length}` } }; + }, + async getWorkflowRun({ run_id }) { + const workflow = f.dispatched[run_id - 1].workflow_id; + f.inspected.push({ workflow, dispatched: f.dispatched.length }); + return { data: { + head_sha: branches.find(branch => branch.name === 'main').commit.sha, + status: 'completed', conclusion: 'success', run_attempt: 1, + ...f.responses[workflow]?.shift(), + } }; + }, + async reRunWorkflowFailedJobs({ run_id }) { + f.retried.push(run_id); }, }, }, }; - await dispatch({ github, context, core }); - return dispatched; + f.run = () => dispatch({ github, context, core, sleep: async delay => { + assert.equal(delay, 60_000); + assert.ok(++f.sleeps < 10, 'nightly should eventually finish'); + } }); + return f; } test('first nightly builds every workflow on an existing main branch', async () => { - const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }]); - assert.deepEqual(runs.map(run => run.workflow_id), workflows); - assert.ok(runs.every(run => run.ref === 'main')); - assert.deepEqual(runs[0].inputs, { sign_windows: false }); - assert.ok(runs.slice(1).every(run => !run.inputs)); + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + assert.equal(await f.run(), 'new'); + assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows); + assert.ok(f.dispatched.every(run => run.ref === 'main')); + assert.deepEqual(f.dispatched[0].inputs, { sign_windows: false }); + assert.ok(f.dispatched.slice(1).every(run => !run.inputs)); + assert.deepEqual(f.inspected.map(run => run.workflow), workflows); + assert.ok(f.inspected.every(run => run.dispatched === 4)); }); test('unchanged branches skip builds and unmonitored branches are ignored', async () => { const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'same'])); - const runs = await nightly([ + const f = nightly([ { name: 'main', commit: { sha: 'same' } }, { name: 'feature', commit: { sha: 'new' } }, ], successfulRuns); - assert.deepEqual(runs, []); + assert.equal(await f.run(), 'same'); + assert.deepEqual(f.dispatched, []); + assert.equal(f.sleeps, 0); }); test('nightly retries a failed component using its last successful commit', async () => { const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'new'])); successfulRuns['main/gui.yml'] = 'old'; - const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns); - assert.deepEqual(runs.map(run => run.workflow_id), ['gui.yml']); + const f = nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns); + assert.equal(await f.run(), 'new'); + assert.deepEqual(f.dispatched.map(run => run.workflow_id), ['gui.yml']); }); test('nightly only builds main even when other branches exist', async () => { - const runs = await nightly([ + const f = nightly([ { name: 'main', commit: { sha: 'new' } }, { name: 'develop', commit: { sha: 'new' } }, { name: 'feature', commit: { sha: 'new' } }, ]); - assert.deepEqual(runs.map(run => run.workflow_id), workflows); - assert.ok(runs.every(run => run.ref === 'main')); + await f.run(); + assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows); + assert.ok(f.dispatched.every(run => run.ref === 'main')); +}); + +test('nightly waits for every dispatched build before allowing publication', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['core.yml'] = [{ status: 'queued' }, { status: 'in_progress' }, {}]; + f.responses['test.yml'] = [{ status: 'in_progress' }, {}]; + assert.equal(await f.run(), 'new'); + assert.equal(f.sleeps, 2); + assert.deepEqual(f.inspected.map(run => run.workflow), [ + ...workflows, 'core.yml', 'test.yml', 'core.yml', + ]); +}); + +test('nightly retries failed jobs once and waits for the new attempt', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['gui.yml'] = [ + { conclusion: 'failure' }, + { conclusion: 'failure' }, // The API still returns the first attempt. + { status: 'in_progress', run_attempt: 2 }, + { run_attempt: 2 }, + ]; + assert.equal(await f.run(), 'new'); + assert.deepEqual(f.retried, [2]); + assert.equal(f.sleeps, 3); +}); + +test('nightly fails instead of publishing if a retried build fails again', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['mobile.yml'] = [{ conclusion: 'failure' }, { conclusion: 'failure', run_attempt: 2 }]; + await assert.rejects(f.run(), /mobile.yml: run 3 finished with failure/); + assert.deepEqual(f.retried, [3]); +}); + +test('nightly does not retry cancelled or timed-out builds', async () => { + for (const conclusion of ['cancelled', 'timed_out']) { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['test.yml'] = [{ conclusion }]; + await assert.rejects(f.run(), new RegExp(`test.yml: run 4 finished with ${conclusion}`)); + assert.deepEqual(f.retried, []); + } +}); + +test('nightly rejects a dispatch that races with a new main commit', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['gui.yml'] = [{ head_sha: 'newer', status: 'queued' }]; + await assert.rejects(f.run(), /main changed while dispatching/); + assert.deepEqual(f.retried, []); }); process.env.CORE_RUN_ID = '1'; diff --git a/.github/scripts/nightly-build.js b/.github/scripts/nightly-build.js index f3acaefe..40e99d3f 100644 --- a/.github/scripts/nightly-build.js +++ b/.github/scripts/nightly-build.js @@ -1,4 +1,6 @@ -module.exports = async ({ github, context, core }) => { +const { setTimeout } = require('node:timers/promises'); + +module.exports = async ({ github, context, core, sleep = setTimeout }) => { const repository = context.repo; const branches = await github.paginate(github.rest.repos.listBranches, { ...repository, @@ -6,6 +8,7 @@ module.exports = async ({ github, context, core }) => { }); for (const branch of branches.filter(({ name }) => name === 'main')) { + const pending = new Map(); for (const workflow of ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml']) { const { data } = await github.rest.actions.listWorkflowRuns({ ...repository, @@ -20,13 +23,48 @@ module.exports = async ({ github, context, core }) => { continue; } - await github.rest.actions.createWorkflowDispatch({ + const { data: run } = await github.rest.actions.createWorkflowDispatch({ ...repository, workflow_id: workflow, ref: branch.name, + // This API version returns the dispatched run ID. + headers: { 'X-GitHub-Api-Version': '2026-03-10' }, ...(workflow === 'core.yml' ? { inputs: { sign_windows: false } } : {}), }); - core.info(`${workflow}: dispatched a full build for ${branch.name}.`); + pending.set(workflow, { id: run.workflow_run_id, attempt: 1 }); + core.info(`${workflow}: dispatched ${run.html_url} for ${branch.name}.`); } + + // GITHUB_TOKEN dispatches do not trigger downstream workflow_run jobs. + // Wait here so publication and retries do not depend on those events. + while (pending.size) { + for (const [workflow, build] of pending) { + const { data: run } = await github.rest.actions.getWorkflowRun({ + ...repository, run_id: build.id, + }); + if (run.head_sha !== branch.commit.sha) { + throw new Error(`${workflow}: main changed while dispatching; run Nightly again.`); + } + // A rerun request can return before the new attempt becomes visible. + if (run.run_attempt < build.attempt || run.status !== 'completed') continue; + if (run.conclusion === 'success') { + core.info(`${workflow}: run ${build.id} succeeded.`); + pending.delete(workflow); + } else if (run.conclusion === 'failure' && run.run_attempt === 1) { + await github.rest.actions.reRunWorkflowFailedJobs({ + ...repository, run_id: build.id, + }); + build.attempt = 2; + core.info(`${workflow}: retrying failed jobs in run ${build.id}.`); + } else { + throw new Error(`${workflow}: run ${build.id} finished with ${run.conclusion}.`); + } + } + if (pending.size) { + core.info(`Waiting for ${[...pending.keys()].join(', ')} at ${branch.commit.sha}.`); + await sleep(60_000); + } + } + return branch.commit.sha; } }; diff --git a/.github/scripts/nightly-release.js b/.github/scripts/nightly-release.js index 59564051..a88cdb93 100644 --- a/.github/scripts/nightly-release.js +++ b/.github/scripts/nightly-release.js @@ -32,10 +32,10 @@ function releaseNotes(repository, baseline, sha, commits, runs) { ].join('\n'); } -async function prepare({ github, context, core }) { +async function prepare({ github, context, core, sha }) { const repository = context.repo; const { data: branch } = await github.rest.repos.getBranch({ ...repository, branch: 'main' }); - const sha = context.payload.workflow_run?.head_sha || branch.commit.sha; + sha ||= context.payload.workflow_run?.head_sha || branch.commit.sha; const { data: ancestry } = await github.rest.repos.compareCommitsWithBasehead({ ...repository, basehead: `${sha}...${branch.commit.sha}`, per_page: 1, }); diff --git a/.github/scripts/nightly-release.test.js b/.github/scripts/nightly-release.test.js index 80363d95..7f4e4f1d 100644 --- a/.github/scripts/nightly-release.test.js +++ b/.github/scripts/nightly-release.test.js @@ -178,6 +178,15 @@ test('manual publication checks the current main commit', async () => { assert.equal((await prepare(f)).sha, 'main-head'); }); +test('scheduled publication uses the completed commit even after main advances', async () => { + const f = fixture(); + f.context.payload = {}; + const build = await prepare({ ...f, sha: 'built' }); + assert.equal(build.sha, 'built'); + assert.ok(f.calls.some(call => call[0] === 'compare' && call[1] === 'built...main-head')); + assert.deepEqual(build.runs, { core: 1, gui: 2, mobile: 3, test: 4 }); +}); + test('nightly requires a published version baseline and a mutable release', async () => { const f = fixture(); f.state.releases = []; diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 0ab31470..468f44ac 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -2,6 +2,11 @@ name: EasyTier Nightly Release on: workflow_call: + inputs: + sha: + description: Commit whose full builds the caller has waited for + required: true + type: string workflow_run: workflows: [EasyTier Core, EasyTier GUI, EasyTier Mobile, EasyTier Test] branches: [main] @@ -33,11 +38,13 @@ jobs: - name: Select complete main builds and prepare release notes id: prepare uses: actions/github-script@v8 + env: + BUILD_SHA: ${{ inputs.sha }} with: script: | const fs = require('node:fs'); const { prepare } = require('./.github/scripts/nightly-release.js'); - const build = await prepare({ github, context, core }); + const build = await prepare({ github, context, core, sha: process.env.BUILD_SHA }); if (!build) return; fs.writeFileSync('nightly-release.json', JSON.stringify(build)); for (const workflow of ['core', 'gui', 'mobile']) { diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6fef62ff..349a8166 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,15 +18,19 @@ jobs: dispatch: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest + timeout-minutes: 180 + outputs: + sha: ${{ steps.build.outputs.sha }} steps: - uses: actions/checkout@v5 - - name: Build branches with new commits + - name: Build main and wait for successful results + id: build uses: actions/github-script@v8 with: script: | const dispatch = require('./.github/scripts/nightly-build.js'); - await dispatch({ github, context, core }); + core.setOutput('sha', await dispatch({ github, context, core })); release: needs: dispatch @@ -34,3 +38,5 @@ jobs: actions: read contents: write uses: ./.github/workflows/nightly-release.yml + with: + sha: ${{ needs.dispatch.outputs.sha }}