diff --git a/.github/CI.md b/.github/CI.md index bdc564f6..1f9448a4 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -34,14 +34,25 @@ Compilation outputs are cached. Ordinary pushes do not cancel full builds. ## Nightly -Scheduled at 03:17 Asia/Shanghai. Check Core, GUI, Mobile, and Test separately -on each existing `main`/`develop` branch. Compare HEAD with that workflow's -latest successful `workflow_dispatch` run: skip matching commits; dispatch -otherwise. Commits without a successful run are retried on the next check. - -Nightly only dispatches jobs; results appear in the corresponding workflows. +Scheduled at 03:17 Asia/Shanghai, or started manually on `main`. Check Core, +GUI, Mobile, and Test separately. Compare main HEAD with that workflow's latest +successful `workflow_dispatch` run: skip matching commits; dispatch otherwise. Core's `sign_windows` is set to `false` for nightly builds. +Nightly waits for the dispatched runs, retrying failed jobs once. A second +failure or a cancelled run fails Nightly and prevents publication. The build +and wait job has a three-hour timeout. Once all four components succeed at the +same commit, Nightly updates the rolling `nightly` prerelease using that commit, +even if main advances while builds are running. If main changes between the +dispatch requests and the builds target different commits, Nightly fails and +must be started again. + +Builds dispatched with `GITHUB_TOKEN` do not trigger downstream `workflow_run` +jobs, so scheduled publication and retries run directly in Nightly. Completion +events still handle full main builds dispatched manually by a maintainer. +Nightly Release can also be started manually to publish existing main HEAD +builds without rebuilding them. + ## Manual builds and releases Repository Write access is required. In Actions, run Core, GUI, and Mobile diff --git a/.github/CI.zh-CN.md b/.github/CI.zh-CN.md index 4809f7ef..f44fdf2f 100644 --- a/.github/CI.zh-CN.md +++ b/.github/CI.zh-CN.md @@ -31,13 +31,20 @@ Core 二进制、GUI 安装包、APK 和 Magisk 产物仅在全量模式上传 ## 每日构建 -定时为北京时间 03:17。对实际存在的 `main`/`develop` 分支,分别检查 Core、 -GUI、Mobile、Test:比较分支 HEAD 与该 workflow 最近一次成功的 +定时为北京时间 03:17,也可以在 `main` 上手动运行。分别检查 Core、GUI、 +Mobile、Test:比较 main HEAD 与该 workflow 最近一次成功的 `workflow_dispatch` 提交,相同则跳过,不同或无成功记录则触发该 workflow。 -尚未成功构建的提交在下一轮继续触发。 +每日 Core 构建设置 `sign_windows: false`。 -Nightly 只负责触发,结果见对应 workflow。每日 Core 构建设置 -`sign_windows: false`。 +Nightly 等待已触发的构建结束,失败任务自动重试一次。再次失败或取消会使 +Nightly 失败并阻止发布,构建和等待任务的超时为三小时。四个组件在同一提交 +上全部成功后,Nightly 使用该提交更新固定的 `nightly` 预发布版本,即使等待 +期间 main 已有新提交。如果 main 在各次触发请求之间变化,导致构建提交不 +一致,Nightly 会失败,需要重新运行。 + +`GITHUB_TOKEN` 触发的构建不会触发下游 `workflow_run` 任务,因此定时构建的 +发布和重试直接由 Nightly 执行。维护者手动触发的 main 全量构建仍由完成事件 +处理。也可手动运行 Nightly Release,使用 main HEAD 的现有成功构建直接发布。 ## 手动构建与发布 diff --git a/.github/scripts/builds.test.js b/.github/scripts/builds.test.js index bcb37cd8..d8795528 100644 --- a/.github/scripts/builds.test.js +++ b/.github/scripts/builds.test.js @@ -7,8 +7,8 @@ const context = { repo: { owner: 'EasyTier', repo: 'EasyTier' } }; const core = { info() {} }; const workflows = ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml']; -async function nightly(branches, successfulRuns = {}) { - const dispatched = []; +function nightly(branches, successfulRuns = {}) { + const f = { dispatched: [], retried: [], responses: {}, inspected: [], sleeps: 0 }; const github = { paginate: async () => branches, rest: { @@ -21,47 +21,118 @@ async function nightly(branches, successfulRuns = {}) { return { data: { workflow_runs: sha ? [{ head_sha: sha }] : [] } }; }, async createWorkflowDispatch(request) { - dispatched.push(request); + assert.equal(request.headers['X-GitHub-Api-Version'], '2026-03-10'); + f.dispatched.push(request); + return { data: { workflow_run_id: f.dispatched.length, html_url: `run/${f.dispatched.length}` } }; + }, + async getWorkflowRun({ run_id }) { + const workflow = f.dispatched[run_id - 1].workflow_id; + f.inspected.push({ workflow, dispatched: f.dispatched.length }); + return { data: { + head_sha: branches.find(branch => branch.name === 'main').commit.sha, + status: 'completed', conclusion: 'success', run_attempt: 1, + ...f.responses[workflow]?.shift(), + } }; + }, + async reRunWorkflowFailedJobs({ run_id }) { + f.retried.push(run_id); }, }, }, }; - await dispatch({ github, context, core }); - return dispatched; + f.run = () => dispatch({ github, context, core, sleep: async delay => { + assert.equal(delay, 60_000); + assert.ok(++f.sleeps < 10, 'nightly should eventually finish'); + } }); + return f; } test('first nightly builds every workflow on an existing main branch', async () => { - const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }]); - assert.deepEqual(runs.map(run => run.workflow_id), workflows); - assert.ok(runs.every(run => run.ref === 'main')); - assert.deepEqual(runs[0].inputs, { sign_windows: false }); - assert.ok(runs.slice(1).every(run => !run.inputs)); + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + assert.equal(await f.run(), 'new'); + assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows); + assert.ok(f.dispatched.every(run => run.ref === 'main')); + assert.deepEqual(f.dispatched[0].inputs, { sign_windows: false }); + assert.ok(f.dispatched.slice(1).every(run => !run.inputs)); + assert.deepEqual(f.inspected.map(run => run.workflow), workflows); + assert.ok(f.inspected.every(run => run.dispatched === 4)); }); test('unchanged branches skip builds and unmonitored branches are ignored', async () => { const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'same'])); - const runs = await nightly([ + const f = nightly([ { name: 'main', commit: { sha: 'same' } }, { name: 'feature', commit: { sha: 'new' } }, ], successfulRuns); - assert.deepEqual(runs, []); + assert.equal(await f.run(), 'same'); + assert.deepEqual(f.dispatched, []); + assert.equal(f.sleeps, 0); }); test('nightly retries a failed component using its last successful commit', async () => { const successfulRuns = Object.fromEntries(workflows.map(workflow => [`main/${workflow}`, 'new'])); successfulRuns['main/gui.yml'] = 'old'; - const runs = await nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns); - assert.deepEqual(runs.map(run => run.workflow_id), ['gui.yml']); + const f = nightly([{ name: 'main', commit: { sha: 'new' } }], successfulRuns); + assert.equal(await f.run(), 'new'); + assert.deepEqual(f.dispatched.map(run => run.workflow_id), ['gui.yml']); }); test('nightly only builds main even when other branches exist', async () => { - const runs = await nightly([ + const f = nightly([ { name: 'main', commit: { sha: 'new' } }, { name: 'develop', commit: { sha: 'new' } }, { name: 'feature', commit: { sha: 'new' } }, ]); - assert.deepEqual(runs.map(run => run.workflow_id), workflows); - assert.ok(runs.every(run => run.ref === 'main')); + await f.run(); + assert.deepEqual(f.dispatched.map(run => run.workflow_id), workflows); + assert.ok(f.dispatched.every(run => run.ref === 'main')); +}); + +test('nightly waits for every dispatched build before allowing publication', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['core.yml'] = [{ status: 'queued' }, { status: 'in_progress' }, {}]; + f.responses['test.yml'] = [{ status: 'in_progress' }, {}]; + assert.equal(await f.run(), 'new'); + assert.equal(f.sleeps, 2); + assert.deepEqual(f.inspected.map(run => run.workflow), [ + ...workflows, 'core.yml', 'test.yml', 'core.yml', + ]); +}); + +test('nightly retries failed jobs once and waits for the new attempt', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['gui.yml'] = [ + { conclusion: 'failure' }, + { conclusion: 'failure' }, // The API still returns the first attempt. + { status: 'in_progress', run_attempt: 2 }, + { run_attempt: 2 }, + ]; + assert.equal(await f.run(), 'new'); + assert.deepEqual(f.retried, [2]); + assert.equal(f.sleeps, 3); +}); + +test('nightly fails instead of publishing if a retried build fails again', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['mobile.yml'] = [{ conclusion: 'failure' }, { conclusion: 'failure', run_attempt: 2 }]; + await assert.rejects(f.run(), /mobile.yml: run 3 finished with failure/); + assert.deepEqual(f.retried, [3]); +}); + +test('nightly does not retry cancelled or timed-out builds', async () => { + for (const conclusion of ['cancelled', 'timed_out']) { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['test.yml'] = [{ conclusion }]; + await assert.rejects(f.run(), new RegExp(`test.yml: run 4 finished with ${conclusion}`)); + assert.deepEqual(f.retried, []); + } +}); + +test('nightly rejects a dispatch that races with a new main commit', async () => { + const f = nightly([{ name: 'main', commit: { sha: 'new' } }]); + f.responses['gui.yml'] = [{ head_sha: 'newer', status: 'queued' }]; + await assert.rejects(f.run(), /main changed while dispatching/); + assert.deepEqual(f.retried, []); }); process.env.CORE_RUN_ID = '1'; diff --git a/.github/scripts/nightly-build.js b/.github/scripts/nightly-build.js index f3acaefe..40e99d3f 100644 --- a/.github/scripts/nightly-build.js +++ b/.github/scripts/nightly-build.js @@ -1,4 +1,6 @@ -module.exports = async ({ github, context, core }) => { +const { setTimeout } = require('node:timers/promises'); + +module.exports = async ({ github, context, core, sleep = setTimeout }) => { const repository = context.repo; const branches = await github.paginate(github.rest.repos.listBranches, { ...repository, @@ -6,6 +8,7 @@ module.exports = async ({ github, context, core }) => { }); for (const branch of branches.filter(({ name }) => name === 'main')) { + const pending = new Map(); for (const workflow of ['core.yml', 'gui.yml', 'mobile.yml', 'test.yml']) { const { data } = await github.rest.actions.listWorkflowRuns({ ...repository, @@ -20,13 +23,48 @@ module.exports = async ({ github, context, core }) => { continue; } - await github.rest.actions.createWorkflowDispatch({ + const { data: run } = await github.rest.actions.createWorkflowDispatch({ ...repository, workflow_id: workflow, ref: branch.name, + // This API version returns the dispatched run ID. + headers: { 'X-GitHub-Api-Version': '2026-03-10' }, ...(workflow === 'core.yml' ? { inputs: { sign_windows: false } } : {}), }); - core.info(`${workflow}: dispatched a full build for ${branch.name}.`); + pending.set(workflow, { id: run.workflow_run_id, attempt: 1 }); + core.info(`${workflow}: dispatched ${run.html_url} for ${branch.name}.`); } + + // GITHUB_TOKEN dispatches do not trigger downstream workflow_run jobs. + // Wait here so publication and retries do not depend on those events. + while (pending.size) { + for (const [workflow, build] of pending) { + const { data: run } = await github.rest.actions.getWorkflowRun({ + ...repository, run_id: build.id, + }); + if (run.head_sha !== branch.commit.sha) { + throw new Error(`${workflow}: main changed while dispatching; run Nightly again.`); + } + // A rerun request can return before the new attempt becomes visible. + if (run.run_attempt < build.attempt || run.status !== 'completed') continue; + if (run.conclusion === 'success') { + core.info(`${workflow}: run ${build.id} succeeded.`); + pending.delete(workflow); + } else if (run.conclusion === 'failure' && run.run_attempt === 1) { + await github.rest.actions.reRunWorkflowFailedJobs({ + ...repository, run_id: build.id, + }); + build.attempt = 2; + core.info(`${workflow}: retrying failed jobs in run ${build.id}.`); + } else { + throw new Error(`${workflow}: run ${build.id} finished with ${run.conclusion}.`); + } + } + if (pending.size) { + core.info(`Waiting for ${[...pending.keys()].join(', ')} at ${branch.commit.sha}.`); + await sleep(60_000); + } + } + return branch.commit.sha; } }; diff --git a/.github/scripts/nightly-release.js b/.github/scripts/nightly-release.js index 59564051..a88cdb93 100644 --- a/.github/scripts/nightly-release.js +++ b/.github/scripts/nightly-release.js @@ -32,10 +32,10 @@ function releaseNotes(repository, baseline, sha, commits, runs) { ].join('\n'); } -async function prepare({ github, context, core }) { +async function prepare({ github, context, core, sha }) { const repository = context.repo; const { data: branch } = await github.rest.repos.getBranch({ ...repository, branch: 'main' }); - const sha = context.payload.workflow_run?.head_sha || branch.commit.sha; + sha ||= context.payload.workflow_run?.head_sha || branch.commit.sha; const { data: ancestry } = await github.rest.repos.compareCommitsWithBasehead({ ...repository, basehead: `${sha}...${branch.commit.sha}`, per_page: 1, }); diff --git a/.github/scripts/nightly-release.test.js b/.github/scripts/nightly-release.test.js index 80363d95..7f4e4f1d 100644 --- a/.github/scripts/nightly-release.test.js +++ b/.github/scripts/nightly-release.test.js @@ -178,6 +178,15 @@ test('manual publication checks the current main commit', async () => { assert.equal((await prepare(f)).sha, 'main-head'); }); +test('scheduled publication uses the completed commit even after main advances', async () => { + const f = fixture(); + f.context.payload = {}; + const build = await prepare({ ...f, sha: 'built' }); + assert.equal(build.sha, 'built'); + assert.ok(f.calls.some(call => call[0] === 'compare' && call[1] === 'built...main-head')); + assert.deepEqual(build.runs, { core: 1, gui: 2, mobile: 3, test: 4 }); +}); + test('nightly requires a published version baseline and a mutable release', async () => { const f = fixture(); f.state.releases = []; diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 0ab31470..468f44ac 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -2,6 +2,11 @@ name: EasyTier Nightly Release on: workflow_call: + inputs: + sha: + description: Commit whose full builds the caller has waited for + required: true + type: string workflow_run: workflows: [EasyTier Core, EasyTier GUI, EasyTier Mobile, EasyTier Test] branches: [main] @@ -33,11 +38,13 @@ jobs: - name: Select complete main builds and prepare release notes id: prepare uses: actions/github-script@v8 + env: + BUILD_SHA: ${{ inputs.sha }} with: script: | const fs = require('node:fs'); const { prepare } = require('./.github/scripts/nightly-release.js'); - const build = await prepare({ github, context, core }); + const build = await prepare({ github, context, core, sha: process.env.BUILD_SHA }); if (!build) return; fs.writeFileSync('nightly-release.json', JSON.stringify(build)); for (const workflow of ['core', 'gui', 'mobile']) { diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6fef62ff..349a8166 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,15 +18,19 @@ jobs: dispatch: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest + timeout-minutes: 180 + outputs: + sha: ${{ steps.build.outputs.sha }} steps: - uses: actions/checkout@v5 - - name: Build branches with new commits + - name: Build main and wait for successful results + id: build uses: actions/github-script@v8 with: script: | const dispatch = require('./.github/scripts/nightly-build.js'); - await dispatch({ github, context, core }); + core.setOutput('sha', await dispatch({ github, context, core })); release: needs: dispatch @@ -34,3 +38,5 @@ jobs: actions: read contents: write uses: ./.github/workflows/nightly-release.yml + with: + sha: ${{ needs.dispatch.outputs.sha }}