diff --git a/.github/workflows/build-and-make.yaml b/.github/workflows/build-and-make.yaml index 5467249e6..f04ca8330 100644 --- a/.github/workflows/build-and-make.yaml +++ b/.github/workflows/build-and-make.yaml @@ -80,7 +80,7 @@ jobs: echo 'meson=1.7.2' } > "${RUNNER_TEMP}/linux-runtime-toolchain.txt" TOOLCHAIN_SHA256="$(sha256sum "${RUNNER_TEMP}/linux-runtime-toolchain.txt" | cut -d ' ' -f 1)" - SOURCE_SHA256="${{ hashFiles('tools/embedded-mpv/build-linux-runtime.cjs', 'tools/embedded-mpv/build-linux-runtime.mjs', 'tools/embedded-mpv/generate-linux-runtime-notices.cjs', 'tools/embedded-mpv/linux-runtime-manifest.cjs', 'tools/embedded-mpv/linux-source-archive-contract.cjs', 'tools/embedded-mpv/stage-runtime.mjs', 'tools/packaging/prepare-linux-runtime-source-snapshot.cjs') }}" + SOURCE_SHA256="${{ hashFiles('tools/embedded-mpv/build-linux-runtime.cjs', 'tools/embedded-mpv/build-linux-runtime.mjs', 'tools/embedded-mpv/download-pinned-source.mjs', 'tools/embedded-mpv/generate-linux-runtime-notices.cjs', 'tools/embedded-mpv/linux-runtime-manifest.cjs', 'tools/embedded-mpv/linux-source-archive-contract.cjs', 'tools/embedded-mpv/stage-runtime.mjs', 'tools/packaging/prepare-linux-runtime-source-snapshot.cjs') }}" echo "toolchain-sha256=${TOOLCHAIN_SHA256}" >> "${GITHUB_OUTPUT}" echo "key=linux-frame-copy-runtime-v5-ubuntu-22.04-${TOOLCHAIN_SHA256}-${SOURCE_SHA256}" >> "${GITHUB_OUTPUT}" @@ -197,6 +197,7 @@ jobs: cp \ tools/embedded-mpv/build-linux-runtime.cjs \ tools/embedded-mpv/build-linux-runtime.mjs \ + tools/embedded-mpv/download-pinned-source.mjs \ tools/embedded-mpv/generate-linux-runtime-notices.cjs \ tools/embedded-mpv/linux-runtime-manifest.cjs \ tools/embedded-mpv/linux-source-archive-contract.cjs \ diff --git a/tools/embedded-mpv/README.md b/tools/embedded-mpv/README.md index 688f174a5..ffaafbca1 100644 --- a/tools/embedded-mpv/README.md +++ b/tools/embedded-mpv/README.md @@ -99,6 +99,20 @@ patchelf, and `readelf`. It builds into an owned staging directory and publishes atomically, so it will not delete or overwrite an arbitrary destination. +The Linux builder acquires its archives through the same +`downloadPinnedSource` helper, so a pin whose primary host is unavailable +falls through to its pinned `mirrors` before the build fails; every candidate +is verified against the pinned SHA-256, and a mismatch is discarded rather +than used. FreeType, Fontconfig, and libdisplay-info carry mirrors because +their primary hosts are single points of failure — freedesktop.org in +particular answers GitHub runners with HTTP 418 under load. Unlike the macOS +manifest, the Linux manifest keeps `sourceUrl` at the canonical pinned value +even when a mirror served the bytes: notice generation and the Snap +publication boundary compare that field against the immutable pin. A used +mirror is reported in the build log instead. `download-pinned-source.mjs` is +part of the released source-archive tooling set and of the Linux runtime +cache key. + The pinned Linux source stack currently includes FFmpeg 8.1, mpv 0.41.0, libplacebo 7.360.1, libass 0.17.3, FreeType 2.13.3, FriBidi 1.0.16, HarfBuzz 8.5.0, Expat 2.8.2, Fontconfig 2.16.0, OpenSSL 3.5.7, hwdata diff --git a/tools/embedded-mpv/build-linux-runtime.cjs b/tools/embedded-mpv/build-linux-runtime.cjs index a8b4d370c..d6d5b916e 100644 --- a/tools/embedded-mpv/build-linux-runtime.cjs +++ b/tools/embedded-mpv/build-linux-runtime.cjs @@ -27,6 +27,9 @@ const SOURCE_PACKAGES = Object.freeze( sourceKind: 'archive', sourceUrl: 'https://download.savannah.gnu.org/releases/freetype/freetype-2.13.3.tar.xz', + mirrors: Object.freeze([ + 'https://downloads.sourceforge.net/project/freetype/freetype2/2.13.3/freetype-2.13.3.tar.xz', + ]), expectedSha256: '0550350666d427c74daeb85d5ac7bb353acba5f76956395995311a9c6f063289', license: 'FreeType License (FTL)', @@ -67,6 +70,9 @@ const SOURCE_PACKAGES = Object.freeze( sourceKind: 'archive', sourceUrl: 'https://www.freedesktop.org/software/fontconfig/release/fontconfig-2.16.0.tar.xz', + mirrors: Object.freeze([ + 'https://ftp.osuosl.org/pub/blfs/conglomeration/fontconfig/fontconfig-2.16.0.tar.xz', + ]), expectedSha256: '6a33dc555cc9ba8b10caf7695878ef134eeb36d0af366041f639b1da9b6ed220', license: 'MIT', @@ -127,6 +133,9 @@ const SOURCE_PACKAGES = Object.freeze( sourceKind: 'archive', sourceUrl: 'https://gitlab.freedesktop.org/emersion/libdisplay-info/-/releases/0.1.1/downloads/libdisplay-info-0.1.1.tar.xz', + mirrors: Object.freeze([ + 'https://ftp.osuosl.org/pub/blfs/conglomeration/libdisplay-info/libdisplay-info-0.1.1.tar.xz', + ]), expectedSha256: '0d8731588e9f82a9cac96324a3d7c82e2ba5b1b5e006143fefe692c74069fb60', license: 'MIT', diff --git a/tools/embedded-mpv/build-linux-runtime.mjs b/tools/embedded-mpv/build-linux-runtime.mjs index e9efefd8f..a79bd5bbb 100644 --- a/tools/embedded-mpv/build-linux-runtime.mjs +++ b/tools/embedded-mpv/build-linux-runtime.mjs @@ -8,6 +8,8 @@ import { spawnSync } from 'node:child_process'; import { createRequire } from 'node:module'; import { fileURLToPath } from 'node:url'; +import { downloadPinnedSource } from './download-pinned-source.mjs'; + const require = createRequire(import.meta.url); const { BUILD_RECIPES, @@ -39,7 +41,6 @@ const { resolveSystemPkgConfigDirs, runtimeLibraryNames, selectReachableRuntimeLibraryNames, - sha256Buffer, publishOwnedOutput, validateRuntimeDependencyClosure, } = require('./build-linux-runtime.cjs'); @@ -207,35 +208,37 @@ function sourcePathFor(packageId, sourceRoot) { return path.join(sourceRoot, packageId); } -function sha256File(filePath) { - return sha256Buffer(fs.readFileSync(filePath)); +function sourceUrlsFor(sourcePackage) { + return [sourcePackage.sourceUrl, ...(sourcePackage.mirrors ?? [])]; } -function downloadArchive(sourcePackage, context) { +export function downloadArchive(sourcePackage, context) { const archivePath = archivePathFor(sourcePackage, context.archiveRoot); - if (!fs.existsSync(archivePath)) { - const temporaryArchivePath = `${archivePath}.partial`; - fs.rmSync(temporaryArchivePath, { force: true }); - context.run('curl', [ - '--fail', - '--location', - '--retry', - '3', - '--retry-all-errors', - '--connect-timeout', - '30', - '--proto', - '=https', - '--tlsv1.2', - '--output', - temporaryArchivePath, - sourcePackage.sourceUrl, - ]); - fs.renameSync(temporaryArchivePath, archivePath); - } - - const sourceSha256 = sha256File(archivePath); + const { sourceSha256, sourceUrl } = downloadPinnedSource({ + archivePath, + expectedSha256: sourcePackage.expectedSha256, + urls: sourceUrlsFor(sourcePackage), + download: ({ destinationPath, url }) => + context.run('curl', [ + '--fail', + '--location', + '--retry', + '3', + '--retry-all-errors', + '--connect-timeout', + '30', + '--proto', + '=https', + '--tlsv1.2', + '--output', + destinationPath, + url, + ]), + }); assertArchiveMatchesPin(sourcePackage, sourceSha256); + if (sourceUrl && sourceUrl !== sourcePackage.sourceUrl) { + log(`Downloaded ${sourcePackage.id} from pinned mirror ${sourceUrl}`); + } const packageSourcePath = sourcePathFor( sourcePackage.id, context.sourceRoot diff --git a/tools/embedded-mpv/build-linux-runtime.test.mjs b/tools/embedded-mpv/build-linux-runtime.test.mjs index 61a2cbd0a..379d27750 100644 --- a/tools/embedded-mpv/build-linux-runtime.test.mjs +++ b/tools/embedded-mpv/build-linux-runtime.test.mjs @@ -183,6 +183,9 @@ test('pins the complete source stack and preserves dependency build order', () = sourceKind: 'archive', sourceUrl: 'https://gitlab.freedesktop.org/emersion/libdisplay-info/-/releases/0.1.1/downloads/libdisplay-info-0.1.1.tar.xz', + mirrors: [ + 'https://ftp.osuosl.org/pub/blfs/conglomeration/libdisplay-info/libdisplay-info-0.1.1.tar.xz', + ], expectedSha256: '0d8731588e9f82a9cac96324a3d7c82e2ba5b1b5e006143fefe692c74069fb60', license: 'MIT', @@ -266,6 +269,100 @@ test('verifies source pins before archive extraction or git submodules', () => { ); }); +function createArchiveDownloadHarness(t, { archive, failingUrls = [] }) { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), 'iptvnator-linux-archive-') + ); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const archiveRoot = path.join(root, 'archives'); + const sourceRoot = path.join(root, 'sources'); + fs.mkdirSync(archiveRoot, { recursive: true }); + fs.mkdirSync(sourceRoot, { recursive: true }); + const attempts = []; + const extractions = []; + return { + attempts, + extractions, + archiveRoot, + context: { + archiveRoot, + sourceRoot, + run(command, args) { + if (command !== 'curl') { + extractions.push(args); + return; + } + const url = args.at(-1); + attempts.push({ url, args }); + if (failingUrls.includes(url)) { + throw new Error(`curl failed for ${url}`); + } + fs.writeFileSync(args[args.indexOf('--output') + 1], archive); + }, + }, + }; +} + +test('downloads a pinned archive from the next mirror when its host fails', async (t) => { + const { downloadArchive } = await import(pathToFileURL(builderScript).href); + const fontconfig = SOURCE_PACKAGES.find(({ id }) => id === 'fontconfig'); + const archive = Buffer.from('fontconfig source archive'); + const sourcePackage = { + ...fontconfig, + expectedSha256: crypto + .createHash('sha256') + .update(archive) + .digest('hex'), + }; + const harness = createArchiveDownloadHarness(t, { + archive, + failingUrls: [fontconfig.sourceUrl], + }); + + const record = downloadArchive(sourcePackage, harness.context); + + assert.deepEqual( + harness.attempts.map(({ url }) => url), + [fontconfig.sourceUrl, ...fontconfig.mirrors] + ); + for (const { args } of harness.attempts) { + for (const flag of ['--proto', '=https', '--tlsv1.2', '--fail']) { + assert.ok(args.includes(flag), flag); + } + } + assert.equal(record.sourceSha256, sourcePackage.expectedSha256); + // The manifest, notices and Snap boundary all pin the canonical host, so a + // mirrored download must never rewrite the recorded source URL. + assert.equal(record.sourceUrl, fontconfig.sourceUrl); + assert.equal(harness.extractions.length, 1); + assert.deepEqual( + fs.readFileSync( + path.join(harness.archiveRoot, 'fontconfig-2.16.0.tar.xz') + ), + archive + ); +}); + +test('refuses a mirror whose archive does not match the pinned digest', async (t) => { + const { downloadArchive } = await import(pathToFileURL(builderScript).href); + const fontconfig = SOURCE_PACKAGES.find(({ id }) => id === 'fontconfig'); + const harness = createArchiveDownloadHarness(t, { + archive: Buffer.from('substituted archive'), + failingUrls: [fontconfig.sourceUrl], + }); + + assert.throws( + () => downloadArchive(fontconfig, harness.context), + /Unable to download a verified source archive[\s\S]*SHA-256 mismatch/ + ); + assert.deepEqual( + harness.attempts.map(({ url }) => url), + [fontconfig.sourceUrl, ...fontconfig.mirrors] + ); + assert.deepEqual(harness.extractions, []); + assert.deepEqual(fs.readdirSync(harness.archiveRoot), []); +}); + test('rejects source metadata that does not match the immutable pins', () => { const sourceRecords = createPinnedSourceRecords(); sourceRecords.freetype.sourceSha256 = '0'.repeat(64); @@ -1505,6 +1602,14 @@ test('generates a hash-complete manifest accepted by the Linux validator', (t) = sourcePackage.expectedSha256 ); } + // Notice generation and the Snap publication boundary compare the + // manifest against the immutable pin, so the mirror list must stay + // out of the manifest and sourceUrl must remain the canonical host. + assert.equal( + manifest.packages[sourcePackage.id].sourceUrl, + sourcePackage.sourceUrl + ); + assert.equal(manifest.packages[sourcePackage.id].mirrors, undefined); } assert.equal( manifest.packages.libplacebo.sourceGitCommit, diff --git a/tools/embedded-mpv/download-pinned-source.test.mjs b/tools/embedded-mpv/download-pinned-source.test.mjs index 42c0e5bac..f8d6eeaed 100644 --- a/tools/embedded-mpv/download-pinned-source.test.mjs +++ b/tools/embedded-mpv/download-pinned-source.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import crypto from 'node:crypto'; import fs from 'node:fs'; +import { createRequire } from 'node:module'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; @@ -9,6 +10,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; const currentDir = path.dirname(fileURLToPath(import.meta.url)); const downloaderPath = path.join(currentDir, 'download-pinned-source.mjs'); const macosBuilderPath = path.join(currentDir, 'build-macos-runtime.mjs'); +const linuxBuilderPath = path.join(currentDir, 'build-linux-runtime.mjs'); const workspaceRoot = path.resolve(currentDir, '..', '..'); const buildWorkflowPath = path.join( workspaceRoot, @@ -16,6 +18,14 @@ const buildWorkflowPath = path.join( 'workflows', 'build-and-make.yaml' ); +const snapSourceBindingPath = path.join( + workspaceRoot, + 'tools', + 'packaging', + 'release-snap-source-binding.cjs' +); +const require = createRequire(import.meta.url); +const { SOURCE_PACKAGES } = require('./build-linux-runtime.cjs'); const { downloadPinnedSource } = await import(pathToFileURL(downloaderPath)); function sha256(value) { @@ -77,6 +87,94 @@ test('includes the pinned downloader in the macOS runtime cache key', () => { ); }); +test('routes the Linux runtime builder through the shared pinned downloader', () => { + const builderSource = fs.readFileSync(linuxBuilderPath, 'utf8'); + + assert.match( + builderSource, + /import \{ downloadPinnedSource \} from '\.\/download-pinned-source\.mjs';/ + ); + assert.match( + builderSource, + /const\s*\{\s*sourceSha256,\s*sourceUrl\s*\}\s*=\s*downloadPinnedSource/ + ); + assert.match(builderSource, /urls: sourceUrlsFor\(sourcePackage\)/); + // The Linux manifest, its notices and the Snap publication boundary all + // compare sourceUrl against the immutable pin, so unlike the macOS builder + // the resolved mirror must never overwrite the recorded source URL. + assert.doesNotMatch(builderSource, /sourcePackage\.sourceUrl =/); + for (const flag of [ + "'--fail'", + "'--location'", + "'--proto'", + "'=https'", + "'--tlsv1.2'", + ]) { + assert.ok(builderSource.includes(flag), flag); + } +}); + +test('pins second-host mirrors for single-source Linux archives', () => { + const mirrored = SOURCE_PACKAGES.filter(({ mirrors }) => mirrors); + assert.deepEqual( + mirrored.map(({ id }) => id), + ['freetype', 'fontconfig', 'libdisplay-info'] + ); + + for (const sourcePackage of SOURCE_PACKAGES) { + const primary = new URL(sourcePackage.sourceUrl); + if (/(^|\.)freedesktop\.org$/.test(primary.hostname)) { + // freedesktop.org rate-limits GitHub runners with HTTP 418, which + // is what made a single pinned host flaky in the first place. + assert.ok( + sourcePackage.mirrors?.length > 0, + `${sourcePackage.id} must pin a mirror` + ); + } + for (const mirror of sourcePackage.mirrors ?? []) { + const mirrorUrl = new URL(mirror); + assert.equal(mirrorUrl.protocol, 'https:'); + assert.notEqual(mirrorUrl.hostname, primary.hostname); + assert.equal( + path.posix.basename(mirrorUrl.pathname), + path.posix.basename(primary.pathname) + ); + } + } +}); + +test('ships and cache-keys every released Linux runtime tooling file', () => { + const workflow = fs.readFileSync(buildWorkflowPath, 'utf8'); + const releasedTooling = [ + ...fs + .readFileSync(snapSourceBindingPath, 'utf8') + .matchAll(/archivePath: 'tooling\/([^']+)'/g), + ].map(([, name]) => name); + + assert.ok(releasedTooling.includes('download-pinned-source.mjs')); + + // The archive must carry every build script the Linux builder needs, so + // the workflow's copy list has to stay in step with the released set. + const [, copiedBlock] = + workflow.match( + /cp \\\n((?:\s+\S+ \\\n)+)\s+"\$\{SOURCE_BUNDLE_ROOT\}\/tooling\/"/ + ) ?? []; + assert.ok(copiedBlock, 'source bundle tooling copy step'); + assert.deepEqual( + copiedBlock + .split('\n') + .map((line) => line.trim().replace(/ \\$/, '')) + .filter(Boolean) + .map((toolingPath) => path.posix.basename(toolingPath)) + .sort(), + [...releasedTooling].sort() + ); + assert.match( + workflow, + /hashFiles\([^)]*tools\/embedded-mpv\/download-pinned-source\.mjs[^)]*\)/ + ); +}); + test('uses the next mirror when the primary source is unavailable', () => { withTemporaryDirectory((temporaryDirectory) => { const archive = Buffer.from('verified source archive'); diff --git a/tools/packaging/release-snap-assets.test.mjs b/tools/packaging/release-snap-assets.test.mjs index 8f7006c1a..6619a9687 100644 --- a/tools/packaging/release-snap-assets.test.mjs +++ b/tools/packaging/release-snap-assets.test.mjs @@ -854,6 +854,7 @@ test('hashes the final source archive bytes and reads the exact packaged Snap bi const toolingFiles = [ ['embedded-mpv', 'build-linux-runtime.cjs'], ['embedded-mpv', 'build-linux-runtime.mjs'], + ['embedded-mpv', 'download-pinned-source.mjs'], ['embedded-mpv', 'generate-linux-runtime-notices.cjs'], ['embedded-mpv', 'linux-runtime-manifest.cjs'], ['embedded-mpv', 'linux-source-archive-contract.cjs'], diff --git a/tools/packaging/release-snap-source-binding.cjs b/tools/packaging/release-snap-source-binding.cjs index 966df433b..0bf6f4c54 100644 --- a/tools/packaging/release-snap-source-binding.cjs +++ b/tools/packaging/release-snap-source-binding.cjs @@ -71,6 +71,15 @@ const SOURCE_TOOLING_FILES = Object.freeze([ 'build-linux-runtime.mjs' ), }, + { + archivePath: 'tooling/download-pinned-source.mjs', + checkoutPath: path.join( + __dirname, + '..', + 'embedded-mpv', + 'download-pinned-source.mjs' + ), + }, { archivePath: 'tooling/generate-linux-runtime-notices.cjs', checkoutPath: path.join(