build(site): [SITE-07] verify modified forks and preserve upstream author notices

scripts/site-vendor/console/README.md

@@ -163,11 +163,28 @@ tokenizer members exactly reproduce with the self-contained Babel 6.26.0 compile
 and es2015 loose preset. Archive, compiler, source and output bytes are checked.
 The Git archive's SHA-512 is a measured fingerprint, not an npm registry SRI;
 the recorded immutable URL and dependency string retain its actual provenance.
-These three licenses are included in the 38 console notice outputs. The normal
+These three licenses are included in the console notice outputs. The normal
 site build still executes only the owned TS build; historical reproduction now
-fetches 39 archives only when explicitly run with `--fetch`.
+fetches 42 archives and fixed Git sources only when explicitly run with `--fetch`.
 
-This is partial embedded review. Remaining follow-ups are replicator, Emotion
-stylis/hash/cache and the Component Stack Overflow reference. Source-map names and
-comment links alone are leads, not proof of complete notice coverage. Do not close
-VENDOR-08 solely because all Parcel modules reproduce.
+`attribution.ts` verifies immutable Git content against SHA-256 and Git blob SHA-1,
+and compares compiled forks with the archived runtime before its single terminal
+inline source map. `console-derived-attribution.json` pins the console-feed
+replicator TS revision and TypeScript 4.1.2 ES3/CommonJS/LF compiler options. The
+fork's runtime matches exactly; it is not a verbatim npm replicator release.
+Ivan Nikulin's original MIT notice is retained from the 1.0.x source family.
+Historical compiler extraction permits 16 MiB because this fixed compiler member
+is 9,002,076 bytes; it is not an installed workspace dependency.
+
+Emotion's Stylis source matches its fixed tag commit and declares Stylis 3.5.4.
+The original build recipe modifies Stylis and uses an unversioned online Closure
+service. We preserve that recipe and the matching generated source without claiming
+to rerun the earlier service. Sultan Tarimo's MIT notice covers the upstream
+attribution, including the separately identified rule-sheet reference in cache.
+Gary Court's original README/MIT text and Austin Appleby's public-domain header
+are preserved for the two explicit MurmurHash source references. Original sources
+and excerpts are hashed; attribution is never inferred from the outer Emotion MIT.
+
+There are now 42 console notice outputs. Remaining follow-ups are the Component
+Stack Overflow reference and final embedded-source review. Do not close VENDOR-08
+solely because all Parcel modules reproduce.

scripts/site-vendor/console/attribution.ts

@@ -0,0 +1,29 @@
+import assert from 'node:assert/strict'
+import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { hash } from './provenance.ts'
+
+export interface GitSource { source: string, sha256: string, gitBlobSha: string, apiUrl: string }
+
+export function verifyGitSource(bytes: Uint8Array, source: GitSource): void {
+  assert.equal(hash(bytes), source.sha256, 'Fixed Git source content changed')
+  const blob = createHash('sha1').update(`blob ${bytes.byteLength}\0`).update(bytes).digest('hex')
+  assert.equal(blob, source.gitBlobSha, 'Fixed Git blob identity differs')
+}
+
+export function decodeGitSource(response: Uint8Array, source: GitSource): Uint8Array {
+  const data: { encoding: string, content: string } = JSON.parse(Buffer.from(response).toString('utf8'))
+  assert.equal(data.encoding, 'base64', 'Unexpected Git source encoding')
+  assert.equal(typeof data.content, 'string', 'Missing Git source content')
+  const bytes = Buffer.from(data.content, 'base64')
+  verifyGitSource(bytes, source)
+  return bytes
+}
+
+export function verifyForkOutput(compiled: string, target: string): void {
+  const marker = '//# sourceMappingURL=data:application/json;base64,'
+  const start = target.indexOf(marker)
+  assert(start >= 0 && !target.includes(marker, start + 1), 'Expected one terminal inline source map')
+  assert(/^[A-Z0-9+/=]+$/i.test(target.slice(start + marker.length)), 'Unexpected inline source map trailer')
+  assert.equal(compiled, target.slice(0, start), 'Fork compiled runtime differs')
+}

scripts/site-vendor/console/reproduce.ts

@@ -1,3 +1,4 @@
+import type { GitSource } from './attribution.ts'
 import type { EmbeddedNotice } from './embedded-notices.ts'
 import type { Member, SourceMapRecord, TransformedSource } from './embedded-sources.ts'
 import type { Archive, External, Source } from './provenance.ts'
@@ -10,6 +11,7 @@ import { createRequire } from 'node:module'
 import path from 'node:path'
 import process from 'node:process'
 import { fileURLToPath } from 'node:url'
+import { decodeGitSource, verifyForkOutput, verifyGitSource } from './attribution.ts'
 import { extractNotice } from './embedded-notices.ts'
 import { verifyMappedSources, verifyTransformedSources } from './embedded-sources.ts'
 import { hash, parcelModules, verifyArchive, verifyModules, verifyPackageEdges } from './provenance.ts'
@@ -46,12 +48,23 @@ interface EmbeddedProvenance {
   tokenizer: { dependency: Member, value: string, sources: TransformedSource[] }
 }
 interface HistoricalBabel { version: string, transform: (source: string, options: EmbeddedProvenance['compiler']['options']) => { code: string } }
+interface DerivedProvenance {
+  archives: (Archive & { id: string, notices: Notice[] })[]
+  compiler: { archive: Archive, member: string, sha256: string, version: string, options: { target: number, module: number, newLine: number } }
+  remotes: (GitSource & { id: string })[]
+  replicator: { source: string, target: Member, upstream: Member }
+  stylis: { source: string, target: Member, dependency: Member, dependencyVersion: string, recipe: string }
+  hash: { source: string, notice: EmbeddedNotice, reference: Member, references: string[] }
+  cache: { reference: Member, upstreamUrl: string, notice: string }
+}
+interface HistoricalTypeScript { version: string, transpileModule: (source: string, options: { compilerOptions: DerivedProvenance['compiler']['options'] }) => { outputText: string } }
 
 const root = fileURLToPath(new URL('../../../', import.meta.url))
 const record: Provenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-feed-provenance.json'), 'utf8'))
 const common: SourceGroup<CommonSource> = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-commonjs-provenance.json'), 'utf8'))
 const esm: EsmProvenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-esm-provenance.json'), 'utf8'))
 const embeddedSources: EmbeddedProvenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-embedded-sources.json'), 'utf8'))
+const derived: DerivedProvenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-derived-attribution.json'), 'utf8'))
 assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce.ts [--fetch]')
 assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
 const cacheRoot = fs.realpathSync(path.join(root, 'refactor/.cache'))
@@ -70,7 +83,7 @@ async function download(url: string) {
     throw error
   }
 }
-const allArchives = [record.archive, record.compiler.archive, ...common.archives, ...esm.archives, esm.babel.archive, esm.parcel.archive, ...embeddedSources.archives, embeddedSources.compiler.archive]
+const allArchives = [record.archive, record.compiler.archive, ...common.archives, ...esm.archives, esm.babel.archive, esm.parcel.archive, ...embeddedSources.archives, embeddedSources.compiler.archive, ...derived.archives, derived.compiler.archive]
 const allArchiveIds = new Set(allArchives.map(archive => `${encodeURIComponent(archive.name)}-${archive.version}`))
 assert.equal(allArchiveIds.size, allArchives.length, 'Duplicate reproduction archive')
 for (const archive of allArchives) {
@@ -82,7 +95,7 @@ for (const archive of allArchives) {
   }
   verifyArchive(fs.readFileSync(target), archive)
 }
-const readMember = (name: string, member: string) => execFileSync('tar', ['-xOzf', path.join(cache, name), member], { maxBuffer: 8 * 1024 * 1024 })
+const readMember = (name: string, member: string) => execFileSync('tar', ['-xOzf', path.join(cache, name), member], { maxBuffer: 16 * 1024 * 1024 })
 const compilerBytes = readMember('terser-3.17.0.tgz', record.compiler.member)
 assert.equal(hash(compilerBytes), record.compiler.sha256, 'Compiler member changed')
 const compilerPath = path.join(cache, 'terser.cjs')
@@ -195,7 +208,7 @@ function readEmbedded(source: Member) {
   assert(allArchiveIds.has(source.archive), 'Unknown embedded source archive')
   return readMember(`${source.archive}.tgz`, source.member)
 }
-for (const archive of embeddedSources.archives) {
+for (const archive of [...embeddedSources.archives, ...derived.archives]) {
   assert.equal(archive.id, `${encodeURIComponent(archive.name)}-${archive.version}`, 'Embedded archive ID differs')
   for (const notice of archive.notices) {
     assert.equal(hash(readMember(`${archive.id}.tgz`, notice.member)), notice.sha256, 'Embedded upstream license changed')
@@ -220,4 +233,46 @@ assert.equal(legacyBabel.version, compiler.version, 'Embedded compiler version c
 assert.deepEqual(compiler.options, { presets: [['es2015', { loose: true }]] }, 'Embedded compiler options changed')
 const transformedCount = verifyTransformedSources(tokenizer.sources, readEmbedded, source => legacyBabel.transform(source, compiler.options).code)
 assert.equal(transformedCount, 7, 'Incomplete tokenizer scope')
-console.log(JSON.stringify({ exactModules: identified.size, consoleFeed: count, commonjs: commonCount, esm: esmCount, parcelPrelude: true, packageEdges: true, embeddedNotices: embedded.notices.length, embeddedMappedSources: mappedCount, embeddedTransformedSources: transformedCount, unresolved: 0, licenseClosure: false }))
+const gitSources = new Map<string, Buffer>()
+assert.equal(derived.remotes.length, 5, 'Incomplete derived Git source scope')
+for (const source of derived.remotes) {
+  assert(!gitSources.has(source.id), 'Duplicate derived Git source')
+  const bytes = fs.readFileSync(path.join(root, source.source))
+  verifyGitSource(bytes, source)
+  if (process.argv.includes('--fetch'))
+    decodeGitSource(await download(source.apiUrl), source)
+  gitSources.set(source.id, bytes)
+}
+function gitSource(id: string): Buffer {
+  const bytes = gitSources.get(id)
+  assert(bytes, `Missing derived Git source: ${id}`)
+  return bytes
+}
+function derivedMember(member: Member): Buffer {
+  const bytes = readEmbedded(member)
+  assert.equal(hash(bytes), member.sha256, 'Derived source member changed')
+  return bytes
+}
+const tsCompiler = derived.compiler
+const tsBytes = readMember(`${tsCompiler.archive.name}-${tsCompiler.archive.version}.tgz`, tsCompiler.member)
+assert.equal(hash(tsBytes), tsCompiler.sha256, 'Historical TypeScript compiler changed')
+const tsPath = path.join(cache, 'typescript4.cjs')
+fs.writeFileSync(tsPath, tsBytes)
+const historicalTs = require(tsPath) as HistoricalTypeScript
+assert.equal(historicalTs.version, tsCompiler.version, 'Historical TypeScript version changed')
+assert.deepEqual(tsCompiler.options, { target: 0, module: 1, newLine: 1 }, 'Historical TypeScript options changed')
+const fork = historicalTs.transpileModule(gitSource(derived.replicator.source).toString('utf8'), { compilerOptions: tsCompiler.options }).outputText
+verifyForkOutput(fork, derivedMember(derived.replicator.target).toString('utf8'))
+derivedMember(derived.replicator.upstream)
+assert(gitSource(derived.stylis.source).equals(derivedMember(derived.stylis.target)), 'Emotion Git/npm source differs')
+const stylisPackage: { devDependencies: { stylis: string } } = JSON.parse(derivedMember(derived.stylis.dependency).toString('utf8'))
+assert.equal(stylisPackage.devDependencies.stylis, derived.stylis.dependencyVersion, 'Emotion Stylis dependency changed')
+assert(gitSource(derived.stylis.recipe).toString('utf8').includes('https://closure-compiler.appspot.com/compile'), 'Historical Closure recipe changed')
+const murmurSource = derivedMember(derived.hash.reference).toString('utf8')
+for (const reference of derived.hash.references)
+  assert(murmurSource.includes(reference), 'MurmurHash source attribution changed')
+const publicDomain = extractNotice(gitSource(derived.hash.source), derived.hash.notice)
+assert.equal(fs.readFileSync(path.join(root, derived.hash.notice.source), 'utf8'), publicDomain, 'Public domain notice changed')
+assert(derivedMember(derived.cache.reference).toString('utf8').includes(derived.cache.upstreamUrl), 'Rule-sheet source attribution changed')
+assert(fs.readFileSync(path.join(root, derived.cache.notice), 'utf8').includes('Copyright (c) 2016 Sultan Tarimo'), 'Rule-sheet attribution missing')
+console.log(JSON.stringify({ exactModules: identified.size, consoleFeed: count, commonjs: commonCount, esm: esmCount, parcelPrelude: true, packageEdges: true, embeddedNotices: embedded.notices.length, embeddedMappedSources: mappedCount, embeddedTransformedSources: transformedCount, derivedGitSources: gitSources.size, replicatorFork: true, emotionStylisSource: true, unresolved: 0, licenseClosure: false }))