build(site): [SITE-07] verify embedded inspector and tokenizer sources

scripts/site-vendor/console/README.md

@@ -150,8 +150,24 @@ comes from console-feed string-utils, and Sultan Tarimo's MIT header comes from
 styled-components' rule-sheet source map. Both are independently attributed in
 the public index and protected by the notice CLI's omission checks.
 
-This is partial embedded review. The record lists replicator, simple-html-tokenizer,
-Emotion stylis/hash/cache, the Component Stack Overflow reference, and Babel/
-regenerator inside react-inspector as follow-ups. Source-map names and comment
-links are leads, not proof of a particular upstream version or complete notice
-coverage. Do not close VENDOR-08 solely because all Parcel modules reproduce.
+`embedded-sources.ts` verifies dependency source-map inventories and exact source
+transforms. Its record is `refactor/baselines/console-embedded-sources.json`.
+All 16 Babel members embedded in react-inspector match runtime 7.13.10; its single
+regenerator member matches runtime 0.13.7. The consumer map and each source member
+are hashed, and the checker rejects omitted or extra external map sources. These
+versions identify matching source content; the original full lock is not recovered.
+
+Linkifyjs 2.1.9 pins simple-html-tokenizer to Git commit
+04799f4638ec5ed903a4e5aa6e832269fa59be6b in its published package manifest. All seven
+tokenizer members exactly reproduce with the self-contained Babel 6.26.0 compiler
+and es2015 loose preset. Archive, compiler, source and output bytes are checked.
+The Git archive's SHA-512 is a measured fingerprint, not an npm registry SRI;
+the recorded immutable URL and dependency string retain its actual provenance.
+These three licenses are included in the 38 console notice outputs. The normal
+site build still executes only the owned TS build; historical reproduction now
+fetches 39 archives only when explicitly run with `--fetch`.
+
+This is partial embedded review. Remaining follow-ups are replicator, Emotion
+stylis/hash/cache and the Component Stack Overflow reference. Source-map names and
+comment links alone are leads, not proof of complete notice coverage. Do not close
+VENDOR-08 solely because all Parcel modules reproduce.

scripts/site-vendor/console/embedded-sources.ts

@@ -0,0 +1,35 @@
+import assert from 'node:assert/strict'
+import { hash } from './provenance.ts'
+
+export interface Member { archive: string, member: string, sha256: string }
+export interface MappedSource { path: string, upstream: Member }
+export interface SourceMapRecord { source: Member, externalPrefix: string, sources: MappedSource[] }
+export interface TransformedSource { source: Member, target: Member }
+export type ReadMember = (source: Member) => Uint8Array
+
+function verified(source: Member, read: ReadMember): string {
+  const bytes = read(source)
+  assert.equal(hash(bytes), source.sha256, `Embedded source member changed: ${source.archive}/${source.member}`)
+  return new TextDecoder().decode(bytes)
+}
+
+export function verifyMappedSources(record: SourceMapRecord, read: ReadMember): number {
+  const map: { sources: string[], sourcesContent: (string | null)[] } = JSON.parse(verified(record.source, read))
+  assert(record.externalPrefix.length > 0 && record.sources.length > 0, 'Empty embedded source scope')
+  const expected = record.sources.map(source => source.path)
+  assert.equal(new Set(expected).size, expected.length, 'Duplicate embedded source mapping')
+  assert.deepEqual(map.sources.filter(source => source.startsWith(record.externalPrefix)).sort(), expected.slice().sort(), 'Embedded source map inventory changed')
+  for (const source of record.sources) {
+    const index = map.sources.indexOf(source.path)
+    assert.equal(map.sourcesContent[index], verified(source.upstream, read), `Embedded source content differs: ${source.path}`)
+  }
+  return record.sources.length
+}
+
+export function verifyTransformedSources(sources: TransformedSource[], read: ReadMember, transform: (source: string) => string): number {
+  assert(sources.length > 0, 'Empty transformed source scope')
+  assert.equal(new Set(sources.map(source => `${source.target.archive}/${source.target.member}`)).size, sources.length, 'Duplicate transformed source target')
+  for (const source of sources)
+    assert.equal(transform(verified(source.source, read)), verified(source.target, read), `Embedded source transform differs: ${source.target.member}`)
+  return sources.length
+}

scripts/site-vendor/console/reproduce.ts

@@ -1,4 +1,5 @@
 import type { EmbeddedNotice } from './embedded-notices.ts'
+import type { Member, SourceMapRecord, TransformedSource } from './embedded-sources.ts'
 import type { Archive, External, Source } from './provenance.ts'
 import type { BabelRuntime, EsmSource } from './reconstruction.ts'
 import assert from 'node:assert/strict'
@@ -10,6 +11,7 @@ import path from 'node:path'
 import process from 'node:process'
 import { fileURLToPath } from 'node:url'
 import { extractNotice } from './embedded-notices.ts'
+import { verifyMappedSources, verifyTransformedSources } from './embedded-sources.ts'
 import { hash, parcelModules, verifyArchive, verifyModules, verifyPackageEdges } from './provenance.ts'
 import { reconstructModule, verifyPrelude } from './reconstruction.ts'
 
@@ -37,11 +39,19 @@ interface EsmProvenance extends SourceGroup<EsmSource> {
   parcel: { archive: Archive, prelude: { member: string, sha256: string }, footer: string, notices: Notice[], recipeMembers: { member: string, sha256: string }[] }
   supplementalNotices: { url: string, apiUrl?: string, source: string, sha256: string }[]
 }
+interface EmbeddedProvenance {
+  archives: (Archive & { id: string, notices: Notice[] })[]
+  compiler: { archive: Archive, member: string, sha256: string, version: string, options: { presets: [string, { loose: boolean }][] } }
+  sourceMaps: SourceMapRecord[]
+  tokenizer: { dependency: Member, value: string, sources: TransformedSource[] }
+}
+interface HistoricalBabel { version: string, transform: (source: string, options: EmbeddedProvenance['compiler']['options']) => { code: string } }
 
 const root = fileURLToPath(new URL('../../../', import.meta.url))
 const record: Provenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-feed-provenance.json'), 'utf8'))
 const common: SourceGroup<CommonSource> = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-commonjs-provenance.json'), 'utf8'))
 const esm: EsmProvenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-esm-provenance.json'), 'utf8'))
+const embeddedSources: EmbeddedProvenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-embedded-sources.json'), 'utf8'))
 assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce.ts [--fetch]')
 assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
 const cacheRoot = fs.realpathSync(path.join(root, 'refactor/.cache'))
@@ -60,7 +70,10 @@ async function download(url: string) {
     throw error
   }
 }
-for (const archive of [record.archive, record.compiler.archive, ...common.archives, ...esm.archives, esm.babel.archive, esm.parcel.archive]) {
+const allArchives = [record.archive, record.compiler.archive, ...common.archives, ...esm.archives, esm.babel.archive, esm.parcel.archive, ...embeddedSources.archives, embeddedSources.compiler.archive]
+const allArchiveIds = new Set(allArchives.map(archive => `${encodeURIComponent(archive.name)}-${archive.version}`))
+assert.equal(allArchiveIds.size, allArchives.length, 'Duplicate reproduction archive')
+for (const archive of allArchives) {
   const target = path.join(cache, `${encodeURIComponent(archive.name)}-${archive.version}.tgz`)
   if (process.argv.includes('--fetch')) {
     const bytes = await download(archive.tarball)
@@ -178,4 +191,33 @@ for (const notice of embedded.notices) {
   const text = extractNotice(readMember(`${notice.archive}.tgz`, notice.member), notice)
   assert.equal(fs.readFileSync(path.join(root, notice.source), 'utf8'), text, 'Frozen embedded notice changed')
 }
-console.log(JSON.stringify({ exactModules: identified.size, consoleFeed: count, commonjs: commonCount, esm: esmCount, parcelPrelude: true, packageEdges: true, embeddedNotices: embedded.notices.length, unresolved: 0, licenseClosure: false }))
+function readEmbedded(source: Member) {
+  assert(allArchiveIds.has(source.archive), 'Unknown embedded source archive')
+  return readMember(`${source.archive}.tgz`, source.member)
+}
+for (const archive of embeddedSources.archives) {
+  assert.equal(archive.id, `${encodeURIComponent(archive.name)}-${archive.version}`, 'Embedded archive ID differs')
+  for (const notice of archive.notices) {
+    assert.equal(hash(readMember(`${archive.id}.tgz`, notice.member)), notice.sha256, 'Embedded upstream license changed')
+    assert.equal(hash(fs.readFileSync(path.join(root, notice.source))), notice.sha256, 'Frozen embedded license changed')
+  }
+}
+assert.equal(embeddedSources.sourceMaps.length, 1, 'Incomplete embedded map scope')
+const mappedCount = embeddedSources.sourceMaps.reduce((total, source) => total + verifyMappedSources(source, readEmbedded), 0)
+assert.equal(mappedCount, 17, 'Incomplete react-inspector embedded scope')
+const tokenizer = embeddedSources.tokenizer
+const dependencyBytes = readEmbedded(tokenizer.dependency)
+assert.equal(hash(dependencyBytes), tokenizer.dependency.sha256, 'Tokenizer dependency manifest changed')
+const dependency: { devDependencies: Record<string, string> } = JSON.parse(dependencyBytes.toString('utf8'))
+assert.equal(dependency.devDependencies['simple-html-tokenizer'], tokenizer.value, 'Tokenizer Git dependency changed')
+const compiler = embeddedSources.compiler
+const legacyBytes = readMember(`${compiler.archive.name}-${compiler.archive.version}.tgz`, compiler.member)
+assert.equal(hash(legacyBytes), compiler.sha256, 'Embedded compiler changed')
+const legacyPath = path.join(cache, 'babel6.cjs')
+fs.writeFileSync(legacyPath, legacyBytes)
+const legacyBabel = require(legacyPath) as HistoricalBabel
+assert.equal(legacyBabel.version, compiler.version, 'Embedded compiler version changed')
+assert.deepEqual(compiler.options, { presets: [['es2015', { loose: true }]] }, 'Embedded compiler options changed')
+const transformedCount = verifyTransformedSources(tokenizer.sources, readEmbedded, source => legacyBabel.transform(source, compiler.options).code)
+assert.equal(transformedCount, 7, 'Incomplete tokenizer scope')
+console.log(JSON.stringify({ exactModules: identified.size, consoleFeed: count, commonjs: commonCount, esm: esmCount, parcelPrelude: true, packageEdges: true, embeddedNotices: embedded.notices.length, embeddedMappedSources: mappedCount, embeddedTransformedSources: transformedCount, unresolved: 0, licenseClosure: false }))