build(site): [SITE-07] verify embedded inspector and tokenizer sources

scripts/build-site-notices.mjs

@@ -22,6 +22,7 @@ assert.deepEqual(manifest.groups.filter(group => group.name !== 'console').flatM
 const consoleGroup = manifest.groups.find(group => group.name === 'console')
 assert.deepEqual(consoleGroup?.components?.map(component => component.name).sort(), [
   '@babel/runtime',
+  '@babel/runtime (react-inspector embedded)',
   '@emotion/cache',
   '@emotion/core',
   '@emotion/css',
@@ -52,12 +53,14 @@ assert.deepEqual(consoleGroup?.components?.map(component => component.name).sort
   'react-dom',
   'react-inspector',
   'react-is',
+  'regenerator-runtime',
   'scheduler',
   'shallowequal',
+  'simple-html-tokenizer',
   'styled-components',
   'stylis-rule-sheet',
 ], 'Do not silently drop verified console component attribution')
-assert.equal(consoleGroup?.notices.length, 35, 'Missing reviewed console notice')
+assert.equal(consoleGroup?.notices.length, 38, 'Missing reviewed console notice')
 const vconsoleNotices = manifest.groups.find(group => group.name === 'vconsole')?.notices.map(notice => notice.target)
 for (const name of ['LICENSE', 'MIT-LICENSE', 'ATTRIBUTION.md'])
   assert(vconsoleNotices?.includes(`docs/licenses/vconsole/${name}`), `Missing vConsole notice: ${name}`)

scripts/site-vendor/README.md

@@ -23,7 +23,7 @@ files retain their exact upstream bytes, including final blank lines.
 `yarn check:site-notices`. The write command cannot bless altered vendor assets;
 review the new archive and license evidence before changing the manifest. The
 CLI prevents accidentally dropping any of the three groups, the ten reviewed
-Monaco/vConsole components, the 35 identified console components or their notice
+Monaco/vConsole components, the 38 identified console components or their notice
 count, and vConsole's original license, supplemental MIT body and attribution.
 Component references require their runtime assets and every notice before writing.
 Source notices
@@ -59,8 +59,9 @@ output through PowerShell text redirection. No dependency installation is needed
 The desktop console's owned TS entry/view and lifecycle now build through
 `build:console` / `check:console`; see [console maintenance](console/README.md).
 Its other 100 Parcel modules remain frozen and now reproduce exactly from fixed
-archives, including the Parcel loader. Its 33 package/loader licenses and two
-embedded license headers now ship under `docs/licenses/console/`. Full embedded
+archives, including the Parcel loader. Its package/loader licenses, two embedded
+headers and three additional embedded dependency licenses now ship as 38 files
+under `docs/licenses/console/`. Full embedded
 attribution remains open; the generated index explicitly preserves that boundary.
 
 Follow-up: finish Monaco's broader bundled-component notice audit, the console

scripts/site-vendor/console/README.md

@@ -150,8 +150,24 @@ comes from console-feed string-utils, and Sultan Tarimo's MIT header comes from
 styled-components' rule-sheet source map. Both are independently attributed in
 the public index and protected by the notice CLI's omission checks.
 
-This is partial embedded review. The record lists replicator, simple-html-tokenizer,
-Emotion stylis/hash/cache, the Component Stack Overflow reference, and Babel/
-regenerator inside react-inspector as follow-ups. Source-map names and comment
-links are leads, not proof of a particular upstream version or complete notice
-coverage. Do not close VENDOR-08 solely because all Parcel modules reproduce.
+`embedded-sources.ts` verifies dependency source-map inventories and exact source
+transforms. Its record is `refactor/baselines/console-embedded-sources.json`.
+All 16 Babel members embedded in react-inspector match runtime 7.13.10; its single
+regenerator member matches runtime 0.13.7. The consumer map and each source member
+are hashed, and the checker rejects omitted or extra external map sources. These
+versions identify matching source content; the original full lock is not recovered.
+
+Linkifyjs 2.1.9 pins simple-html-tokenizer to Git commit
+04799f4638ec5ed903a4e5aa6e832269fa59be6b in its published package manifest. All seven
+tokenizer members exactly reproduce with the self-contained Babel 6.26.0 compiler
+and es2015 loose preset. Archive, compiler, source and output bytes are checked.
+The Git archive's SHA-512 is a measured fingerprint, not an npm registry SRI;
+the recorded immutable URL and dependency string retain its actual provenance.
+These three licenses are included in the 38 console notice outputs. The normal
+site build still executes only the owned TS build; historical reproduction now
+fetches 39 archives only when explicitly run with `--fetch`.
+
+This is partial embedded review. Remaining follow-ups are replicator, Emotion
+stylis/hash/cache and the Component Stack Overflow reference. Source-map names and
+comment links alone are leads, not proof of complete notice coverage. Do not close
+VENDOR-08 solely because all Parcel modules reproduce.

scripts/site-vendor/console/embedded-sources.ts

@@ -0,0 +1,35 @@
+import assert from 'node:assert/strict'
+import { hash } from './provenance.ts'
+
+export interface Member { archive: string, member: string, sha256: string }
+export interface MappedSource { path: string, upstream: Member }
+export interface SourceMapRecord { source: Member, externalPrefix: string, sources: MappedSource[] }
+export interface TransformedSource { source: Member, target: Member }
+export type ReadMember = (source: Member) => Uint8Array
+
+function verified(source: Member, read: ReadMember): string {
+  const bytes = read(source)
+  assert.equal(hash(bytes), source.sha256, `Embedded source member changed: ${source.archive}/${source.member}`)
+  return new TextDecoder().decode(bytes)
+}
+
+export function verifyMappedSources(record: SourceMapRecord, read: ReadMember): number {
+  const map: { sources: string[], sourcesContent: (string | null)[] } = JSON.parse(verified(record.source, read))
+  assert(record.externalPrefix.length > 0 && record.sources.length > 0, 'Empty embedded source scope')
+  const expected = record.sources.map(source => source.path)
+  assert.equal(new Set(expected).size, expected.length, 'Duplicate embedded source mapping')
+  assert.deepEqual(map.sources.filter(source => source.startsWith(record.externalPrefix)).sort(), expected.slice().sort(), 'Embedded source map inventory changed')
+  for (const source of record.sources) {
+    const index = map.sources.indexOf(source.path)
+    assert.equal(map.sourcesContent[index], verified(source.upstream, read), `Embedded source content differs: ${source.path}`)
+  }
+  return record.sources.length
+}
+
+export function verifyTransformedSources(sources: TransformedSource[], read: ReadMember, transform: (source: string) => string): number {
+  assert(sources.length > 0, 'Empty transformed source scope')
+  assert.equal(new Set(sources.map(source => `${source.target.archive}/${source.target.member}`)).size, sources.length, 'Duplicate transformed source target')
+  for (const source of sources)
+    assert.equal(transform(verified(source.source, read)), verified(source.target, read), `Embedded source transform differs: ${source.target.member}`)
+  return sources.length
+}

scripts/site-vendor/console/reproduce.ts

@@ -1,4 +1,5 @@
 import type { EmbeddedNotice } from './embedded-notices.ts'
+import type { Member, SourceMapRecord, TransformedSource } from './embedded-sources.ts'
 import type { Archive, External, Source } from './provenance.ts'
 import type { BabelRuntime, EsmSource } from './reconstruction.ts'
 import assert from 'node:assert/strict'
@@ -10,6 +11,7 @@ import path from 'node:path'
 import process from 'node:process'
 import { fileURLToPath } from 'node:url'
 import { extractNotice } from './embedded-notices.ts'
+import { verifyMappedSources, verifyTransformedSources } from './embedded-sources.ts'
 import { hash, parcelModules, verifyArchive, verifyModules, verifyPackageEdges } from './provenance.ts'
 import { reconstructModule, verifyPrelude } from './reconstruction.ts'
 
@@ -37,11 +39,19 @@ interface EsmProvenance extends SourceGroup<EsmSource> {
   parcel: { archive: Archive, prelude: { member: string, sha256: string }, footer: string, notices: Notice[], recipeMembers: { member: string, sha256: string }[] }
   supplementalNotices: { url: string, apiUrl?: string, source: string, sha256: string }[]
 }
+interface EmbeddedProvenance {
+  archives: (Archive & { id: string, notices: Notice[] })[]
+  compiler: { archive: Archive, member: string, sha256: string, version: string, options: { presets: [string, { loose: boolean }][] } }
+  sourceMaps: SourceMapRecord[]
+  tokenizer: { dependency: Member, value: string, sources: TransformedSource[] }
+}
+interface HistoricalBabel { version: string, transform: (source: string, options: EmbeddedProvenance['compiler']['options']) => { code: string } }
 
 const root = fileURLToPath(new URL('../../../', import.meta.url))
 const record: Provenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-feed-provenance.json'), 'utf8'))
 const common: SourceGroup<CommonSource> = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-commonjs-provenance.json'), 'utf8'))
 const esm: EsmProvenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-esm-provenance.json'), 'utf8'))
+const embeddedSources: EmbeddedProvenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/console-embedded-sources.json'), 'utf8'))
 assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce.ts [--fetch]')
 assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
 const cacheRoot = fs.realpathSync(path.join(root, 'refactor/.cache'))
@@ -60,7 +70,10 @@ async function download(url: string) {
     throw error
   }
 }
-for (const archive of [record.archive, record.compiler.archive, ...common.archives, ...esm.archives, esm.babel.archive, esm.parcel.archive]) {
+const allArchives = [record.archive, record.compiler.archive, ...common.archives, ...esm.archives, esm.babel.archive, esm.parcel.archive, ...embeddedSources.archives, embeddedSources.compiler.archive]
+const allArchiveIds = new Set(allArchives.map(archive => `${encodeURIComponent(archive.name)}-${archive.version}`))
+assert.equal(allArchiveIds.size, allArchives.length, 'Duplicate reproduction archive')
+for (const archive of allArchives) {
   const target = path.join(cache, `${encodeURIComponent(archive.name)}-${archive.version}.tgz`)
   if (process.argv.includes('--fetch')) {
     const bytes = await download(archive.tarball)
@@ -178,4 +191,33 @@ for (const notice of embedded.notices) {
   const text = extractNotice(readMember(`${notice.archive}.tgz`, notice.member), notice)
   assert.equal(fs.readFileSync(path.join(root, notice.source), 'utf8'), text, 'Frozen embedded notice changed')
 }
-console.log(JSON.stringify({ exactModules: identified.size, consoleFeed: count, commonjs: commonCount, esm: esmCount, parcelPrelude: true, packageEdges: true, embeddedNotices: embedded.notices.length, unresolved: 0, licenseClosure: false }))
+function readEmbedded(source: Member) {
+  assert(allArchiveIds.has(source.archive), 'Unknown embedded source archive')
+  return readMember(`${source.archive}.tgz`, source.member)
+}
+for (const archive of embeddedSources.archives) {
+  assert.equal(archive.id, `${encodeURIComponent(archive.name)}-${archive.version}`, 'Embedded archive ID differs')
+  for (const notice of archive.notices) {
+    assert.equal(hash(readMember(`${archive.id}.tgz`, notice.member)), notice.sha256, 'Embedded upstream license changed')
+    assert.equal(hash(fs.readFileSync(path.join(root, notice.source))), notice.sha256, 'Frozen embedded license changed')
+  }
+}
+assert.equal(embeddedSources.sourceMaps.length, 1, 'Incomplete embedded map scope')
+const mappedCount = embeddedSources.sourceMaps.reduce((total, source) => total + verifyMappedSources(source, readEmbedded), 0)
+assert.equal(mappedCount, 17, 'Incomplete react-inspector embedded scope')
+const tokenizer = embeddedSources.tokenizer
+const dependencyBytes = readEmbedded(tokenizer.dependency)
+assert.equal(hash(dependencyBytes), tokenizer.dependency.sha256, 'Tokenizer dependency manifest changed')
+const dependency: { devDependencies: Record<string, string> } = JSON.parse(dependencyBytes.toString('utf8'))
+assert.equal(dependency.devDependencies['simple-html-tokenizer'], tokenizer.value, 'Tokenizer Git dependency changed')
+const compiler = embeddedSources.compiler
+const legacyBytes = readMember(`${compiler.archive.name}-${compiler.archive.version}.tgz`, compiler.member)
+assert.equal(hash(legacyBytes), compiler.sha256, 'Embedded compiler changed')
+const legacyPath = path.join(cache, 'babel6.cjs')
+fs.writeFileSync(legacyPath, legacyBytes)
+const legacyBabel = require(legacyPath) as HistoricalBabel
+assert.equal(legacyBabel.version, compiler.version, 'Embedded compiler version changed')
+assert.deepEqual(compiler.options, { presets: [['es2015', { loose: true }]] }, 'Embedded compiler options changed')
+const transformedCount = verifyTransformedSources(tokenizer.sources, readEmbedded, source => legacyBabel.transform(source, compiler.options).code)
+assert.equal(transformedCount, 7, 'Incomplete tokenizer scope')
+console.log(JSON.stringify({ exactModules: identified.size, consoleFeed: count, commonjs: commonCount, esm: esmCount, parcelPrelude: true, packageEdges: true, embeddedNotices: embedded.notices.length, embeddedMappedSources: mappedCount, embeddedTransformedSources: transformedCount, unresolved: 0, licenseClosure: false }))

scripts/site-vendor/manifest.json

@@ -944,7 +944,7 @@
       "name": "console",
       "version": "legacy-vendor-with-TS-adapter",
       "tarball": "https://github.com/zhw2590582/ArtPlayer/blob/5d6b2f22bc75213ac97cff0bcffa4a1ce2bc786c/docs/assets/js/console.js",
-      "review": "The 100 third-party Parcel modules and loader have exact source reproduction. The two owned adapter modules are built from TypeScript. Package license texts and two embedded license headers below are preserved verbatim. Embedded attribution review is still incomplete; this inventory is not publication clearance.",
+      "review": "The 100 third-party Parcel modules and loader have exact source reproduction. The two owned adapter modules are built from TypeScript. Preserved notices also cover two embedded license headers, all 17 react-inspector external source-map members, and all seven HTML tokenizer sources from its fixed Git dependency. Embedded attribution review is still incomplete; this inventory is not publication clearance.",
       "roots": [
         "docs/assets/js/console.js"
       ],
@@ -1130,6 +1130,21 @@
           "source": "refactor/baselines/site-vendor/console-embedded/stylis-rule-sheet-LICENSE.txt",
           "target": "docs/licenses/console/stylis-rule-sheet/LICENSE",
           "sha256": "99a75da65634b772687781c054ffe679569c1770c398f96427677899fe0ca8d7"
+        },
+        {
+          "source": "refactor/baselines/site-vendor/console-embedded/babel-runtime-7.13.10-LICENSE.txt",
+          "target": "docs/licenses/console/react-inspector-babel/LICENSE",
+          "sha256": "117da2af0d4ce0fe1c8e19b5cff9dcd806adf973d328d27b11d4448c4ff24f76"
+        },
+        {
+          "source": "refactor/baselines/site-vendor/console-embedded/regenerator-runtime-0.13.7-LICENSE.txt",
+          "target": "docs/licenses/console/regenerator-runtime/LICENSE",
+          "sha256": "51887a3d47051ac2fce1210562e5b9fe0830a8a8fabeb272c2d586eeb18a05fd"
+        },
+        {
+          "source": "refactor/baselines/site-vendor/console-embedded/simple-html-tokenizer-04799f4-LICENSE.txt",
+          "target": "docs/licenses/console/simple-html-tokenizer/LICENSE",
+          "sha256": "c41881de2a9f2200936648343500524be154eb79f649582f9582dd251b051b11"
         }
       ],
       "components": [
@@ -1517,6 +1532,39 @@
           "notices": [
             "docs/licenses/console/stylis-rule-sheet/LICENSE"
           ]
+        },
+        {
+          "name": "@babel/runtime (react-inspector embedded)",
+          "version": "7.13.10",
+          "tarball": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.13.10.tgz",
+          "assets": [
+            "docs/assets/js/console.js"
+          ],
+          "notices": [
+            "docs/licenses/console/react-inspector-babel/LICENSE"
+          ]
+        },
+        {
+          "name": "regenerator-runtime",
+          "version": "0.13.7",
+          "tarball": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz",
+          "assets": [
+            "docs/assets/js/console.js"
+          ],
+          "notices": [
+            "docs/licenses/console/regenerator-runtime/LICENSE"
+          ]
+        },
+        {
+          "name": "simple-html-tokenizer",
+          "version": "git-04799f4638ec5ed903a4e5aa6e832269fa59be6b",
+          "tarball": "https://api.github.com/repos/nfrasser/simple-html-tokenizer/tarball/04799f4638ec5ed903a4e5aa6e832269fa59be6b",
+          "assets": [
+            "docs/assets/js/console.js"
+          ],
+          "notices": [
+            "docs/licenses/console/simple-html-tokenizer/LICENSE"
+          ]
         }
       ]
     }