build(site): [SITE-07] reproduce historical Unicode data and ship source notices

scripts/build-site-notices.mjs

@@ -6,6 +6,7 @@ import { verifyMonacoCoreNotices } from './site-vendor/monaco/core-origins.ts'
 import { verifyMonacoDomNotices } from './site-vendor/monaco/dom-origins.ts'
 import { verifyMonacoPathNotices } from './site-vendor/monaco/node-path.ts'
 import { verifyMonacoLanguageNotices } from './site-vendor/monaco/notices.ts'
+import { verifyMonacoUnicodeNotices } from './site-vendor/monaco/unicode-origins.ts'
 import { writeOrCheckNotices } from './site-vendor/notices.ts'
 
 assert(process.argv.slice(2).every(arg => arg === '--check'), 'Use yarn build:site-notices [--check]')
@@ -15,6 +16,7 @@ assert.deepEqual(manifest.groups.map(group => group.name).sort(), ['console', 'm
 assert.deepEqual(manifest.groups.filter(group => group.name !== 'console').flatMap(group => (group.components || []).map(component => component.name)).sort(), [
   '@babel/runtime',
   '@vscode/codicons',
+  'Unicode data (Monaco core)',
   'WinJS-derived DOM helpers (Monaco core)',
   'copy-text-to-clipboard',
   'core-js',
@@ -94,6 +96,7 @@ for (const name of ['LICENSE', 'MIT-LICENSE', 'ATTRIBUTION.md'])
 verifyConsoleNoticeSources(process.cwd(), manifest)
 verifyMonacoCoreNotices(process.cwd(), manifest)
 verifyMonacoDomNotices(process.cwd(), manifest)
+verifyMonacoUnicodeNotices(process.cwd(), manifest)
 verifyMonacoPathNotices(process.cwd(), manifest)
 verifyMonacoLanguageNotices(process.cwd(), manifest)
 const count = writeOrCheckNotices(process.cwd(), manifest, process.argv.includes('--check'))

scripts/site-vendor/manifest.json

@@ -1035,6 +1035,16 @@
           "source": "refactor/baselines/site-vendor/monaco-dom/ATTRIBUTION.md.txt",
           "target": "docs/licenses/monaco-editor/core-dom/ATTRIBUTION.md",
           "sha256": "0858d19261742809cecd7174a0a06b2b0013a4b9560b78aede1edd3dd350da21"
+        },
+        {
+          "source": "refactor/baselines/site-vendor/monaco-unicode/LICENSE.txt",
+          "target": "docs/licenses/monaco-editor/core-unicode/LICENSE.txt",
+          "sha256": "80b381b2f157c31e7f3222bdb932bda9fbd7b335e888beec56820fb2993363de"
+        },
+        {
+          "source": "refactor/baselines/site-vendor/monaco-unicode/ATTRIBUTION.md.txt",
+          "target": "docs/licenses/monaco-editor/core-unicode/ATTRIBUTION.md",
+          "sha256": "51d583153fd1ed701a2b8810b3b1994d337002f074255e75a7d06790a381a1a6"
         }
       ],
       "review": "The complete upstream Monaco LICENSE/ThirdPartyNotices are retained. The unmodified bundled Codicons font exactly matches @vscode/codicons 0.0.26; its historical README, CC BY 4.0 content license, MIT code license and added attribution are distributed below. Other site assets remain subject to their separate provenance reviews.",
@@ -1239,6 +1249,19 @@
             "docs/licenses/monaco-editor/core-dom/LICENSE.txt",
             "docs/licenses/monaco-editor/core-dom/ATTRIBUTION.md"
           ]
+        },
+        {
+          "name": "Unicode data (Monaco core)",
+          "version": "mixed historical inputs: 10.0.0d5, 13.0 drafts, Emoji 13.1",
+          "tarball": "https://github.com/alexdima/unicode-utils",
+          "assets": [
+            "docs/assets/js/vs/editor/editor.main.js",
+            "docs/assets/js/vs/base/worker/workerMain.js"
+          ],
+          "notices": [
+            "docs/licenses/monaco-editor/core-unicode/LICENSE.txt",
+            "docs/licenses/monaco-editor/core-unicode/ATTRIBUTION.md"
+          ]
         }
       ]
     },

scripts/site-vendor/monaco/README.md

@@ -388,7 +388,7 @@ Do not replace this distinction with a guessed version during later maintenance.
 
 The ordinary notice build binds the reference license and attribution to the
 editor asset; it rejects omitted terms or a changed source/asset relationship.
-The site now carries 88 notice files plus its index. Frozen source and delivered
+That checkpoint added two notices, bringing the site to 88 files plus its index. Frozen source and delivered
 terms preserve original bytes through Git attributes. No dependencies or editor
 runtime bytes changed; the verifier uses the existing TypeScript parser.
 
@@ -406,3 +406,50 @@ native geometry. The site test checks all notice HTTP bytes, relative links and
 real mobile-page playback. Unicode/other embedded sources, unknown original
 WinJS version, complete original core compilation and SITE-07 acceptance remain
 open. Source comments and this bounded proof are not full provenance clearance.
+
+## Historical Unicode data
+
+`unicode.ts` extracts the four top-level data expressions using TypeScript 5.9.3
+AST boundaries. It compares regex text exactly, the imprecise emoji predicate as
+tokens (ignoring formatting/comments only), and every integer of the grapheme
+tree. `unicode-origins.ts` binds the complete historical Unicode terms and
+attribution to the shipped editor and base worker. The ordinary notice build now
+delivers 90 notice files plus the index; it does not run old dependencies.
+
+`reproduce-unicode.ts` checks 15 fixed Git blobs, two npm archives, both original
+source maps and both shipped assets. It replays three original generator scripts
+against frozen inputs. Their dependency closure consists of 17 locked archives,
+including regexpu 3.3.0. Original lock integrity (SHA-1 or SHA-512) plus independent
+SHA-512/SHA-256 is verified before individual member extraction into the isolated
+`refactor/.cache/monaco-review/unicode-generator/compiler/node_modules` tree.
+No install/lifecycle scripts run, and root dependencies/yarn.lock are unchanged.
+
+The input versions deliberately differ: RTL uses the 2017 Unicode 10.0.0d5
+snapshot, emoji uses Unicode 13.0 plus Emoji 13.1, and grapheme data uses the 2019
+Unicode 13 draft snapshot. That last combination reproduces all 5,034 integers.
+Version labels come from the frozen fetch recipes and file headers; do not
+replace them with a guessed single final Unicode release. Generator package
+metadata declares MIT and Alex Dima; its frozen tree has no standalone LICENSE.
+The complete Unicode terms are a fixed 2021 official repository reference,
+explicitly identified as such rather than asserted to be an original bundled file.
+
+Generator execution allows only the recorded data reads, captures expected output
+files in memory and rejects unknown imports, unused inputs and missing/repeated
+outputs. The VM is an I/O adapter for verified historical code, not a security
+sandbox for arbitrary scripts. Never point it at unverified executable sources.
+
+```sh
+yarn verify:monaco-unicode --fetch
+yarn verify:monaco-unicode
+node --test test/monaco-unicode.test.js test/monaco-provenance.test.js test/site-notices.test.js
+yarn typecheck:docs-tools
+yarn check:site-notices
+yarn test:browser editor-unicode.spec.js site-vendor.spec.js --workers=1
+```
+
+Offline reproduction needs the verified archive cache from the first command.
+Unit tests guard data substitution, expression boundaries and recipe I/O. Browser
+tests load the actual shipped Monaco module, exercise RTL/emoji/grapheme cases and
+mixed-text model edit/undo, and verify HTTP notices plus mobile-page playback.
+These checks are not full Unicode conformance or full original Monaco compilation;
+other embedded origins and the remaining SITE-07 work stay open.

scripts/site-vendor/monaco/reproduce-unicode.ts

@@ -0,0 +1,58 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+import fs from 'node:fs'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import process from 'node:process'
+import { fileURLToPath } from 'node:url'
+import { ArchiveCache } from './archives.ts'
+import { readUnicodeOrigins, verifyMonacoUnicodeNotices } from './unicode-origins.ts'
+import { readUnicodeTables, replayUnicodeGenerator, verifyUnicodeGeneration } from './unicode.ts'
+
+const root = fileURLToPath(new URL('../../../', import.meta.url))
+assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce-unicode.ts [--fetch]')
+assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
+const record = readUnicodeOrigins(root)
+const cache = new ArchiveCache(root, path.join(root, 'refactor/.cache/monaco-review'))
+const generators = new ArchiveCache(root, path.join(cache.directory, 'unicode-generator'))
+await cache.verify(record.archives, record.remotes, process.argv.includes('--fetch'))
+await generators.verify(record.generatorArchives, [], process.argv.includes('--fetch'))
+verifyMonacoUnicodeNotices(root, JSON.parse(fs.readFileSync(path.join(root, 'scripts/site-vendor/manifest.json'), 'utf8')))
+const lock: { lockfileVersion: number, dependencies: Record<string, { version: string, resolved: string, integrity: string, dependencies?: unknown }> } = JSON.parse(fs.readFileSync(path.join(root, record.lockSource), 'utf8'))
+assert.equal(lock.lockfileVersion, 1)
+assert.deepEqual(record.generatorArchives.map(archive => archive.name).sort(), Object.keys(lock.dependencies).sort(), 'Incomplete Unicode dependency closure')
+for (const archive of record.generatorArchives) {
+  const dependency = lock.dependencies[archive.name]!
+  assert.equal(dependency.dependencies, undefined, 'Unexpected nested Unicode dependency')
+  assert.deepEqual([archive.version, archive.tarball, archive.lockedIntegrity], [dependency.version, dependency.resolved, dependency.integrity], 'Changed Unicode lock resolution')
+  const bytes = fs.readFileSync(path.join(generators.directory, `${archive.name}-${archive.version}.tgz`))
+  const algorithm = archive.lockedIntegrity.split('-')[0]!
+  assert(['sha1', 'sha512'].includes(algorithm), 'Unexpected original Unicode integrity algorithm')
+  assert.equal(`${algorithm}-${createHash(algorithm).update(bytes).digest('base64')}`, archive.lockedIntegrity, 'Changed original Unicode archive integrity')
+  generators.extractCompiler(archive)
+}
+const require = createRequire(path.join(generators.directory, 'compiler/entry.cjs'))
+assert.equal(require('regexpu/package.json').version, '3.3.0')
+
+const recipeSources = new Set(record.remotes.map(remote => remote.source))
+assert.deepEqual(record.generators.map(generator => generator.kind), ['rtl', 'emoji', 'grapheme'])
+const outputs = record.generators.map((generator) => {
+  assert(recipeSources.has(generator.source), 'Unverified Unicode generator')
+  const inputs = new Map(Object.entries(generator.inputs).map(([name, source]) => {
+    assert(recipeSources.has(source), 'Unverified Unicode data input')
+    return [name, fs.readFileSync(path.join(root, source))]
+  }))
+  return replayUnicodeGenerator(fs.readFileSync(path.join(root, generator.source), 'utf8'), inputs, generator.outputs, require('regexpu'))
+})
+const tables = readUnicodeTables(fs.readFileSync(path.join(root, record.vscodeSource), 'utf8'))
+verifyUnicodeGeneration(tables, outputs[0]!, outputs[1]!, outputs[2]!)
+assert.deepEqual(record.maps.map(entry => entry.member), ['package/dev/vs/editor/editor.main.js.map', 'package/dev/vs/base/worker/workerMain.js.map'], 'Incomplete Unicode map coverage')
+for (const entry of record.maps) {
+  const map: { sources: string[], sourcesContent: string[] } = JSON.parse(cache.member(entry).toString('utf8'))
+  assert.equal(map.sources.filter(source => source === entry.source).length, 1, 'Missing or repeated Unicode mapped source')
+  assert.deepEqual(readUnicodeTables(map.sourcesContent[map.sources.indexOf(entry.source)]!), tables, 'Mapped Unicode differs from fixed VS Code')
+}
+assert.deepEqual(record.shipped.map(asset => asset.target), record.component.assets, 'Incomplete Unicode shipped coverage')
+for (const asset of record.shipped)
+  assert.deepEqual(fs.readFileSync(path.join(root, asset.target)), cache.member(asset), 'Shipped Unicode asset changed')
+console.log(JSON.stringify({ archives: record.archives.length, generatorArchives: record.generatorArchives.length, gitSources: record.remotes.length, maps: record.maps.length, shippedAssets: record.shipped.length, exactGenerated: record.generated, fullMonacoCompilation: false, unicodeConformance: false }))

scripts/site-vendor/monaco/unicode-origins.ts

@@ -0,0 +1,46 @@
+import type { VendorManifest } from '../notices.ts'
+import type { Archive, Member, Remote } from './archives.ts'
+import type { UnicodeTables } from './unicode.ts'
+import assert from 'node:assert/strict'
+import { Buffer } from 'node:buffer'
+import fs from 'node:fs'
+import path from 'node:path'
+import { hash } from './archives.ts'
+import { readUnicodeTables } from './unicode.ts'
+
+export interface UnicodeOrigins {
+  archives: Archive[]
+  generatorArchives: (Archive & { lockedIntegrity: string })[]
+  remotes: Remote[]
+  vscodeSource: string
+  packageSource: string
+  lockSource: string
+  generators: { kind: 'rtl' | 'emoji' | 'grapheme', source: string, inputs: Record<string, string>, outputs: string[] }[]
+  maps: (Member & { source: string })[]
+  shipped: (Member & { target: string })[]
+  generated: Record<keyof UnicodeTables, { sha256: string, length: number }>
+  component: { name: string, version: string, tarball: string, assets: string[], notices: string[] }
+  notices: { source: string, target: string, sha256: string }[]
+}
+
+export function readUnicodeOrigins(root: string): UnicodeOrigins {
+  return JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/monaco-unicode-provenance.json'), 'utf8'))
+}
+
+export function verifyMonacoUnicodeNotices(root: string, manifest: VendorManifest): void {
+  const record = readUnicodeOrigins(root)
+  assert.equal(record.component.name, 'Unicode data (Monaco core)')
+  assert.equal(record.component.version, 'mixed historical inputs: 10.0.0d5, 13.0 drafts, Emoji 13.1')
+  assert.deepEqual(record.component.assets, ['docs/assets/js/vs/editor/editor.main.js', 'docs/assets/js/vs/base/worker/workerMain.js'], 'Wrong Unicode origin assets')
+  assert.deepEqual(record.notices.map(notice => notice.target), ['docs/licenses/monaco-editor/core-unicode/LICENSE.txt', 'docs/licenses/monaco-editor/core-unicode/ATTRIBUTION.md'], 'Missing complete Unicode notice set')
+  assert.deepEqual(record.component.notices, record.notices.map(notice => notice.target), 'Missing Unicode attribution or terms')
+  const group = manifest.groups.find(group => group.name === 'monaco-editor')
+  assert(group, 'Missing Monaco Unicode group')
+  assert.deepEqual(group.components?.filter(item => item.name === record.component.name), [record.component], 'Wrong Monaco Unicode notice binding')
+  for (const notice of record.notices) {
+    assert.deepEqual(group.notices.filter(item => item.target === notice.target), [notice], 'Missing Monaco Unicode notice')
+    assert.equal(hash(fs.readFileSync(path.join(root, notice.source))), notice.sha256, 'Changed Monaco Unicode terms')
+  }
+  const tables = readUnicodeTables(fs.readFileSync(path.join(root, record.vscodeSource), 'utf8'))
+  assert.deepEqual(record.generated, Object.fromEntries(Object.entries(tables).map(([name, value]) => [name, { sha256: hash(Buffer.from(typeof value === 'string' ? value : JSON.stringify(value))), length: value.length }])), 'Changed Unicode data fragments')
+}

scripts/site-vendor/monaco/unicode.ts

@@ -0,0 +1,102 @@
+import assert from 'node:assert/strict'
+import vm from 'node:vm'
+import ts from 'typescript'
+
+export interface UnicodeTables { rtl: string, emoji: string, imprecise: string, grapheme: number[] }
+export interface UnicodeOutput { logs: string[], writes: Map<string, string> }
+
+function expressionText(expression: ts.Expression, file: ts.SourceFile): string {
+  while (ts.isParenthesizedExpression(expression))
+    expression = expression.expression
+  const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.Standard, expression.getText(file))
+  const tokens: string[] = []
+  while (scanner.scan() !== ts.SyntaxKind.EndOfFileToken)
+    tokens.push(scanner.getTokenText())
+  return tokens.join(' ')
+}
+
+export function readUnicodeTables(source: string): UnicodeTables {
+  const file = ts.createSourceFile('strings.ts', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS)
+  const found = new Map<string, ts.Expression>()
+  const put = (name: string, value: ts.Expression | undefined) => {
+    assert(value && !found.has(name), 'Missing or repeated Unicode declaration')
+    found.set(name, value)
+  }
+  for (const node of file.statements) {
+    if (ts.isVariableStatement(node)) {
+      for (const declaration of node.declarationList.declarations) {
+        if (ts.isIdentifier(declaration.name) && ['CONTAINS_RTL', 'CONTAINS_EMOJI'].includes(declaration.name.text))
+          put(declaration.name.text, declaration.initializer)
+      }
+    }
+    if (ts.isFunctionDeclaration(node) && node.name && ['isEmojiImprecise', 'getGraphemeBreakRawData'].includes(node.name.text)) {
+      assert.equal(node.body?.statements.length, 1, 'Unexpected Unicode function body')
+      const statement = node.body.statements[0]!
+      assert(ts.isReturnStatement(statement), 'Expected Unicode return')
+      put(node.name.text, statement.expression)
+    }
+  }
+  assert.equal(found.size, 4, 'Missing Unicode declarations')
+  const regex = (name: string) => {
+    const node = found.get(name)!
+    assert(ts.isRegularExpressionLiteral(node), 'Expected Unicode regex literal')
+    assert(node.text.startsWith('/') && node.text.endsWith('/'), 'Unexpected Unicode regex flags')
+    return node.text.slice(1, -1)
+  }
+  const tree = found.get('getGraphemeBreakRawData')!
+  assert(ts.isCallExpression(tree) && tree.expression.getText(file) === 'JSON.parse' && tree.arguments.length === 1 && ts.isStringLiteral(tree.arguments[0]!), 'Expected literal Unicode tree')
+  const grapheme: unknown = JSON.parse(tree.arguments[0].text)
+  assert(Array.isArray(grapheme) && grapheme.length > 0 && grapheme.length % 3 === 0 && grapheme.every(value => Number.isSafeInteger(value) && value >= 0), 'Invalid Unicode tree integers')
+  return { rtl: regex('CONTAINS_RTL'), emoji: regex('CONTAINS_EMOJI'), imprecise: expressionText(found.get('isEmojiImprecise')!, file), grapheme }
+}
+
+// Execute only hash-verified upstream recipes. This VM controls recipe I/O, not hostile code.
+export function replayUnicodeGenerator(source: string, inputs: Map<string, Uint8Array>, outputs: string[], regexpu: unknown): UnicodeOutput {
+  const logs: string[] = []
+  const writes = new Map<string, string>()
+  const reads = new Set<string>()
+  const io = {
+    readFileSync(name: string) {
+      assert(inputs.has(name), `Unexpected Unicode input: ${name}`)
+      reads.add(name)
+      return inputs.get(name)!
+    },
+    writeFileSync(name: string, value: string) {
+      assert(outputs.includes(name) && !writes.has(name), 'Unexpected or repeated Unicode output')
+      assert.equal(typeof value, 'string', 'Expected Unicode output text')
+      writes.set(name, value)
+    },
+  }
+  vm.runInNewContext(source, {
+    require(name: string) {
+      if (name === 'fs')
+        return io
+      if (name === 'assert')
+        return assert
+      assert.equal(name, 'regexpu', 'Unexpected Unicode import')
+      return regexpu
+    },
+    console: { log: (...args: unknown[]) => logs.push(args.map(String).join(' ')) },
+  }, { timeout: 15000 })
+  assert.deepEqual([...reads].sort(), [...inputs.keys()].sort(), 'Unused Unicode input')
+  assert.deepEqual([...writes.keys()].sort(), [...outputs].sort(), 'Missing Unicode output')
+  return { logs, writes }
+}
+
+export function verifyUnicodeGeneration(tables: UnicodeTables, rtl: UnicodeOutput, emoji: UnicodeOutput, grapheme: UnicodeOutput): void {
+  const pattern = (output: UnicodeOutput) => {
+    const markers = output.logs.flatMap((line, index) => line === '------' ? [index] : [])
+    assert(markers.length === 2 && markers[1] === markers[0]! + 2, 'Missing Unicode regex delimiters')
+    return output.logs[markers[0]! + 1]
+  }
+  assert.equal(pattern(rtl), tables.rtl, 'Generated RTL differs from Monaco')
+  assert.equal(pattern(emoji), tables.emoji, 'Generated emoji differs from Monaco')
+  const expressions = emoji.logs.filter(line => line.startsWith('very imprecise test :: '))
+  assert.equal(expressions.length, 1, 'Missing or repeated imprecise emoji expression')
+  const file = ts.createSourceFile('expression.ts', expressions[0]!.slice('very imprecise test :: '.length), ts.ScriptTarget.Latest, true, ts.ScriptKind.TS)
+  assert.equal(file.statements.length, 1, 'Unexpected imprecise emoji expression')
+  const statement = file.statements[0]!
+  assert(ts.isExpressionStatement(statement), 'Expected imprecise emoji expression')
+  assert.equal(expressionText(statement.expression, file), tables.imprecise, 'Generated imprecise emoji differs from Monaco')
+  assert.deepEqual(JSON.parse(grapheme.logs[grapheme.logs.length - 1]!), tables.grapheme, 'Generated grapheme tree differs from Monaco')
+}