build(site): [SITE-07] reproduce historical Unicode data and ship source notices

test/browser/editor-unicode.spec.js

@@ -0,0 +1,50 @@
+import { expect, test } from './fixtures.js'
+
+test('docs Monaco retains historical RTL, emoji and grapheme data during model edits', async ({ page }, testInfo) => {
+  await page.goto('/test/player.html?core=candidate&chapter=published')
+  await page.addScriptTag({ url: '/assets/js/vs/loader.js' })
+  const observed = await page.evaluate(async () => {
+    window.require.config({ paths: { vs: '/assets/js/vs' } })
+    const load = ids => new Promise((resolve, reject) => window.require(ids, (...modules) => resolve(modules), reject))
+    await load(['vs/editor/editor.main'])
+    const [strings] = await load(['vs/base/common/strings'])
+    const text = 'Hello שלום مرحبا 😀 e\u0301 👩\u200D💻'
+    const host = document.createElement('div')
+    host.style.cssText = 'height:200px;width:600px'
+    document.body.append(host)
+    const editor = window.monaco.editor.create(host, { value: text, language: 'plaintext', minimap: { enabled: false } })
+    try {
+      const model = editor.getModel()
+      const initial = model.getValue()
+      model.pushEditOperations([], [{ range: new window.monaco.Range(1, 1, 1, 1), text: '前缀 ' }], () => null)
+      const edited = model.getValue()
+      model.undo()
+      return {
+        initial,
+        edited,
+        restored: model.getValue(),
+        rtl: ['ascii', 'שלום', 'مرحبا', '😀'].map(value => strings.containsRTL(value)),
+        emoji: ['ascii', '😀', '🌍', '🫠'].map(value => strings.containsEmoji(value)),
+        imprecise: [0x41, 0x1F600, 0x1FAE0].map(value => strings.isEmojiImprecise(value)),
+        grapheme: [0x41, 0x0D, 0x0A, 0x0301, 0x200D, 0x1F1E6, 0x1F469].map(value => strings.getGraphemeBreakType(value)),
+        breaks: [[0x0D, 0x0A], [0x41, 0x0301], [0x41, 0x42], [0x200D, 0x1F469]].map(([a, b]) => strings.breakBetweenGraphemeBreakType(strings.getGraphemeBreakType(a), strings.getGraphemeBreakType(b))),
+      }
+    }
+    finally {
+      const model = editor.getModel()
+      editor.dispose()
+      model.dispose()
+      host.remove()
+    }
+  })
+  expect(observed.initial).toBe('Hello שלום مرحبا 😀 e\u0301 👩\u200D💻')
+  expect(observed.edited).toBe(`前缀 ${observed.initial}`)
+  expect(observed.restored).toBe(observed.initial)
+  expect(observed.rtl).toEqual([false, true, true, false])
+  // Keep the historical approximate predicate; this is not a current Unicode conformance test.
+  expect(observed.emoji).toEqual([false, true, true, false])
+  expect(observed.imprecise).toEqual([false, true, false])
+  expect(observed.grapheme).toEqual([0, 2, 3, 5, 13, 6, 14])
+  expect(observed.breaks).toEqual([false, false, true, false])
+  await testInfo.attach('monaco-unicode-observations', { contentType: 'application/json', body: JSON.stringify(observed) })
+})

test/browser/site-vendor.spec.js

@@ -45,6 +45,7 @@ test('mobile vConsole shows logs and upstream site notice texts are served uncha
   expect(await index.text()).toContain('Included component: @vscode/codicons 0.0.26')
   expect(await index.text()).toContain('Included component: typescript (Monaco worker) 4.4.4')
   expect(await index.text()).toContain('Included component: dompurify (Monaco core) 2.3.1')
+  expect(await index.text()).toContain('Included component: Unicode data (Monaco core) mixed historical inputs: 10.0.0d5, 13.0 drafts, Emoji 13.1')
   expect(await index.text()).toContain('Included component: marked (Monaco core) 3.0.2')
   expect(await index.text()).toContain('Included component: vscode-html-languageservice 4.1.1')
   expect(await index.text()).toContain('Included component: vscode-json-languageservice 4.1.9')

test/monaco-unicode.test.js

@@ -0,0 +1,63 @@
+import assert from 'node:assert/strict'
+import { Buffer } from 'node:buffer'
+import fs from 'node:fs'
+// eslint-disable-next-line test/no-import-node-test -- Exercise frozen Unicode recipes and source boundaries.
+import test from 'node:test'
+import { verifyMonacoUnicodeNotices } from '../scripts/site-vendor/monaco/unicode-origins.ts'
+import { readUnicodeTables, replayUnicodeGenerator, verifyUnicodeGeneration } from '../scripts/site-vendor/monaco/unicode.ts'
+
+const source = 'const CONTAINS_RTL = /a/; const CONTAINS_EMOJI = /b/; export function isEmojiImprecise(x: number) { return (x === 1); } function getGraphemeBreakRawData() { return JSON.parse("[0,0,0]"); }'
+const output = logs => ({ logs, writes: new Map() })
+const rtl = output(['------', 'a', '------'])
+const emoji = output(['very imprecise test :: x === 1', '------', 'b', '------'])
+const tree = output(['[0,0,0]'])
+
+test('Unicode extraction identifies exact top-level declarations and literal data', () => {
+  const tables = readUnicodeTables(source)
+  assert.deepEqual(tables, { rtl: 'a', emoji: 'b', imprecise: 'x === 1', grapheme: [0, 0, 0] })
+  assert.deepEqual(readUnicodeTables(`${source}\nconst text = 'CONTAINS_RTL = /wrong/;';`), tables)
+  assert.deepEqual(readUnicodeTables(source.replace('x === 1', 'x\n /* trivia */ === 1')), tables)
+  assert.throws(() => readUnicodeTables(`${source} const CONTAINS_RTL = /c/;`), /repeated Unicode/)
+  assert.throws(() => readUnicodeTables(source.replace('const CONTAINS_RTL = /a/;', '')), /Missing Unicode/)
+  assert.throws(() => readUnicodeTables(source.replace('/a/', 'getRegex()')), /regex literal/)
+  assert.throws(() => readUnicodeTables(source.replace('/a/', '/a/g')), /regex flags/)
+  assert.throws(() => readUnicodeTables(source.replace('JSON.parse("[0,0,0]")', 'getTree()')), /literal Unicode tree/)
+  assert.throws(() => readUnicodeTables(source.replace('[0,0,0]', '[0,0]')), /tree integers/)
+  assert.throws(() => readUnicodeTables(source.replace('[0,0,0]', '[0,-1,0]')), /tree integers/)
+})
+
+test('Unicode comparison rejects dataset substitutions and incomplete generator output', () => {
+  const tables = readUnicodeTables(source)
+  verifyUnicodeGeneration(tables, rtl, emoji, tree)
+  assert.throws(() => verifyUnicodeGeneration({ ...tables, rtl: 'c' }, rtl, emoji, tree), /RTL differs/)
+  assert.throws(() => verifyUnicodeGeneration({ ...tables, emoji: 'c' }, rtl, emoji, tree), /emoji differs/)
+  assert.throws(() => verifyUnicodeGeneration({ ...tables, imprecise: 'x === 2' }, rtl, emoji, tree), /imprecise emoji differs/)
+  assert.throws(() => verifyUnicodeGeneration(tables, rtl, emoji, output(['[0,1,0]'])), /grapheme tree differs/)
+  assert.throws(() => verifyUnicodeGeneration(tables, output(['a']), emoji, tree), /regex delimiters/)
+  assert.throws(() => verifyUnicodeGeneration(tables, rtl, output(['------', 'b', '------']), tree), /imprecise emoji expression/)
+})
+
+test('Unicode recipes only read declared inputs and capture declared writes without filesystem output', () => {
+  const inputs = new Map([['./data.txt', Buffer.from('frozen')]])
+  const recipe = 'const fs = require("fs"); fs.writeFileSync("out.txt", fs.readFileSync("./data.txt").toString("utf8")); console.log("result");'
+  const actual = replayUnicodeGenerator(recipe, inputs, ['out.txt'], {})
+  assert.deepEqual(actual, { logs: ['result'], writes: new Map([['out.txt', 'frozen']]) })
+  assert.throws(() => replayUnicodeGenerator(recipe.replace('./data.txt', '../outside'), inputs, ['out.txt'], {}), /Unexpected Unicode input/)
+  assert.throws(() => replayUnicodeGenerator(recipe.replace('out.txt', '../outside'), inputs, ['out.txt'], {}), /Unexpected or repeated Unicode output/)
+  assert.throws(() => replayUnicodeGenerator('require("node:fs")', new Map(), [], {}), /Unexpected Unicode import/)
+  assert.throws(() => replayUnicodeGenerator('', inputs, [], {}), /Unused Unicode input/)
+  assert.throws(() => replayUnicodeGenerator('', new Map(), ['out.txt'], {}), /Missing Unicode output/)
+  assert.throws(() => replayUnicodeGenerator('require("fs").writeFileSync("out.txt", "a"); require("fs").writeFileSync("out.txt", "b");', new Map(), ['out.txt'], {}), /repeated Unicode output/)
+})
+
+test('Unicode notices bind complete data terms and attribution to editor and worker', () => {
+  const manifest = JSON.parse(fs.readFileSync('scripts/site-vendor/manifest.json', 'utf8'))
+  verifyMonacoUnicodeNotices(process.cwd(), manifest)
+  const group = manifest.groups.find(item => item.name === 'monaco-editor')
+  const component = group.components.find(item => item.name === 'Unicode data (Monaco core)')
+  component.assets.pop()
+  assert.throws(() => verifyMonacoUnicodeNotices(process.cwd(), manifest), /notice binding/)
+  component.assets.push('docs/assets/js/vs/base/worker/workerMain.js')
+  group.notices = group.notices.filter(item => !item.target.endsWith('core-unicode/LICENSE.txt'))
+  assert.throws(() => verifyMonacoUnicodeNotices(process.cwd(), manifest), /Missing Monaco Unicode notice/)
+})