build(site): [SITE-07] reproduce basic languages and verify upstream token cases

scripts/site-vendor/monaco/README.md

@@ -109,7 +109,37 @@ the editor's document-format action formats JSON/HTML while preserving content.
 All four mode URLs must return 200; disposed models must lose their diagnostics.
 This complements direct worker methods and the typed player test.
 
-## Language-service notices
+## Basic-language sources and tokenizer regression
+
+`reproduce-basic.ts` verifies all 76 shipped basic-language bundles against 76
+original TypeScript sources, including the shared TypeScript grammar in JavaScript.
+The fixed Monaco commit's repository archive is SHA-256/size checked; every selected
+member additionally has a Git blob ID and SHA-256. Only named members are read.
+Historical compilers stay in the ignored cache; no install scripts or new root
+dependencies are used. HTML/PHP need an explicit RequireJS separator after trailing
+comments. Complete fragment coverage and complete minified bytes are both required.
+
+`basic-fixtures.ts` evaluates the fixed test definitions with a recording runner,
+preserving generated Clojure cases and SCSS preprocessing. Imports are restricted
+to the runner and the verified Clojure grammar; this VM is not a security sandbox.
+The 2,511 cases are frozen for ordinary offline browser CI; full reproduction
+regenerates and compares them with TypeScript 4.4.4. The upstream runner, recipe,
+config, LICENSE and third-party notices are retained alongside the provenance.
+
+```sh
+yarn verify:monaco-basic-sources --fetch
+yarn verify:monaco-basic-sources
+yarn test:browser test/browser/editor-basic-languages.spec.js --workers=1
+```
+
+The browser test uses Monaco's real AMD loader, language registration and tokenizer.
+It retains every upstream case unchanged, and adds 12 explicit cases: INI has no
+upstream test; CSP/ECL have empty suites; pgsql/redshift upstream tests actually
+target SQL. All 76 grammar URLs must load successfully. Token offsets/types and
+multiline state are compared, not just module loading. This is editor tokenization
+evidence, not playback, physical-device evidence or a complete embedded-origin review.
+
+## Language-service notice bindings
 
 `monaco-language-notices.json` binds seven verified package versions to their nine
 complete original LICENSE/third-party notice files. Two exact source-comment slices

scripts/site-vendor/monaco/basic-fixtures.ts

@@ -0,0 +1,40 @@
+import assert from 'node:assert/strict'
+import vm from 'node:vm'
+
+export interface TokenLine { line: string, tokens: { startIndex: number, type: string }[] }
+export interface TokenFixture { source: string, language: string, languages: string[], cases: TokenLine[][] }
+
+// Evaluate fixed upstream test definitions with a recording runner, including their
+// generated cases and SCSS preprocessing. This VM is not a security sandbox.
+export function extractBasicFixtures(ts: typeof import('typescript'), tests: string[], read: (file: string) => string): TokenFixture[] {
+  const fixtures: TokenFixture[] = []
+  const emit = (source: string) => ts.transpileModule(source, {
+    compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES5 },
+  }).outputText
+  let clojure: object | undefined
+  for (const file of tests) {
+    assert(/^monaco-languages\/src\/[^/]+\/[^/]+\.test\.ts$/.test(file), 'Unexpected test member')
+    const language = file.split('/')[2]!
+    vm.runInNewContext(emit(read(file)), {
+      exports: {},
+      require(name: string) {
+        if (name === '../test/testRunner') {
+          return {
+            testTokenization(languages: string | string[], cases: TokenLine[][]) {
+              fixtures.push(JSON.parse(JSON.stringify({ source: file, language, languages: typeof languages === 'string' ? [languages] : languages, cases })))
+            },
+          }
+        }
+        if (language === 'clojure' && name === './clojure') {
+          if (!clojure) {
+            clojure = {}
+            vm.runInNewContext(emit(read('monaco-languages/src/clojure/clojure.ts')), { exports: clojure }, { timeout: 1000, contextCodeGeneration: { strings: false, wasm: false } })
+          }
+          return clojure
+        }
+        throw new Error(`Unexpected test import: ${name}`)
+      },
+    }, { timeout: 1000, contextCodeGeneration: { strings: false, wasm: false } })
+  }
+  return fixtures
+}

scripts/site-vendor/monaco/reproduce-basic.ts

@@ -0,0 +1,112 @@
+import type { Archive, Member } from './archives.ts'
+import assert from 'node:assert/strict'
+import { Buffer } from 'node:buffer'
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import fs from 'node:fs'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import process from 'node:process'
+import { fileURLToPath } from 'node:url'
+import { ArchiveCache, hash } from './archives.ts'
+import { extractBasicFixtures } from './basic-fixtures.ts'
+import { emitAmd } from './compiler.ts'
+import { nameModule, verifyWorkerSources } from './languages.ts'
+
+interface Provenance {
+  commit: string
+  archives: Archive[]
+  compilerMembers: Member[]
+  sourceArchive: { url: string, file: string, bytes: number, sha256: string }
+  members: { path: string, member: string, sha256: string, gitBlobSha: string, frozen?: string }[]
+  modules: { id: string, path: string, suffix: string }[]
+  files: { language: string, development: Member, target: { path: string, sha256: string }, ids: string[] }[]
+  fixtures: { path: string, sha256: string, tests: string[], cases: number, suites: number }
+}
+
+const root = fileURLToPath(new URL('../../../', import.meta.url))
+assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce-basic.ts [--fetch]')
+assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
+const record: Provenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/monaco-basic-provenance.json'), 'utf8'))
+const cache = new ArchiveCache(root, path.join(root, 'refactor/.cache/monaco-review'))
+await cache.verify(record.archives, [], process.argv.includes('--fetch'))
+assert.equal(record.sourceArchive.file, 'monaco-editor-source.tgz')
+assert.equal(record.sourceArchive.url, `https://codeload.github.com/microsoft/monaco-editor/tar.gz/${record.commit}`)
+const archiveFile = path.join(cache.directory, record.sourceArchive.file)
+let archive: Buffer
+if (process.argv.includes('--fetch')) {
+  const response = await fetch(record.sourceArchive.url, { signal: AbortSignal.timeout(60000) })
+  assert(response.ok, `Repository archive HTTP ${response.status}`)
+  archive = Buffer.from(await response.arrayBuffer())
+}
+else {
+  archive = fs.readFileSync(archiveFile)
+}
+assert.equal(archive.length, record.sourceArchive.bytes)
+assert.equal(hash(archive), record.sourceArchive.sha256, 'Repository archive changed')
+if (process.argv.includes('--fetch'))
+  fs.writeFileSync(archiveFile, archive)
+const sources = new Map<string, string>()
+for (const member of record.members) {
+  assert.equal(member.member, `monaco-editor-${record.commit}/${member.path}`)
+  assert(!member.path.split('/').includes('..'), 'Invalid repository member')
+  const bytes = execFileSync('tar', ['-xOzf', archiveFile, member.member], { maxBuffer: 20 * 1024 * 1024 })
+  assert.equal(hash(bytes), member.sha256, `Source changed: ${member.path}`)
+  assert.equal(createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'), member.gitBlobSha, 'Git blob changed')
+  if (member.frozen)
+    assert.deepEqual(fs.readFileSync(path.join(root, member.frozen)), bytes, 'Frozen upstream recipe/notice changed')
+  sources.set(member.path, bytes.toString('utf8'))
+}
+function read(file: string) {
+  const source = sources.get(file)
+  assert(source !== undefined, `Unverified source: ${file}`)
+  return source
+}
+for (const name of ['typescript', 'terser', 'source-map']) {
+  const archive = record.archives.find(item => item.name === name)
+  assert(archive, 'Missing compiler dependency')
+  cache.extractCompiler(archive)
+}
+for (const member of record.compilerMembers)
+  cache.member(member)
+const require = createRequire(path.join(cache.directory, 'compiler/entry.cjs'))
+const ts = require('typescript') as typeof import('typescript')
+const terser = require('terser') as { minify: (source: string, options: { output: { comments: string } }) => Promise<{ code: string }> }
+
+assert.equal(ts.version, '4.4.4')
+assert.equal(require('terser/package.json').version, '5.9.0')
+assert.equal(require('source-map/package.json').version, '0.7.3')
+
+const config = JSON.parse(read('monaco-languages/src/tsconfig.json')).compilerOptions
+assert.equal(config.module, 'amd')
+assert.equal(config.target, 'es5')
+const modules = new Map(record.modules.map((module) => {
+  // RequireJS adds this separator after HTML/PHP's trailing source comments.
+  const suffix = ['vs/basic-languages/html/html', 'vs/basic-languages/php/php'].includes(module.id) ? '\n;' : ''
+  assert.equal(module.suffix, suffix, 'Unexpected optimizer adaptation')
+  return [module.id, nameModule(emitAmd(ts, read(module.path), config.lib, config.strict), module.id) + suffix]
+}))
+assert.equal(modules.size, 76)
+assert.deepEqual(record.files.map(file => file.language), fs.readdirSync(path.join(root, 'docs/assets/js/vs/basic-languages')).sort(), 'Missing basic-language bundle')
+const results = []
+for (const file of record.files) {
+  const development = cache.member(file.development).toString('utf8')
+  verifyWorkerSources(development, file.ids, file.ids.map((id) => {
+    const source = modules.get(id)
+    assert(source, `Missing grammar: ${id}`)
+    return { id, source }
+  }))
+  const target = fs.readFileSync(path.join(root, file.target.path))
+  assert.equal(hash(target), file.target.sha256, 'Shipped grammar changed')
+  const header = `/*!-----------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * monaco-languages version: 0.30.1(${record.commit})\n * Released under the MIT license\n * https://github.com/Microsoft/monaco-languages/blob/master/LICENSE.md\n *-----------------------------------------------------------------------------*/\n`
+  const { code } = await terser.minify(development, { output: { comments: 'some' } })
+  assert.equal(header + code, target.toString('utf8'), 'Minified grammar differs')
+  results.push({ language: file.language, modules: file.ids.length, exactSources: true, exactOutput: true, sha256: hash(target) })
+}
+const fixtureBytes = fs.readFileSync(path.join(root, record.fixtures.path))
+assert.equal(hash(fixtureBytes), record.fixtures.sha256, 'Frozen token cases changed')
+const fixtures = extractBasicFixtures(ts, record.fixtures.tests, read)
+assert.deepEqual(fixtures, JSON.parse(fixtureBytes.toString('utf8')).fixtures, 'Upstream test semantics changed')
+assert.equal(fixtures.length, record.fixtures.suites)
+assert.equal(fixtures.reduce((total, fixture) => total + fixture.cases.length, 0), record.fixtures.cases)
+console.log(JSON.stringify({ files: results, sources: sources.size, upstreamSuites: fixtures.length, upstreamCases: record.fixtures.cases, coreAndEmbeddedOriginsReviewed: false }))