build(site): [SITE-07] reproduce and validate Monaco language modes

scripts/site-vendor/manifest.json

@@ -1037,7 +1037,8 @@
           "version": "3.0.0",
           "tarball": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz",
           "assets": [
-            "docs/assets/js/vs/language/json/jsonWorker.js"
+            "docs/assets/js/vs/language/json/jsonWorker.js",
+            "docs/assets/js/vs/language/json/jsonMode.js"
           ],
           "notices": [
             "docs/licenses/monaco-editor/language-services/jsonc-parser/LICENSE.md",
@@ -1103,7 +1104,10 @@
           "assets": [
             "docs/assets/js/vs/language/css/cssWorker.js",
             "docs/assets/js/vs/language/html/htmlWorker.js",
-            "docs/assets/js/vs/language/json/jsonWorker.js"
+            "docs/assets/js/vs/language/json/jsonWorker.js",
+            "docs/assets/js/vs/language/css/cssMode.js",
+            "docs/assets/js/vs/language/html/htmlMode.js",
+            "docs/assets/js/vs/language/json/jsonMode.js"
           ],
           "notices": [
             "docs/licenses/monaco-editor/language-services/vscode-languageserver-types/License.txt",

scripts/site-vendor/monaco/README.md

@@ -80,6 +80,35 @@ disposes both workers and models. Monaco 0.30.1 has no public CSS/HTML/JSON work
 getter equivalent to `getTypeScriptWorker`. Tests exercise valid/invalid CSS and
 JSON, HTML tag completion, document symbols, and HTML/JSON formatting on three engines.
 
+## Mode registration bundles
+
+`reproduce-modes.ts` covers cssMode, htmlMode, jsonMode and tsMode. The fixed record
+has six archives, 23 Git files, 14 Monaco sources, six npm source modules and two
+package aliases. All 26 module instances and intervening helper bytes match the
+development bundles; their four complete minified outputs match existing assets.
+`jsonc-parser` and `vscode-languageserver-types` notice bindings include these mode
+assets as well as workers. No runtime or notice text is rewritten.
+
+`compiler.ts` shares pinned TypeScript emission with the worker reproducer.
+JSON tokenization needs the original jsonc-parser 3.0.0 main.d.ts to inline const
+enums. The compiler host resolves only explicitly supplied declarations, avoiding
+accidental resolution against current root dependencies. Module names may contain
+a dot (`lib.index`); traversal segments remain rejected. TypeScript mode compilation
+uses the fixed strict config. Exact emission is not a complete upstream typecheck.
+
+```sh
+yarn verify:monaco-mode-sources --fetch
+yarn verify:monaco-mode-sources
+yarn verify:monaco-language-sources
+yarn test:browser test/browser/editor-modes.spec.js --workers=1
+```
+
+`editor-modes.spec.js` uses a real editor, automatic mode loading/provider registration
+and model switching. CSS/JSON/TypeScript errors appear and clear after correction;
+the editor's document-format action formats JSON/HTML while preserving content.
+All four mode URLs must return 200; disposed models must lose their diagnostics.
+This complements direct worker methods and the typed player test.
+
 ## Language-service notices
 
 `monaco-language-notices.json` binds seven verified package versions to their nine
@@ -99,7 +128,7 @@ ranges. Preserve raw bytes and the `-text` Git attributes for frozen/output noti
 12 relative links, alongside mobile player playback and console cleanup.
 
 This does not close all Monaco embedded-source review. Core embedded libraries,
-mode bundles, language definitions and remaining upstream data/origin details stay
+language definitions and remaining upstream data/origin details stay
 under VENDOR-06. Revalidate editor-types and site-vendor tests when changing
 declarations or delivered notices. Whole-site, physical-device and remote release
 gates remain separate.

scripts/site-vendor/monaco/compiler.ts

@@ -0,0 +1,34 @@
+import assert from 'node:assert/strict'
+
+export interface CompilerModule { name: string, source: string }
+
+// Reproduce emission with pinned standard libraries and optional verified module
+// declarations. This is intentionally not a full upstream semantic typecheck.
+export function emitAmd(ts: typeof import('typescript'), source: string, libraries: string[], strict = false, modules: CompilerModule[] = []): string {
+  const options: import('typescript').CompilerOptions = {
+    target: ts.ScriptTarget.ES5,
+    module: ts.ModuleKind.AMD,
+    newLine: ts.NewLineKind.LineFeed,
+    noResolve: modules.length === 0,
+    lib: libraries.map(library => `lib.${library}.d.ts`),
+    strict,
+  }
+  const host = ts.createCompilerHost(options)
+  const getSourceFile = host.getSourceFile.bind(host)
+  const inputs = new Map([['worker.ts', source], ...modules.map((module, index): [string, string] => [`pinned-${index}.d.ts`, module.source])])
+  host.getSourceFile = (file, languageVersion, onError, createNew) => inputs.has(file)
+    ? ts.createSourceFile(file, inputs.get(file)!, languageVersion, true)
+    : getSourceFile(file, languageVersion, onError, createNew)
+  host.resolveModuleNames = names => names.map((name) => {
+    const index = modules.findIndex(module => module.name === name)
+    return index < 0 ? undefined : { resolvedFileName: `pinned-${index}.d.ts`, extension: ts.Extension.Dts }
+  })
+  let output = ''
+  host.writeFile = (file, text) => {
+    assert(file.endsWith('.js') && !output, 'Unexpected compiler output')
+    output = text
+  }
+  const result = ts.createProgram(['worker.ts'], options, host).emit()
+  assert(!result.emitSkipped && output, 'Missing source emission')
+  return output
+}

scripts/site-vendor/monaco/languages.ts

@@ -25,7 +25,7 @@ export function moduleIds(source: string): string[] {
 // RequireJS names one anonymous AMD definition and removes newline-ended maps.
 // Locate the call syntactically so comments/strings containing define( are inert.
 export function nameModule(source: string, id: string): string {
-  assert(/^[\w/-]+$/.test(id), 'Unexpected AMD identifier')
+  assert(/^[\w./-]+$/.test(id) && id.split('/').every(part => part !== '.' && part !== '..'), 'Unexpected AMD identifier')
   const calls = definitions(source)
   assert.equal(calls.length, 1, 'Expected one anonymous AMD definition')
   const call = calls[0]!
@@ -70,7 +70,7 @@ export function verifyWorkerSources(worker: string, ids: string[], fragments: {
 }
 
 export function languageHeader(language: string): string {
-  assert(['css', 'html', 'json'].includes(language), 'Unexpected Monaco language')
+  assert(['css', 'html', 'json', 'typescript'].includes(language), 'Unexpected Monaco language')
   return `/*!-----------------------------------------------------------------------------
  * Copyright (c) Microsoft Corporation. All rights reserved.
  * monaco-${language} version: 0.30.1(5a7ba61be909ae9e4889768a3453ebb0dec392e2)

scripts/site-vendor/monaco/notices.ts

@@ -29,6 +29,7 @@ function readNotices(root: string): NoticeRecord {
 export function verifyMonacoLanguageNotices(root: string, manifest: VendorManifest): number {
   const record = readNotices(root)
   const sources: Sources = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/monaco-languages-provenance.json'), 'utf8'))
+  const modes: { files: Sources['workers'] } = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/monaco-modes-provenance.json'), 'utf8'))
   assert.equal(record.notices.length, 12, 'Missing Monaco language notice source')
   assert.equal(record.components.length, 9, 'Missing Monaco language component source')
   assert.deepEqual(record.packages.map(item => item.name).sort(), [...new Set(sources.modules.map(item => item.id.split('/')[0]))].sort(), 'Incomplete Monaco service attribution')
@@ -38,7 +39,7 @@ export function verifyMonacoLanguageNotices(root: string, manifest: VendorManife
     const archive = sources.archives.find(item => `${item.name}-${item.version}` === pkg.archive)
     assert(archive && archive.name === pkg.name && archive.version === pkg.version && archive.tarball === pkg.tarball, 'Wrong Monaco service source identity')
     const modules = sources.modules.filter(item => item.archive === pkg.archive).map(item => item.id)
-    const assets = sources.workers.filter(worker => worker.ids.some(id => modules.includes(id))).map(worker => worker.target.path)
+    const assets = [...sources.workers, ...modes.files].filter(worker => worker.ids.some(id => modules.includes(id))).map(worker => worker.target.path)
     assert.deepEqual(pkg.assets, assets, 'Wrong Monaco service asset binding')
   }
   for (const expected of record.components) {

scripts/site-vendor/monaco/reproduce-languages.ts

@@ -6,6 +6,7 @@ import path from 'node:path'
 import process from 'node:process'
 import { fileURLToPath } from 'node:url'
 import { ArchiveCache, hash } from './archives.ts'
+import { emitAmd } from './compiler.ts'
 import { aliasModule, languageHeader, nameModule, verifyWorkerSources } from './languages.ts'
 import { verifyMonacoLanguageNoticeArchives } from './notices.ts'
 
@@ -65,32 +66,7 @@ assert.equal(require('terser/package.json').version, '5.9.0', 'Wrong Terser comp
 
 assert.equal(require('source-map/package.json').version, '0.7.3', 'Wrong minifier dependency')
 
-function compile(source: string): string {
-  // Emit with the real pinned standard libraries. Isolated transpileModule loses
-  // Promise type resolution and emits a different async helper argument here.
-  // Unresolved service imports are intentionally not a full upstream typecheck.
-  const options: import('typescript').CompilerOptions = {
-    target: ts.ScriptTarget.ES5,
-    module: ts.ModuleKind.AMD,
-    newLine: ts.NewLineKind.LineFeed,
-    noResolve: true,
-    lib: ['lib.es5.d.ts', 'lib.dom.d.ts', 'lib.es2015.collection.d.ts', 'lib.es2015.promise.d.ts', 'lib.es2015.iterable.d.ts'],
-  }
-  const host = ts.createCompilerHost(options)
-  const getSourceFile = host.getSourceFile.bind(host)
-  host.getSourceFile = (file, languageVersion, onError, createNew) => file === 'worker.ts'
-    ? ts.createSourceFile(file, source, languageVersion, true)
-    : getSourceFile(file, languageVersion, onError, createNew)
-  let output = ''
-  host.writeFile = (file, text) => {
-    assert(file.endsWith('.js') && !output, 'Unexpected compiler output')
-    output = text
-  }
-  const program = ts.createProgram(['worker.ts'], options, host)
-  const result = program.emit()
-  assert(!result.emitSkipped && output, 'Missing language source emission')
-  return output
-}
+const compile = (source: string) => emitAmd(ts, source, ['es5', 'dom', 'es2015.collection', 'es2015.promise', 'es2015.iterable'])
 const modules = new Map(record.modules.map(member => [member.id, nameModule(cache.member(member).toString('utf8'), member.id)]))
 const aliases = new Map(record.aliases.map(alias => [alias.id, aliasModule(alias.id, alias.main)]))
 const results = []

scripts/site-vendor/monaco/reproduce-modes.ts

@@ -0,0 +1,81 @@
+import type { Archive, Member, Remote } from './archives.ts'
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import process from 'node:process'
+import { fileURLToPath } from 'node:url'
+import { ArchiveCache, hash } from './archives.ts'
+import { emitAmd } from './compiler.ts'
+import { aliasModule, languageHeader, nameModule, verifyWorkerSources } from './languages.ts'
+
+interface Provenance {
+  archives: Archive[]
+  remotes: Remote[]
+  compilerMembers: Member[]
+  declaration: Member
+  npmModules: (Member & { id: string })[]
+  aliases: { id: string, main: string }[]
+  files: {
+    language: string
+    development: Member
+    target: { path: string, sha256: string }
+    ids: string[]
+    modules: { id: string, source: string, resolveJsonc: boolean }[]
+    config: string
+  }[]
+}
+const root = fileURLToPath(new URL('../../../', import.meta.url))
+assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce-modes.ts [--fetch]')
+assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
+const record: Provenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/monaco-modes-provenance.json'), 'utf8'))
+assert.deepEqual(record.files.map(file => file.language), ['css', 'html', 'json', 'typescript'], 'Missing mode bundle')
+assert.equal(record.files.flatMap(file => file.modules).length, 14, 'Missing Monaco mode source')
+assert.equal(record.npmModules.length, 6, 'Missing mode dependency source')
+const cache = new ArchiveCache(root, path.join(root, 'refactor/.cache/monaco-review'))
+await cache.verify(record.archives, record.remotes, process.argv.includes('--fetch'))
+for (const name of ['typescript', 'terser', 'source-map']) {
+  const archive = record.archives.find(item => item.name === name)
+  assert(archive, 'Missing historical compiler dependency')
+  cache.extractCompiler(archive)
+}
+for (const member of record.compilerMembers)
+  cache.member(member)
+const require = createRequire(path.join(cache.directory, 'compiler/entry.cjs'))
+const ts = require('typescript') as typeof import('typescript')
+
+const terser = require('terser') as { minify: (source: string, options: { output: { comments: string } }) => Promise<{ code: string }> }
+
+assert.equal(ts.version, '4.4.4')
+assert.equal(require('terser/package.json').version, '5.9.0')
+
+assert.equal(require('source-map/package.json').version, '0.7.3')
+
+const jsonc = cache.member(record.declaration).toString('utf8')
+const modules = new Map(record.npmModules.map(member => [member.id, nameModule(cache.member(member).toString('utf8'), member.id)]))
+const aliases = new Map(record.aliases.map(alias => [alias.id, aliasModule(alias.id, alias.main)]))
+const results = []
+for (const file of record.files) {
+  assert(record.remotes.some(remote => remote.source === file.config), 'Unverified mode compiler config')
+  const config: { compilerOptions: { module: string, target: string, lib: string[], strict?: boolean } } = JSON.parse(fs.readFileSync(path.join(root, file.config), 'utf8'))
+  assert.equal(config.compilerOptions.module, 'amd')
+  assert.equal(config.compilerOptions.target, 'es5')
+  const local = new Map(file.modules.map((module) => {
+    assert(record.remotes.some(remote => remote.source === module.source), 'Unverified mode source')
+    const source = fs.readFileSync(path.join(root, module.source), 'utf8')
+    const compiled = emitAmd(ts, source, config.compilerOptions.lib, config.compilerOptions.strict, module.resolveJsonc ? [{ name: 'jsonc-parser', source: jsonc }] : [])
+    return [module.id, nameModule(compiled, module.id)]
+  }))
+  const development = cache.member(file.development).toString('utf8')
+  verifyWorkerSources(development, file.ids, file.ids.map((id) => {
+    const source = local.get(id) ?? modules.get(id) ?? aliases.get(id)
+    assert(source, `Missing mode source: ${id}`)
+    return { id, source }
+  }))
+  const target = fs.readFileSync(path.join(root, file.target.path))
+  assert.equal(hash(target), file.target.sha256, 'Shipped Monaco mode changed')
+  const { code } = await terser.minify(development, { output: { comments: 'some' } })
+  assert.equal(languageHeader(file.language) + code, target.toString('utf8'), 'Minified mode bundle differs')
+  results.push({ language: file.language, modules: file.ids.length, exactSources: true, exactOutput: true, sha256: hash(target) })
+}
+console.log(JSON.stringify({ archives: record.archives.length, gitSources: record.remotes.length, ownModules: 14, npmModules: modules.size, aliases: aliases.size, modes: results, coreAndBasicLanguagesReviewed: false }))