build(site): [SITE-07] reproduce complete Monaco contribution assembly

scripts/site-vendor/monaco/README.md

@@ -236,4 +236,44 @@ Remaining origin review can start from the map inventory: `dom.ts` references Wi
 marked names a Stack Overflow snippet; `path.ts` identifies Node 14.16.0 while the
 older supplied notice links a different commit. Reference comments alone do not
 establish copied expression or a missing license; inspect actual boundaries and
-original terms before deciding. Core registration assembly and CSS remain separate.
+original terms before deciding. Registration assembly is covered below; CSS remains
+separate.
+
+## Contribution registration assembly
+
+`reproduce-contributions.ts` closes the appended-registration gap left by the
+core minifier check. Six fixed npm archives and the fixed Monaco Git archive supply
+TypeScript 4.4.4, RequireJS 2.3.6, Terser 5.9.0 and its nested source-map 0.7.3.
+The top-level lock also contains source-map 0.6.1; it is not Terser's dependency.
+Nothing is installed in the workspace and no historical install script runs.
+
+The record covers 102 Git members, including 88 TypeScript module inputs and the
+original metadata, release recipe, five bundler recipes/configurations and license.
+Modules are emitted with the original ES5/AMD/strict settings. Standalone fillers
+use program emission: isolated transpilation would incorrectly wrap their existing
+AMD calls in another module. RequireJS itself names and orders definitions and
+removes standalone strict directives. Each run uses a fresh ignored directory so
+missing modules cannot resolve from stale outputs. This proves output equivalence,
+not a full upstream semantic TypeScript check.
+
+`contributions.ts` parses the fixed metadata statically and verifies the five-group
+order: TypeScript (3 modules), CSS (2), JSON (2), HTML (2), basic languages (79).
+After Terser minification and version headers, the release recipe injects the core
+API dependency into each filler, renames the original editor entry, appends groups
+and an API-returning alias, and preserves the exact source-map trailer. Both full
+development and minified editor files must match their npm archive bytes; the
+minified file must also match the unchanged site asset. Core's nested anonymous UMD
+definitions are allowed; the unique renamed entry must be used by an AMD call,
+including the archived string-table lookup (index 719 in both core builds).
+
+```sh
+yarn verify:monaco-contributions --fetch
+yarn verify:monaco-contributions
+yarn test:browser test/browser/editor-modes.spec.js test/browser/editor-basic-languages.spec.js --workers=1
+```
+
+Unit counterexamples cover missing/reordered groups, computed/duplicate metadata,
+foreign modules, missing/repeated/already-injected dependencies and false entry
+strings. Browser regressions exercise real diagnostics/formatting and all pinned
+tokenizer cases. This does not close original core TS compilation, CSS reproduction,
+remaining embedded-origin review, or the broader SITE-07 release requirements.

scripts/site-vendor/monaco/contributions.ts

@@ -0,0 +1,98 @@
+import assert from 'node:assert/strict'
+import ts from 'typescript'
+import { moduleIds } from './languages.ts'
+
+export interface Contribution { name: string, contrib: string, modulePrefix: string }
+
+// Read the fixed metadata without running its browser/CommonJS wrapper.
+export function readContributionMetadata(source: string): Contribution[] {
+  const ast = ts.createSourceFile('metadata.js', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS)
+  const declarations: ts.VariableDeclaration[] = []
+  const visit = (node: ts.Node) => {
+    if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.name.text === 'METADATA')
+      declarations.push(node)
+    ts.forEachChild(node, visit)
+  }
+  visit(ast)
+  assert.equal(declarations.length, 1, 'Expected one METADATA declaration')
+  function property(object: ts.Node | undefined, key: string): ts.Expression {
+    assert(object && ts.isObjectLiteralExpression(object), 'Expected metadata object')
+    const values = object.properties.filter(item => ts.isPropertyAssignment(item) && (ts.isIdentifier(item.name) || ts.isStringLiteral(item.name)) && item.name.text === key)
+    assert.equal(values.length, 1, `Expected one metadata property: ${key}`)
+    return (values[0] as ts.PropertyAssignment).initializer
+  }
+  function literal(object: ts.Node, key: string): string {
+    const value = property(object, key)
+    assert(ts.isStringLiteral(value), 'Expected literal metadata value')
+    return value.text
+  }
+  const plugins = property(declarations[0]!.initializer, 'PLUGINS')
+  assert(ts.isArrayLiteralExpression(plugins), 'Expected plugin array')
+  const result = plugins.elements.map((plugin) => {
+    const name = literal(plugin, 'name')
+    const contrib = literal(plugin, 'contrib')
+    const modulePrefix = literal(plugin, 'modulePrefix')
+    assert.equal(literal(plugin, 'rootPath'), `./${name}`)
+    assert.equal(contrib, `${modulePrefix}/monaco.contribution`)
+    for (const type of ['dev', 'min'])
+      assert.equal(literal(property(plugin, 'paths'), type), `./release/${type}`)
+    return { name, contrib, modulePrefix }
+  })
+  assert.deepEqual(result.map(item => item.name), ['monaco-typescript', 'monaco-css', 'monaco-json', 'monaco-html', 'monaco-languages'], 'Contribution order changed')
+  assert.deepEqual(result.map(item => item.modulePrefix), ['vs/language/typescript', 'vs/language/css', 'vs/language/json', 'vs/language/html', 'vs/basic-languages'], 'Contribution prefix changed')
+  return result
+}
+
+// The upstream release injects this dependency AFTER per-plugin minification.
+export function injectCoreDependency(source: string, prefix: string): string {
+  const pattern = /define\((['"][a-z/\-]+\/fillers\/monaco-editor-core['"]),\[\],/g
+  const matches = [...source.matchAll(pattern)]
+  assert.equal(matches.length, 1, 'Expected one empty core filler dependency')
+  assert.equal(matches[0]![1]!.slice(1, -1), `${prefix}/fillers/monaco-editor-core`, 'Wrong core filler')
+  return source.replace(pattern, 'define($1,[\'vs/editor/editor.api\'],')
+}
+
+export function assembleContributions(core: string, groups: (Contribution & { source: string })[]): string {
+  assert.deepEqual(groups.map(item => item.modulePrefix), ['vs/language/typescript', 'vs/language/css', 'vs/language/json', 'vs/language/html', 'vs/basic-languages'], 'Contribution order changed')
+  const entry = '"vs/editor/editor.main"'
+  assert.equal(core.split(entry).length, 2, 'Expected unique quoted core entry')
+  // Core contains nested UMD calls and addresses AMD names through a string table.
+  // Verify the entry's actual define() use, not merely a matching literal string.
+  let entries = 0
+  const tables = new Map<string, number>()
+  const ast = ts.createSourceFile('core.js', core, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS)
+  const findTables = (node: ts.Node) => {
+    if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer && ts.isArrayLiteralExpression(node.initializer)) {
+      const index = node.initializer.elements.findIndex(item => ts.isStringLiteral(item) && item.text === 'vs/editor/editor.main')
+      if (index >= 0) {
+        assert(node.initializer.elements.every(item => ts.isStringLiteral(item)), 'Expected literal core module table')
+        tables.set(node.name.text, index)
+      }
+    }
+    ts.forEachChild(node, findTables)
+  }
+  findTables(ast)
+  const visit = (node: ts.Node) => {
+    if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === 'define') {
+      const name = node.arguments[0]
+      if (name && ts.isStringLiteral(name) && name.text === 'vs/editor/editor.main')
+        entries++
+      if (name && ts.isElementAccessExpression(name) && ts.isIdentifier(name.expression) && ts.isNumericLiteral(name.argumentExpression)
+        && tables.get(name.expression.text) === Number(name.argumentExpression.text)) { entries++ }
+    }
+    ts.forEachChild(node, visit)
+  }
+  visit(ast)
+  assert.equal(entries, 1, 'Missing core AMD entry')
+  core = core.replace(entry, '"vs/editor/edcore.main"')
+  const extra = groups.map((group) => {
+    assert.equal(group.contrib, `${group.modulePrefix}/monaco.contribution`, 'Wrong contribution entry')
+    const ids = moduleIds(group.source)
+    assert(ids.includes(group.contrib) && ids.every(id => id.startsWith(`${group.modulePrefix}/`)), 'Wrong contribution ownership')
+    return injectCoreDependency(group.source, group.modulePrefix)
+  })
+  extra.push(`define("vs/editor/editor.main", ["vs/editor/edcore.main","${groups.map(group => group.contrib).join('","')}"], function(api) { return api; });`)
+  const map = core.lastIndexOf('//# sourceMappingURL=')
+  const offset = map === -1 ? core.length : map
+  return `${core.slice(0, offset)}\n${extra.join('\n')}\n${core.slice(offset)}`
+}

scripts/site-vendor/monaco/reproduce-contributions.ts

@@ -0,0 +1,134 @@
+import type { Archive, Member } from './archives.ts'
+import type { Contribution } from './contributions.ts'
+import assert from 'node:assert/strict'
+import { Buffer } from 'node:buffer'
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import fs from 'node:fs'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import process from 'node:process'
+import { fileURLToPath } from 'node:url'
+import { ArchiveCache, hash } from './archives.ts'
+import { emitAmd } from './compiler.ts'
+import { assembleContributions, readContributionMetadata } from './contributions.ts'
+import { moduleIds } from './languages.ts'
+
+interface Provenance {
+  commit: string
+  archives: Archive[]
+  compilerMembers: Member[]
+  sourceArchive: { url: string, file: string, bytes: number, sha256: string }
+  members: { path: string, member: string, sha256: string, gitBlobSha: string, frozen?: string }[]
+  modules: { id: string, path: string }[]
+  groups: Contribution[]
+  files: { type: 'dev' | 'min', core: Member, editor: Member }[]
+  target: { path: string, sha256: string }
+}
+
+const root = fileURLToPath(new URL('../../../', import.meta.url))
+assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce-contributions.ts [--fetch]')
+assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
+const record: Provenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/monaco-contributions-provenance.json'), 'utf8'))
+const cache = new ArchiveCache(root, path.join(root, 'refactor/.cache/monaco-review'))
+const online = process.argv.includes('--fetch')
+await cache.verify(record.archives, [], online)
+assert.equal(record.sourceArchive.file, 'monaco-editor-source.tgz')
+assert.equal(record.sourceArchive.url, `https://codeload.github.com/microsoft/monaco-editor/tar.gz/${record.commit}`)
+const archiveFile = path.join(cache.directory, record.sourceArchive.file)
+let archive: Buffer
+if (online) {
+  const response = await fetch(record.sourceArchive.url, { signal: AbortSignal.timeout(60000) })
+  assert(response.ok, `Repository archive HTTP ${response.status}`)
+  archive = Buffer.from(await response.arrayBuffer())
+}
+else {
+  archive = fs.readFileSync(archiveFile)
+}
+assert.equal(archive.length, record.sourceArchive.bytes)
+assert.equal(hash(archive), record.sourceArchive.sha256, 'Repository archive changed')
+if (online)
+  fs.writeFileSync(archiveFile, archive)
+const sources = new Map<string, string>()
+for (const member of record.members) {
+  assert.equal(member.member, `monaco-editor-${record.commit}/${member.path}`)
+  assert(!member.path.split('/').includes('..'), 'Invalid repository member')
+  const bytes = execFileSync('tar', ['-xOzf', archiveFile, member.member], { maxBuffer: 20 * 1024 * 1024 })
+  assert.equal(hash(bytes), member.sha256, `Source changed: ${member.path}`)
+  assert.equal(createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'), member.gitBlobSha, 'Git blob changed')
+  if (member.frozen)
+    assert.deepEqual(fs.readFileSync(path.join(root, member.frozen)), bytes, 'Frozen recipe changed')
+  assert(!sources.has(member.path), 'Duplicate source')
+  sources.set(member.path, bytes.toString('utf8'))
+}
+function read(file: string): string {
+  const source = sources.get(file)
+  assert(source !== undefined, `Unverified source: ${file}`)
+  return source
+}
+const groups = readContributionMetadata(read('monaco-editor/metadata.js'))
+assert.deepEqual(groups, record.groups)
+const lock = JSON.parse(read('package-lock.json')).dependencies
+for (const name of ['typescript', 'terser', 'source-map', 'requirejs']) {
+  const item = record.archives.find(item => item.name === name)
+  const dependency = name === 'source-map' ? lock.terser.dependencies[name] : lock[name]
+  assert(item && item.version === dependency.version && item.integrity === dependency.integrity, `Wrong historical tool: ${name}`)
+  cache.extractCompiler(item)
+}
+for (const member of record.compilerMembers)
+  cache.member(member)
+const require = createRequire(path.join(cache.directory, 'compiler/entry.cjs'))
+const ts = require('typescript') as typeof import('typescript')
+const terser = require('terser') as { minify: (source: string, options: { output: { comments: string } }) => Promise<{ code: string }> }
+
+assert.equal(ts.version, '4.4.4')
+assert.equal(require('terser/package.json').version, '5.9.0')
+assert.equal(require('requirejs/package.json').version, '2.3.6')
+
+assert.equal(record.modules.length, 88)
+assert.equal(new Set(record.modules.map(module => module.id)).size, 88)
+
+// Isolate output from earlier runs, so a missing source cannot resolve from cache.
+const work = fs.mkdtempSync(path.join(cache.directory, 'contributions-'))
+const built: (Contribution & { dev: string, min: string, modules: number })[] = []
+for (const group of groups) {
+  const config = JSON.parse(read(`${group.name}/src/tsconfig.json`)).compilerOptions
+  assert.equal(config.module, 'amd')
+  assert.equal(config.target, 'es5')
+  const modules = record.modules.filter(module => module.id.startsWith(`${group.modulePrefix}/`))
+  for (const module of modules) {
+    const relative = module.id.slice(group.modulePrefix.length + 1)
+    assert(/^[\w./-]+$/.test(relative) && !relative.split('/').includes('..'), 'Invalid AMD input')
+    assert.equal(module.path, `${group.name}/src/${relative.replace(/fillers\/monaco-editor-core$/, 'fillers/monaco-editor-core-amd')}.ts`)
+    const source = read(module.path)
+    // transpileModule wraps non-module scripts in AMD. Compile the five standalone
+    // fillers as programs to preserve their single define([], ...) registration.
+    const emitted = relative === 'fillers/monaco-editor-core'
+      ? emitAmd(ts, source, config.lib, config.strict)
+      : ts.transpileModule(source, { compilerOptions: { target: ts.ScriptTarget.ES5, module: ts.ModuleKind.AMD, newLine: ts.NewLineKind.LineFeed, strict: config.strict } }).outputText
+    const file = path.join(work, `${module.id}.js`)
+    fs.mkdirSync(path.dirname(file), { recursive: true })
+    fs.writeFileSync(file, emitted)
+  }
+  const output = path.join(work, `${group.name}.js`)
+  execFileSync(process.execPath, [require.resolve('requirejs/bin/r.js'), '-o', `baseUrl=${work}`, `name=${group.contrib}`, `out=${output}`, 'optimize=none'], { maxBuffer: 10 * 1024 * 1024 })
+  const dev = fs.readFileSync(output, 'utf8')
+  assert.deepEqual(moduleIds(dev), modules.map(module => module.id), 'Contribution source coverage/order changed')
+  const header = `/*!-----------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * ${group.name} version: 0.30.1(${record.commit})\n * Released under the MIT license\n * https://github.com/Microsoft/${group.name}/blob/master/LICENSE.md\n *-----------------------------------------------------------------------------*/\n`
+  const min = header + (await terser.minify(dev, { output: { comments: 'some' } })).code
+  built.push({ ...group, dev, min, modules: modules.length })
+}
+assert.equal(built.reduce((total, group) => total + group.modules, 0), record.modules.length)
+assert.deepEqual(record.files.map(file => file.type), ['dev', 'min'])
+const outputs = record.files.map((file) => {
+  const assembled = assembleContributions(cache.member(file.core).toString('utf8'), built.map(group => ({ ...group, source: group[file.type] })))
+  const expected = cache.member(file.editor)
+  assert(Buffer.from(assembled).equals(expected), `Complete ${file.type} editor assembly differs`)
+  if (file.type === 'min') {
+    const target = fs.readFileSync(path.join(root, record.target.path))
+    assert.equal(hash(target), record.target.sha256, 'Shipped editor changed')
+    assert.deepEqual(target, expected, 'Shipped editor differs from archive')
+  }
+  return { type: file.type, bytes: expected.length, sha256: hash(expected), exactAssembly: true }
+})
+console.log(JSON.stringify({ archives: record.archives.length, gitSources: sources.size, groups: built.map(group => ({ name: group.name, modules: group.modules })), modules: record.modules.length, outputs, fullUpstreamTypecheck: false, coreTypeScriptAndCssRebuilt: false }))