build(site): [SITE-07] reproduce Monaco CSS HTML and JSON workers

scripts/site-vendor/monaco/README.md

@@ -40,8 +40,49 @@ CopyrightNotice and ThirdPartyNoticeText. The last file includes upstream third-
 terms, not just Apache 2.0. Preserve raw bytes, including encoding and line endings.
 The new attribution explains upstream modifications and the stale version label.
 
-The other language services, localization shim, core embedded libraries and
-language definitions remain under VENDOR-06 review. A match for this worker does
-not close those components. Revalidate actual editor diagnostics/emission and
-notice HTTP delivery after changes; use the committed editor-types and site-vendor
-browser tests. Whole-site, physical-device and remote release gates remain separate.
+## CSS, HTML and JSON workers
+
+`languages.ts` names anonymous AMD modules syntactically, accounts for exact map
+trailers and package aliases, and checks every source fragment and intervening byte.
+It rejects missing modules, renamed dependencies, uncovered helper code, overlaps
+and duplicate definitions. The npm sources are whole UMD files, including their
+helpers and comments; module bodies alone are insufficient evidence.
+
+`archives.ts` verifies archives and fixed Git files and extracts only historical
+compiler inputs into the isolated cache. `reproduce-languages.ts` uses the fixed
+upstream lock and recipes to match 91 npm modules from seven packages, eight root
+aliases, the shared Monaco localization shim and three worker adapters. The complete
+CSS/HTML/JSON development files contain 48/38/33 module instances respectively.
+All bytes except whitespace are assigned to a verified source fragment. Terser
+5.9.0 with the original options and header reproduces all three shipped workers.
+
+The shim is Monaco's own `src/fillers/vscode-nls.ts`, identical in the three packages;
+it is not the installed `vscode-nls` npm implementation. Compile the shim and worker
+adapters with TypeScript 4.4.4 and its ES5/DOM/collection/promise/iterable libraries.
+The compiler program intentionally omits import resolution; exact emission is
+verified, but this is not a full upstream build or semantic typecheck. Standalone
+`transpileModule` changes an async helper's Promise argument and does not reproduce
+these adapters. No service runtime or install script executes during source checks.
+
+```sh
+yarn verify:monaco-language-sources --fetch
+yarn verify:monaco-language-sources
+node --test test/monaco-provenance.test.js
+yarn typecheck:docs-tools
+yarn test:browser test/browser/editor-languages.spec.js --workers=1
+```
+
+The source record is `refactor/baselines/monaco-languages-provenance.json`; its 13
+fixed Git references include the shared upstream lock plus all three bundle recipes,
+compiler configs, shims and adapters. The browser suite creates real workers with
+Monaco's public `createWebWorker`, using the archived mode-manager options, and
+disposes both workers and models. Monaco 0.30.1 has no public CSS/HTML/JSON worker
+getter equivalent to `getTypeScriptWorker`. Tests exercise valid/invalid CSS and
+JSON, HTML tag completion, document symbols, and HTML/JSON formatting on three engines.
+
+This source checkpoint does not close embedded license review. Follow up with the
+seven packages' original licenses/third-party notices, including HTML beautifiers
+and data sources. Core embedded libraries, mode bundles and language definitions
+also remain under VENDOR-06. Revalidate editor-types and site-vendor tests when
+changing declarations or delivered notices. Whole-site, physical-device and remote
+release gates remain separate.

scripts/site-vendor/monaco/archives.ts

@@ -0,0 +1,80 @@
+import assert from 'node:assert/strict'
+import { Buffer } from 'node:buffer'
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import fs from 'node:fs'
+import path from 'node:path'
+
+export interface Archive { name: string, version: string, tarball: string, integrity: string, sha256: string }
+export interface Member { archive: string, member: string, sha256: string }
+export interface Remote { source: string, sha256: string, gitBlobSha: string, apiUrl: string }
+export const hash = (bytes: Uint8Array) => createHash('sha256').update(bytes).digest('hex')
+
+async function download(url: string): Promise<Buffer> {
+  const response = await fetch(url, { signal: AbortSignal.timeout(60000) })
+  assert(response.ok, `${url}: HTTP ${response.status}`)
+  return Buffer.from(await response.arrayBuffer())
+}
+
+// Historical inputs stay in an isolated cache; never install their package scripts.
+export class ArchiveCache {
+  readonly root: string
+  readonly directory: string
+
+  constructor(root: string, directory: string) {
+    this.root = root
+    this.directory = directory
+    fs.mkdirSync(directory, { recursive: true })
+    assert.equal(fs.realpathSync(directory).toLowerCase(), path.resolve(directory).toLowerCase(), 'Redirected Monaco cache')
+  }
+
+  async verify(archives: Archive[], remotes: Remote[], fetch: boolean): Promise<void> {
+    for (const archive of archives) {
+      const file = path.join(this.directory, `${archive.name}-${archive.version}.tgz`)
+      const bytes = fetch ? await download(archive.tarball) : fs.readFileSync(file)
+      assert.equal(hash(bytes), archive.sha256, `Monaco archive changed: ${archive.name}`)
+      assert.equal(`sha512-${createHash('sha512').update(bytes).digest('base64')}`, archive.integrity, 'Monaco archive integrity changed')
+      if (fetch)
+        fs.writeFileSync(file, bytes)
+    }
+    for (const remote of remotes) {
+      const verify = (bytes: Buffer) => {
+        assert.equal(hash(bytes), remote.sha256, 'Monaco fixed Git content changed')
+        assert.equal(createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'), remote.gitBlobSha, 'Monaco Git blob changed')
+      }
+      verify(fs.readFileSync(path.join(this.root, remote.source)))
+      if (fetch) {
+        const data: { encoding: string, content: string } = JSON.parse((await download(remote.apiUrl)).toString('utf8'))
+        assert.equal(data.encoding, 'base64', 'Expected Git content encoding')
+        verify(Buffer.from(data.content, 'base64'))
+      }
+    }
+  }
+
+  read(archive: string, member: string): Buffer {
+    return execFileSync('tar', ['-xOzf', path.join(this.directory, `${archive}.tgz`), member], { maxBuffer: 20 * 1024 * 1024 })
+  }
+
+  member(member: Member): Buffer {
+    const bytes = this.read(member.archive, member.member)
+    assert.equal(hash(bytes), member.sha256, `Monaco source member changed: ${member.member}`)
+    return bytes
+  }
+
+  extractCompiler(archive: Archive): void {
+    const name = `${archive.name}-${archive.version}`
+    const members = execFileSync('tar', ['-tzf', path.join(this.directory, `${name}.tgz`)], { encoding: 'utf8' }).trim().split(/\r?\n/)
+    const directory = path.join(this.directory, 'compiler/node_modules', archive.name)
+    fs.mkdirSync(directory, { recursive: true })
+    assert.equal(fs.realpathSync(directory).toLowerCase(), path.resolve(directory).toLowerCase(), 'Redirected compiler directory')
+    for (const member of members.filter(member => !member.endsWith('/'))) {
+      assert(member.startsWith('package/') && !member.includes('\\') && !member.split('/').includes('..'), 'Unsafe compiler archive member')
+      const destination = path.resolve(directory, member.slice('package/'.length))
+      assert(destination.startsWith(`${directory}${path.sep}`), 'Compiler member escapes cache')
+      fs.mkdirSync(path.dirname(destination), { recursive: true })
+      assert.equal(fs.realpathSync(path.dirname(destination)).toLowerCase(), path.dirname(destination).toLowerCase(), 'Redirected compiler member directory')
+      assert(!fs.existsSync(destination) || !fs.lstatSync(destination).isSymbolicLink(), 'Redirected compiler file')
+      fs.writeFileSync(destination, this.read(name, member))
+    }
+  }
+}

scripts/site-vendor/monaco/languages.ts

@@ -0,0 +1,81 @@
+import assert from 'node:assert/strict'
+import ts from 'typescript'
+
+function definitions(source: string): ts.CallExpression[] {
+  const ast = ts.createSourceFile('worker.js', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS)
+  const result: ts.CallExpression[] = []
+  const walk = (node: ts.Node) => {
+    if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === 'define')
+      result.push(node)
+    ts.forEachChild(node, walk)
+  }
+  walk(ast)
+  return result
+}
+
+export function moduleIds(source: string): string[] {
+  const ids = definitions(source).map((call) => {
+    assert(call.arguments[0] && ts.isStringLiteral(call.arguments[0]), 'Expected literal AMD name')
+    return call.arguments[0].text
+  })
+  assert.equal(new Set(ids).size, ids.length, 'Duplicate AMD module')
+  return ids
+}
+
+// RequireJS names one anonymous AMD definition and removes newline-ended maps.
+// Locate the call syntactically so comments/strings containing define( are inert.
+export function nameModule(source: string, id: string): string {
+  assert(/^[\w/-]+$/.test(id), 'Unexpected AMD identifier')
+  const calls = definitions(source)
+  assert.equal(calls.length, 1, 'Expected one anonymous AMD definition')
+  const call = calls[0]!
+  assert(call.arguments[0] && ts.isArrayLiteralExpression(call.arguments[0]), 'Expected anonymous AMD dependency array')
+  const offset = call.expression.end
+  assert.equal(source[offset], '(', 'Unexpected AMD opening boundary')
+  const map = /^\/\/# sourceMappingURL=[^\r\n]+(?:\r?\n)?$/gm
+  const maps = [...source.matchAll(map)]
+  assert(maps.length <= 1, 'Duplicate source map trailer')
+  source = `${source.slice(0, offset + 1)}'${id}',${source.slice(offset + 1)}`
+  // types/main and uri/index have no final newline. The archived optimizer kept
+  // their map comment and appended a semicolon; account for those bytes too.
+  if (maps[0] && !maps[0][0].endsWith('\n'))
+    return `${source};`
+  return source.replace(map, '').trimEnd()
+}
+
+export function aliasModule(id: string, main: string): string {
+  assert(/^[\w-]+$/.test(id) && /^[\w/-]+$/.test(main), 'Unexpected AMD alias')
+  return `define('${id}', ['${id}/${main}'], function (main) { return main; });`
+}
+
+// Check whole source fragments, their module ownership and every intervening byte.
+// A module inventory alone would overlook helper code outside define() calls.
+export function verifyWorkerSources(worker: string, ids: string[], fragments: { id: string, source: string }[]): void {
+  assert.deepEqual(moduleIds(worker), ids, 'Worker AMD inventory changed')
+  assert.deepEqual(fragments.map(item => item.id).sort(), [...ids].sort(), 'Incomplete source coverage')
+  const intervals = fragments.map(({ id, source }) => {
+    assert.deepEqual(moduleIds(source), [id], 'Wrong source module ownership')
+    const start = worker.indexOf(source)
+    assert(start >= 0, `Source differs: ${id}`)
+    assert.equal(worker.indexOf(source, start + 1), -1, `Repeated source: ${id}`)
+    return { start, end: start + source.length }
+  }).sort((a, b) => a.start - b.start)
+  let end = 0
+  for (const interval of intervals) {
+    assert(interval.start >= end, 'Overlapping source fragments')
+    assert.equal(worker.slice(end, interval.start).trim(), '', 'Unattributed worker content')
+    end = interval.end
+  }
+  assert.equal(worker.slice(end).trim(), '', 'Unattributed worker suffix')
+}
+
+export function languageHeader(language: string): string {
+  assert(['css', 'html', 'json'].includes(language), 'Unexpected Monaco language')
+  return `/*!-----------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * monaco-${language} version: 0.30.1(5a7ba61be909ae9e4889768a3453ebb0dec392e2)
+ * Released under the MIT license
+ * https://github.com/Microsoft/monaco-${language}/blob/master/LICENSE.md
+ *-----------------------------------------------------------------------------*/
+`
+}

scripts/site-vendor/monaco/reproduce-languages.ts

@@ -0,0 +1,118 @@
+import type { Archive, Member, Remote } from './archives.ts'
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import process from 'node:process'
+import { fileURLToPath } from 'node:url'
+import { ArchiveCache, hash } from './archives.ts'
+import { aliasModule, languageHeader, nameModule, verifyWorkerSources } from './languages.ts'
+
+interface Provenance {
+  archives: Archive[]
+  remotes: Remote[]
+  compilerMembers: Member[]
+  modules: (Member & { id: string })[]
+  aliases: { id: string, main: string }[]
+  workers: {
+    language: string
+    development: Member
+    target: { path: string, sha256: string }
+    ids: string[]
+    source: string
+    shim: string
+    config: string
+  }[]
+}
+const root = fileURLToPath(new URL('../../../', import.meta.url))
+assert(process.argv.slice(2).every(arg => arg === '--fetch'), 'Use reproduce-languages.ts [--fetch]')
+assert.equal(process.version, `v${fs.readFileSync(path.join(root, '.node-version'), 'utf8').trim()}`, 'Use canonical Node')
+const record: Provenance = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/monaco-languages-provenance.json'), 'utf8'))
+assert.deepEqual(record.workers.map(worker => worker.language), ['css', 'html', 'json'], 'Missing language worker')
+assert.equal(record.modules.length, 91, 'Missing historical npm module')
+assert.equal(new Set(record.modules.map(member => member.id)).size, 91, 'Duplicate npm module source')
+assert.equal(record.aliases.length, 8, 'Missing historical package alias')
+assert.equal(new Set(record.aliases.map(alias => alias.id)).size, 8, 'Duplicate package alias')
+const lock = JSON.parse(fs.readFileSync(path.join(root, 'refactor/baselines/site-vendor/monaco-typescript/package-lock.json.txt'), 'utf8'))
+for (const archive of record.archives.filter(item => item.name !== 'monaco-editor')) {
+  const entry = archive.name === 'source-map' ? lock.dependencies.terser.dependencies['source-map'] : lock.dependencies[archive.name]
+  assert(entry && entry.version === archive.version && entry.resolved === archive.tarball && entry.integrity === archive.integrity, 'Archive differs from the fixed upstream lock')
+}
+for (const member of record.modules) {
+  const [name, ...parts] = member.id.split('/')
+  const archive = record.archives.find(item => item.name === name)
+  assert(archive && member.archive === `${name}-${archive.version}`, 'Wrong module archive binding')
+  assert.equal(member.member, `package/lib/umd/${parts.join('/')}.js`, 'Wrong UMD source path')
+}
+const cache = new ArchiveCache(root, path.join(root, 'refactor/.cache/monaco-review'))
+await cache.verify(record.archives, record.remotes, process.argv.includes('--fetch'))
+for (const name of ['typescript', 'terser', 'source-map']) {
+  const archive = record.archives.find(item => item.name === name)
+  assert(archive, 'Missing historical compiler dependency')
+  cache.extractCompiler(archive)
+}
+for (const member of record.compilerMembers)
+  cache.member(member)
+const require = createRequire(path.join(cache.directory, 'compiler/entry.cjs'))
+const ts = require('typescript') as typeof import('typescript')
+
+const terser = require('terser') as { minify: (source: string, options: { output: { comments: string } }) => Promise<{ code: string }> }
+
+assert.equal(ts.version, '4.4.4', 'Wrong TypeScript compiler')
+assert.equal(require('terser/package.json').version, '5.9.0', 'Wrong Terser compiler')
+
+assert.equal(require('source-map/package.json').version, '0.7.3', 'Wrong minifier dependency')
+
+function compile(source: string): string {
+  // Emit with the real pinned standard libraries. Isolated transpileModule loses
+  // Promise type resolution and emits a different async helper argument here.
+  // Unresolved service imports are intentionally not a full upstream typecheck.
+  const options: import('typescript').CompilerOptions = {
+    target: ts.ScriptTarget.ES5,
+    module: ts.ModuleKind.AMD,
+    newLine: ts.NewLineKind.LineFeed,
+    noResolve: true,
+    lib: ['lib.es5.d.ts', 'lib.dom.d.ts', 'lib.es2015.collection.d.ts', 'lib.es2015.promise.d.ts', 'lib.es2015.iterable.d.ts'],
+  }
+  const host = ts.createCompilerHost(options)
+  const getSourceFile = host.getSourceFile.bind(host)
+  host.getSourceFile = (file, languageVersion, onError, createNew) => file === 'worker.ts'
+    ? ts.createSourceFile(file, source, languageVersion, true)
+    : getSourceFile(file, languageVersion, onError, createNew)
+  let output = ''
+  host.writeFile = (file, text) => {
+    assert(file.endsWith('.js') && !output, 'Unexpected compiler output')
+    output = text
+  }
+  const program = ts.createProgram(['worker.ts'], options, host)
+  const result = program.emit()
+  assert(!result.emitSkipped && output, 'Missing language source emission')
+  return output
+}
+const modules = new Map(record.modules.map(member => [member.id, nameModule(cache.member(member).toString('utf8'), member.id)]))
+const aliases = new Map(record.aliases.map(alias => [alias.id, aliasModule(alias.id, alias.main)]))
+const results = []
+for (const worker of record.workers) {
+  const config = JSON.parse(fs.readFileSync(path.join(root, worker.config), 'utf8')).compilerOptions
+  assert.equal(config.module, 'amd')
+  assert.equal(config.target, 'es5')
+  assert.deepEqual(config.lib, ['dom', 'es5', 'es2015.collection', 'es2015.promise', 'es2015.iterable'])
+  const ownId = `vs/language/${worker.language}/${worker.language}Worker`
+  const local = new Map([
+    [ownId, nameModule(compile(fs.readFileSync(path.join(root, worker.source), 'utf8')), ownId)],
+    ['vscode-nls/vscode-nls', nameModule(compile(fs.readFileSync(path.join(root, worker.shim), 'utf8')), 'vscode-nls/vscode-nls')],
+  ])
+  const development = cache.member(worker.development).toString('utf8')
+  const fragments = worker.ids.map((id) => {
+    const source = local.get(id) ?? modules.get(id) ?? aliases.get(id)
+    assert(source, `Missing source for ${id}`)
+    return { id, source }
+  })
+  verifyWorkerSources(development, worker.ids, fragments)
+  const target = fs.readFileSync(path.join(root, worker.target.path))
+  assert.equal(hash(target), worker.target.sha256, 'Shipped Monaco language worker changed')
+  const { code } = await terser.minify(development, { output: { comments: 'some' } })
+  assert.equal(languageHeader(worker.language) + code, target.toString('utf8'), 'Minified language worker differs')
+  results.push({ language: worker.language, modules: worker.ids.length, exactSources: true, exactWorker: true, sha256: hash(target) })
+}
+console.log(JSON.stringify({ archives: record.archives.length, gitSources: record.remotes.length, npmModules: modules.size, aliases: aliases.size, workers: results, licenseReviewComplete: false, otherMonacoComponentsReviewed: false }))