refactor(docs): [SITE-AI-DOCS-01] make generation offline and translation draft based

This commit is contained in:
Harvey Zhao committed 2026-09-14 08:22:44 +08:00
1 parent 647f3e7f12
commit 078cd9ef72
27 files changed
+8642 -4263

No files matched your search

+89
View File
@@ -0,0 +1,89 @@
# Documentation generation and translation
Run from the repository root with Node from `.node-version` and Yarn Classic
1.22.22. The old `build-llm.js` and `trans-docs.js` paths remain checked JS CLI
adapters. These tools do not change player APIs or published declarations.
## Commands and outputs
```sh
yarn build:llm
yarn check:llm
yarn trans:docs
yarn trans:docs --remote
yarn trans:docs --validate refactor/.cache/translations/draft-EXAMPLE
yarn trans:docs --apply refactor/.cache/translations/draft-EXAMPLE
yarn typecheck:docs-tools
node --test test/documentation-pipeline.test.js
```
`build:llm` is now offline and deterministic. It preserves complete source text
(LF-normalized) from English Markdown, the actual editor declaration list,
examples and the VAST declaration notices. It writes the existing `docs/llms.txt`
path plus `docs/llms.manifest.json` with source/output SHA-256 fingerprints.
`check:llm` checks without writing; CI checks it and `ci:build` regenerates it
after editor declarations. Regenerate after changing any input. Content presence
does not prove API correctness, example playback or English coverage; SITE-04
and EX-03 still own those reviews. Unloaded legacy WebSR declarations are not
included simply because they remain on disk.
`trans:docs` now only reports a local plan by default. Only `--remote` imports
dotenv, reads `DEEPSEEK_API_KEY` and calls the existing DeepSeek endpoint/model.
It writes a unique ignored draft directory, never the live English documents.
Its scope remains index, advanced, component and start; adding plugin translation
is separate work. Remote requests may incur provider charges. CI never calls them.
Review the draft's prose and diff before applying it. If prose needs editing,
edit the draft then run `--validate`: it verifies structure and source/target
fingerprints and records the edited draft hashes. It cannot judge translation
quality. `--apply` checks all fingerprints and structure again before replacing
the selected English files. Unselected pages remain. Rebuild site assets, the
LLM bundle and the documentation site after applying reviewed translations.
This intentionally replaces the old destructive default, which removed the
English directory before the first request. Existing automation that intended
remote translation must explicitly select `--remote`, review, then `--apply`.
No actual remote translation or paid service acceptance was performed for this
migration; local tests inject responses or use a loopback HTTP server.
## Ownership and failure behavior
| Module | Responsibility |
| ---------------- | -------------------------------------------------------------------------------- |
| `cli.ts` | Argument dispatch; offline default; remote credential boundary |
| `corpus.ts` | Ordered source inventory and deterministic output/manifest |
| `markdown.ts` | Protected block placeholders, structural validation, bounded chunks |
| `remote.ts` | Request/body timeout, bounded retry, response validation and cancellation |
| `translation.ts` | Draft state, worker cancellation/join, fingerprint validation and apply/rollback |
| `files.ts` | LF reads, hashes, owned paths and adjacent temporary-file replacement |
Draft state advances incomplete -> complete -> applied; a failed request or
invalid response leaves a failed draft for diagnosis. One worker failure aborts
the shared signal and waits for every worker before returning. HTTP 429/5xx and
transport failures have bounded retries; invalid/empty JSON and authorization
fail without retrying. The per-attempt timeout covers body reading too. Every
timer is cleared and rejected HTTP bodies are cancelled.
Code, inline code, HTML, links and structural Markdown tokens must survive.
Fenced blocks and directives are restored from local originals, not model text.
An invalid result is rejected instead of heuristically inserting code fences.
Unusual Markdown or a chunk boundary may be rejected; inspect the retained
draft and improve protection rather than weakening validation. This is not a
Markdown sanitizer or proof that prose preserves meaning.
Apply preflights the whole set, rechecks each write, uses adjacent temporary
files and rolls back completed writes on caught failures, preserving original
bytes. A concurrent edit prevents rollback of that file, producing an aggregate
error instead of overwriting the other writer. **This is not a multi-file crash
transaction**: backups live in process memory, and power loss or forced process
termination can leave a partial apply. Commit existing English changes before
applying, keep the draft, and inspect Git diff after any failure. Path validation
rejects traversal and existing junctions/symlinks outside the owned root; it is
not a defense against hostile concurrent filesystem mutation.
No new dependencies were added: use the root's pinned TypeScript 5.9.3,
markdown-it types 14.1.2, glob 13.0.6 and dotenv 17.2.4. For follow-up maintenance,
start from the responsible module, rerun its filesystem/request regressions,
strict docs-tools types and root lint, then regenerate/check the source bundle
and site inventory. Browser/player testing is needed when a change also affects
page content or runtime, not as a substitute for these tool failure tests.
+75
View File
@@ -0,0 +1,75 @@
import assert from 'node:assert/strict'
import process from 'node:process'
import { buildCorpus } from './corpus.ts'
import { atomicWrite, ownedPath, read } from './files.ts'
import { translateChunk } from './remote.ts'
import {
applyTranslationDraft,
createTranslationDraft,
translationPlan,
validateTranslationDraft,
} from './translation.ts'
export function runCorpus(args: string[], root = process.cwd()): void {
assert(
args.every(arg => arg === '--check'),
'Use yarn build:llm [--check]; generation is offline',
)
const outputs = buildCorpus(root)
for (const [relative, text] of outputs) {
const file = ownedPath(root, relative)
if (args.includes('--check'))
assert.equal(read(file), text, `LLM source bundle drift: ${relative}`)
else atomicWrite(file, text)
}
console.log(
`Offline LLM source bundle ${args.includes('--check') ? 'checked' : 'generated'}: ${outputs.size} outputs`,
)
}
export async function runTranslation(
args: string[],
root = process.cwd(),
): Promise<void> {
if (!args.length || (args.length === 1 && args[0] === '--plan')) {
console.log(
JSON.stringify(
{
mode: 'plan-only',
files: translationPlan(root),
next: 'Use --remote to create a reviewable draft; --apply <draft-directory> applies validated files. No network or English writes occurred.',
},
null,
2,
),
)
return
}
if (args.length === 2 && args[0] === '--validate' && args[1]) {
validateTranslationDraft(root, args[1])
console.log(
'Draft structure and source/target fingerprints validated; English files unchanged',
)
return
}
if (args.length === 2 && args[0] === '--apply' && args[1]) {
applyTranslationDraft(root, args[1])
console.log(
'Validated translation draft applied; rebuild site assets and LLM bundle before checking generated output',
)
return
}
assert(
args.length === 1 && args[0] === '--remote',
'Use yarn trans:docs [--plan | --remote | --validate <draft-directory> | --apply <draft-directory>]',
)
const dotenv = await import('dotenv')
dotenv.config({ path: ownedPath(root, '.env'), quiet: true })
const key = process.env.DEEPSEEK_API_KEY
assert(key, 'Missing DEEPSEEK_API_KEY; no translation requests were made')
const directory = await createTranslationDraft(root, (content, signal) =>
translateChunk(content, { key, signal }))
console.log(
`Translation draft ready: ${directory}. Review files before using --apply.`,
)
}
+88
View File
@@ -0,0 +1,88 @@
import assert from 'node:assert/strict'
import { globSync } from 'glob'
import ts from 'typescript'
import { ownedPath, read, sha256 } from './files.ts'
export function buildCorpus(root: string): Map<string, string> {
const common = ts.createSourceFile(
'common.js',
read(ownedPath(root, 'docs/assets/js/common.js')),
ts.ScriptTarget.Latest,
true,
ts.ScriptKind.JS,
)
const declarations: string[] = []
let lists = 0
function visit(node: ts.Node): void {
if (
ts.isVariableDeclaration(node)
&& ts.isIdentifier(node.name)
&& node.name.text === 'libUris'
) {
lists++
assert(
node.initializer && ts.isArrayLiteralExpression(node.initializer),
'Expected literal editor library list',
)
for (const element of node.initializer.elements) {
assert(
ts.isStringLiteral(element)
&& /^\.\/assets\/ts\/[\w.-]+\.d\.ts$/.test(element.text),
'Unexpected editor library path',
)
declarations.push(`docs/${element.text.slice(2)}`)
}
}
ts.forEachChild(node, visit)
}
visit(common)
assert(
lists === 1
&& declarations.length
&& new Set(declarations).size === declarations.length,
'Missing or duplicate editor library list',
)
const sections = [
{
title: 'Documentation Summary',
files: globSync('packages/artplayer-vitepress/docs/en/**/*.md', {
cwd: root,
posix: true,
}).sort(),
},
{ title: 'Type Definitions Overview', files: declarations },
{
title: 'Examples Summary',
files: globSync('docs/assets/example/*.js', {
cwd: root,
posix: true,
}).sort(),
},
{
title: 'Third-party Type Notices',
files: ['docs/assets/ts/artplayer-plugin-vast.LICENSE.txt'],
},
]
const sources: { file: string, sha256Lf: string }[] = []
let text
= 'ArtPlayer documentation source bundle\nGenerated offline by yarn build:llm. Source text is preserved after LF normalization.\nThese are source references, not proof that every example or documented feature has passed release review.\n'
for (const section of sections) {
assert(
section.files.length,
`Empty documentation section: ${section.title}`,
)
text += `\n===== ${section.title} =====\n`
for (const file of section.files) {
const content = read(ownedPath(root, file))
sources.push({ file, sha256Lf: sha256(content) })
text += `\n===== ${file} =====\n\n${content}\n`
}
}
return new Map([
['docs/llms.txt', text],
[
'docs/llms.manifest.json',
`${JSON.stringify({ schemaVersion: 1, generation: 'offline-source-preserving', sources, outputSha256Lf: sha256(text) }, null, 2)}\n`,
],
])
}
+58
View File
@@ -0,0 +1,58 @@
import assert from 'node:assert/strict'
import crypto from 'node:crypto'
import fs from 'node:fs'
import path from 'node:path'
export function sha256(text: string): string {
return crypto.createHash('sha256').update(text).digest('hex')
}
export function read(file: string): string {
return fs.readFileSync(file, 'utf8').replaceAll('\r\n', '\n')
}
export function ownedPath(root: string, relative: string): string {
assert(
relative
&& !relative.includes('\\')
&& !path.isAbsolute(relative)
&& !relative.split('/').includes('..'),
'Invalid documentation path',
)
const base = fs.realpathSync(root)
const file = path.resolve(base, relative)
let ancestor = file
while (true) {
try {
fs.lstatSync(ancestor)
break
}
catch (error) {
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT'))
throw error
const parent = path.dirname(ancestor)
assert(parent !== ancestor, 'No existing documentation path ancestor')
ancestor = parent
}
}
const inside = path.relative(base, fs.realpathSync(ancestor))
assert(
inside !== '..'
&& !inside.startsWith(`..${path.sep}`)
&& !path.isAbsolute(inside),
'Documentation path escapes workspace',
)
return file
}
export function atomicWrite(file: string, content: string): void {
fs.mkdirSync(path.dirname(file), { recursive: true })
const temporary = `${file}.${crypto.randomUUID()}.tmp`
try {
fs.writeFileSync(temporary, content, { flag: 'wx' })
fs.renameSync(temporary, file)
}
finally {
if (fs.existsSync(temporary))
fs.unlinkSync(temporary)
}
}
+150
View File
@@ -0,0 +1,150 @@
import assert from 'node:assert/strict'
import MarkdownIt from 'markdown-it'
import { sha256 } from './files.ts'
const parser = new MarkdownIt({ html: true })
export function markdownSignature(source: string): string[] {
const result: string[] = []
function visit(tokens: ReturnType<MarkdownIt['parse']>): void {
for (const token of tokens) {
if (
[
'fence',
'code_block',
'code_inline',
'html_block',
'html_inline',
].includes(token.type)
) {
result.push(JSON.stringify([token.type, token.info, token.content]))
}
if (
[
'heading_open',
'bullet_list_open',
'ordered_list_open',
'list_item_open',
'blockquote_open',
'table_open',
'tr_open',
'th_open',
'td_open',
].includes(token.type)
) {
result.push(JSON.stringify([token.type, token.tag]))
}
if (token.type === 'link_open')
result.push(`link:${token.attrGet('href')}`)
if (token.type === 'image')
result.push(`image:${token.attrGet('src')}`)
if (token.children)
visit(token.children)
}
}
visit(parser.parse(source, {}))
result.push(...source.split('\n').filter(line => /^\s*:::/.test(line)))
return result
}
export function protectMarkdown(source: string): {
masked: string
restore: (translated: string) => string
} {
const lines = source.split('\n')
const ranges: [number, number][] = []
for (const token of parser.parse(source, {})) {
if (
!token.map
|| !['fence', 'code_block', 'html_block'].includes(token.type)
) {
continue
}
if (token.type === 'fence') {
const closing = lines[token.map[1] - 1]?.trim() || ''
assert(
closing.length >= token.markup.length
&& [...closing].every(char => char === token.markup[0]),
'Unclosed source code fence',
)
}
ranges.push([...token.map])
}
lines.forEach((line, index) => {
if (/^\s*:::/.test(line))
ranges.push([index, index + 1])
})
const merged: [number, number][] = []
for (const range of ranges.sort((a, b) => a[0] - b[0])) {
const previous = merged[merged.length - 1]
if (previous && range[0] <= previous[1])
previous[1] = Math.max(previous[1], range[1])
else merged.push(range)
}
const prefix = `ARTPLAYER_KEEP_${sha256(source).slice(0, 16)}_`
assert(!source.includes(prefix), 'Protection marker collision')
const blocks: { marker: string, content: string }[] = []
const output: string[] = []
let cursor = 0
for (const [start, end] of merged) {
output.push(...lines.slice(cursor, start))
const marker = `${prefix}${blocks.length}_END`
blocks.push({ marker, content: lines.slice(start, end).join('\n') })
output.push(marker)
cursor = end
}
output.push(...lines.slice(cursor))
return {
masked: output.join('\n'),
restore(translated: string): string {
let result = translated
for (const block of blocks) {
assert(
result.split(block.marker).length === 2,
'Translation lost or duplicated a protected block',
)
result = result.replace(block.marker, () => block.content)
}
assert(
!result.includes(prefix),
'Unexpected protection marker in translation',
)
assert.deepEqual(
markdownSignature(result),
markdownSignature(source),
'Translation changed code, HTML, links or Markdown structure',
)
return result
},
}
}
export function splitTranslation(masked: string, limit = 4000): string[] {
assert(
Number.isInteger(limit) && limit >= 128,
'Invalid translation chunk limit',
)
const chunks: string[] = []
let current = ''
for (let line of masked.split('\n')) {
while (line.length > limit) {
if (current.trim())
chunks.push(current)
current = ''
let end = limit
if (/[\uD800-\uDBFF]/.test(line.charAt(end - 1)))
end--
chunks.push(line.slice(0, end))
line = line.slice(end)
}
if (current.length + line.length + 1 > limit) {
if (current.trim())
chunks.push(current)
current = ''
}
current += `${current ? '\n' : ''}${line}`
}
if (current.trim())
chunks.push(current)
return chunks
}
+100
View File
@@ -0,0 +1,100 @@
import assert from 'node:assert/strict'
import { setTimeout as delay } from 'node:timers/promises'
interface RemoteOptions {
key: string
request?: typeof fetch
signal?: AbortSignal
timeoutMs?: number
retries?: number
wait?: (ms: number) => Promise<unknown>
}
function object(value: unknown): Record<string, unknown> {
return value !== null && typeof value === 'object'
? (value as Record<string, unknown>)
: {}
}
export async function translateChunk(
content: string,
options: RemoteOptions,
): Promise<string> {
assert(options.key, 'Missing DEEPSEEK_API_KEY')
const retries = options.retries ?? 3
assert(
Number.isInteger(retries) && retries > 0 && retries <= 5,
'Invalid retry limit',
)
const timeoutMs = options.timeoutMs ?? 60000
assert(
Number.isFinite(timeoutMs) && timeoutMs > 0,
'Invalid request timeout',
)
for (let attempt = 1; attempt <= retries; attempt++) {
options.signal?.throwIfAborted()
const controller = new AbortController()
const timeout = setTimeout(
() => controller.abort(new Error('Translation request timed out')),
timeoutMs,
)
const signal = options.signal
? AbortSignal.any([controller.signal, options.signal])
: controller.signal
let retry = false
try {
const response = await (options.request ?? fetch)(
'https://api.deepseek.com/v1/chat/completions',
{
method: 'POST',
signal,
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${options.key}`,
},
body: JSON.stringify({
model: 'deepseek-chat',
temperature: 0.2,
messages: [
{
role: 'user',
content: `Translate this Chinese technical Markdown to English. Preserve structure, inline code, URLs and every ARTPLAYER_KEEP marker exactly once. Return only Markdown, without an outer code fence or explanations.\n\n${content}`,
},
],
}),
},
)
retry = response.status === 429 || response.status >= 500
if (!response.ok)
await response.body?.cancel()
assert(response.ok, `Translation HTTP ${response.status}`)
const data: unknown = await response.json().catch((error: unknown) => {
if (signal.aborted)
throw error
assert.fail('Invalid translation JSON response')
})
const choices = object(data).choices
const text = object(
object(Array.isArray(choices) ? choices[0] : undefined).message,
).content
assert(
typeof text === 'string' && text.trim(),
'Invalid or empty translation response',
)
return text.trim()
}
catch (error) {
if (options.signal?.aborted)
throw error
if (!(error instanceof assert.AssertionError))
retry = true
if (!retry || attempt === retries)
throw error
}
finally {
clearTimeout(timeout)
}
await (options.wait ?? delay)(500 * attempt)
}
throw new Error('Translation retries exhausted')
}
+273
View File
@@ -0,0 +1,273 @@
import assert from 'node:assert/strict'
import fs from 'node:fs'
import path from 'node:path'
import { globSync } from 'glob'
import { atomicWrite, ownedPath, read, sha256 } from './files.ts'
import {
markdownSignature,
protectMarkdown,
splitTranslation,
} from './markdown.ts'
interface TranslationEntry {
source: string
target: string
relative: string
sourceHash: string
targetHash: string | null
translatedHash?: string
}
interface Draft {
schemaVersion: 1
status: 'incomplete' | 'failed' | 'complete' | 'applied'
entries: TranslationEntry[]
error?: string
}
const docs = 'packages/artplayer-vitepress/docs'
const drafts = 'refactor/.cache/translations'
export function translationPlan(root: string): TranslationEntry[] {
return [
'index.md',
...globSync('{advanced,component,start}/**/*.md', {
cwd: ownedPath(root, docs),
posix: true,
}).sort(),
].map((relative) => {
const source = `${docs}/${relative}`
const target = `${docs}/en/${relative}`
const destination = ownedPath(root, target)
return {
source,
target,
relative,
sourceHash: sha256(read(ownedPath(root, source))),
targetHash: fs.existsSync(destination) ? sha256(read(destination)) : null,
}
})
}
export async function createTranslationDraft(
root: string,
translate: (content: string, signal: AbortSignal) => Promise<string>,
concurrency = 3,
): Promise<string> {
assert(Number.isInteger(concurrency) && concurrency > 0 && concurrency <= 5)
const entries = translationPlan(root)
const cache = ownedPath(root, drafts)
fs.mkdirSync(cache, { recursive: true })
const directory = fs.mkdtempSync(path.join(cache, 'draft-'))
const manifest: Draft = { schemaVersion: 1, status: 'incomplete', entries }
const save = () =>
atomicWrite(
path.join(directory, 'manifest.json'),
`${JSON.stringify(manifest, null, 2)}\n`,
)
save()
const controller = new AbortController()
let cursor = 0
let failure: unknown
async function worker(): Promise<void> {
try {
while (!controller.signal.aborted) {
const entry = entries[cursor++]
if (!entry)
return
const source = read(ownedPath(root, entry.source))
assert.equal(
sha256(source),
entry.sourceHash,
'Source changed before translation',
)
const protectedSource = protectMarkdown(source)
const chunks: string[] = []
for (const chunk of splitTranslation(protectedSource.masked)) {
controller.signal.throwIfAborted()
chunks.push(await translate(chunk, controller.signal))
}
controller.signal.throwIfAborted()
const translated = protectedSource.restore(chunks.join('\n\n'))
assert(translated.trim(), 'Empty translated document')
atomicWrite(ownedPath(directory, entry.relative), translated)
entry.translatedHash = sha256(translated)
save()
}
}
catch (error) {
if (!controller.signal.aborted)
failure = error
controller.abort(error)
}
}
await Promise.all(
Array.from({ length: Math.min(concurrency, entries.length) }, worker),
)
if (controller.signal.aborted) {
manifest.status = 'failed'
manifest.error
= failure instanceof Error ? failure.message : 'Translation failed'
save()
throw new Error(
`Translation draft failed; existing English files unchanged. Draft: ${directory}. ${manifest.error}`,
)
}
manifest.status = 'complete'
save()
return directory
}
function inspectDraft(root: string, directory: string, checkHash = true) {
const relative = path
.relative(fs.realpathSync(root), path.resolve(directory))
.split(path.sep)
.join('/')
assert(
relative.startsWith(`${drafts}/draft-`),
'Apply requires an owned translation draft',
)
const safeDirectory = ownedPath(root, relative)
const manifest = JSON.parse(
read(ownedPath(safeDirectory, 'manifest.json')),
) as Draft
assert(
manifest.schemaVersion === 1
&& manifest.status === 'complete'
&& Array.isArray(manifest.entries),
'Draft is not complete',
)
const current = translationPlan(root)
assert.equal(
manifest.entries.length,
current.length,
'Translation source set changed',
)
const changes = current.map((entry, index) => {
const saved = manifest.entries[index]
assert(
saved
&& saved.source === entry.source
&& saved.target === entry.target
&& saved.relative === entry.relative,
'Invalid draft file mapping',
)
assert.equal(
saved.sourceHash,
entry.sourceHash,
'Source changed since translation',
)
assert.equal(
saved.targetHash,
entry.targetHash,
'English document changed since translation',
)
const content = read(ownedPath(safeDirectory, entry.relative))
if (checkHash) {
assert.equal(
sha256(content),
saved.translatedHash,
'Draft content changed without validation',
)
}
assert.deepEqual(
markdownSignature(content),
markdownSignature(read(ownedPath(root, entry.source))),
'Draft changed protected Markdown',
)
const file = ownedPath(root, entry.target)
return {
...entry,
file,
content,
before: fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : null,
}
})
return { safeDirectory, manifest, changes }
}
export function validateTranslationDraft(
root: string,
directory: string,
): void {
const { safeDirectory, manifest, changes } = inspectDraft(
root,
directory,
false,
)
changes.forEach((change, index) => {
const entry = manifest.entries[index]
assert(entry)
entry.translatedHash = sha256(change.content)
})
atomicWrite(
ownedPath(safeDirectory, 'manifest.json'),
`${JSON.stringify(manifest, null, 2)}\n`,
)
}
export function applyTranslationDraft(
root: string,
directory: string,
write: typeof atomicWrite = atomicWrite,
): void {
const { safeDirectory, manifest, changes } = inspectDraft(root, directory)
const applied: typeof changes = []
try {
for (const change of changes) {
assert.equal(
sha256(read(ownedPath(root, change.source))),
change.sourceHash,
'Source changed during apply',
)
assert.equal(
read(ownedPath(safeDirectory, change.relative)),
change.content,
'Draft changed during apply',
)
assert.equal(ownedPath(root, change.target), change.file)
const currentHash = fs.existsSync(change.file)
? sha256(read(change.file))
: null
assert.equal(
currentHash,
change.targetHash,
'English document changed during apply',
)
write(change.file, change.content)
applied.push(change)
}
for (const change of changes) {
assert.equal(
sha256(read(ownedPath(root, change.source))),
change.sourceHash,
'Source changed during apply',
)
}
manifest.status = 'applied'
atomicWrite(
ownedPath(safeDirectory, 'manifest.json'),
`${JSON.stringify(manifest, null, 2)}\n`,
)
}
catch (error) {
const failures: unknown[] = [error]
for (const change of applied.reverse()) {
try {
assert.equal(
sha256(read(change.file)),
sha256(change.content),
'Concurrent change prevents safe rollback',
)
if (change.before === null)
fs.unlinkSync(change.file)
else atomicWrite(change.file, change.before)
}
catch (failure) {
failures.push(failure)
}
}
throw new AggregateError(
failures,
'Draft apply failed; inspect errors and retained draft before retrying',
)
}
}