feat: add Upscaler web interface and core functionality

- Implemented a new HTML interface for image and video upscaling with WebGPU and WebGL support.
- Created the Upscaler class to handle image and video processing, including loading weights, managing workers, and handling real-time upscaling.
- Added support for selecting processing modes (small, medium, large) and handling file uploads.
- Integrated progress tracking and result display for processed images and videos.
- Implemented functionality for downloading processed results and real-time video processing from a URL.
This commit is contained in:
Harvey Zhao committed 2026-03-01 11:50:15 +08:00
1 parent 1841f754dc
commit f750e77e4f
2 files changed
+1612

No files matched your search

+997
View File
@@ -0,0 +1,997 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Upscaler 测试</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: "Segoe UI", Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
display: flex;
align-items: center;
justify-content: center;
padding: 20px;
}
.container {
background: white;
border-radius: 12px;
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
max-width: 600px;
width: 100%;
padding: 40px;
}
h1 {
color: #333;
margin-bottom: 30px;
text-align: center;
font-size: 2em;
}
.info-box {
background: #f0f4ff;
border-left: 4px solid #667eea;
padding: 15px;
margin-bottom: 20px;
border-radius: 4px;
font-size: 14px;
color: #555;
}
.form-group {
margin-bottom: 20px;
}
label {
display: block;
margin-bottom: 8px;
font-weight: 600;
color: #333;
}
input[type="file"],
select {
width: 100%;
padding: 12px;
border: 2px solid #ddd;
border-radius: 6px;
font-size: 14px;
transition: border-color 0.3s;
}
input[type="file"]:focus,
select:focus {
outline: none;
border-color: #667eea;
}
button {
width: 100%;
padding: 14px;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
border: none;
border-radius: 6px;
font-size: 16px;
font-weight: 600;
cursor: pointer;
transition:
transform 0.2s,
box-shadow 0.2s;
margin-top: 10px;
}
button:hover:not(:disabled) {
transform: translateY(-2px);
box-shadow: 0 10px 20px rgba(102, 126, 234, 0.4);
}
button:disabled {
opacity: 0.6;
cursor: not-allowed;
}
.progress-section {
margin-top: 30px;
display: none;
}
.progress-section.active {
display: block;
}
.progress-bar {
width: 100%;
height: 8px;
background: #eee;
border-radius: 4px;
overflow: hidden;
margin-bottom: 10px;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);
width: 0%;
transition: width 0.3s ease;
}
.progress-text {
text-align: center;
color: #666;
font-size: 14px;
}
.result-section {
margin-top: 30px;
display: none;
border-top: 2px solid #eee;
padding-top: 20px;
}
.result-section.active {
display: block;
}
.image-viewer {
display: none;
}
.image-viewer.active {
display: block;
}
.result-image {
max-width: 100%;
border-radius: 6px;
margin-bottom: 15px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
}
.comparison-container {
position: relative;
width: 100%;
aspect-ratio: auto;
max-width: 100%;
border-radius: 6px;
overflow: hidden;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
margin-bottom: 15px;
background: #f0f0f0;
user-select: none;
}
.comparison-inner {
position: relative;
width: 100%;
height: auto;
}
.comparison-img {
display: block;
width: 100%;
height: auto;
}
.comparison-img.original {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
object-fit: contain;
-webkit-mask-image: linear-gradient(
to right,
black 0%,
black 50%,
transparent 50%,
transparent 100%
);
mask-image: linear-gradient(
to right,
black 0%,
black 50%,
transparent 50%,
transparent 100%
);
}
.comparison-img.enhanced {
width: 100%;
height: auto;
}
.comparison-container video {
display: block;
width: 100%;
height: auto;
border-radius: 6px;
background: #000;
}
.comparison-container video.original {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
border-radius: 6px;
}
.comparison-handle {
position: absolute;
left: 50%;
top: 0;
transform: translateX(-50%);
width: 4px;
height: 100%;
background: white;
cursor: ew-resize;
z-index: 10;
box-shadow: 0 0 8px rgba(0, 0, 0, 0.3);
transition: background-color 0.2s;
}
.comparison-handle:hover {
background: #667eea;
width: 6px;
}
.comparison-handle::before {
content: "";
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 40px;
height: 40px;
background: white;
border-radius: 50%;
box-shadow: 0 0 8px rgba(0, 0, 0, 0.2);
display: flex;
align-items: center;
justify-content: center;
}
.comparison-labels {
position: absolute;
top: 10px;
width: 100%;
display: flex;
justify-content: space-between;
padding: 0 15px;
pointer-events: none;
z-index: 5;
font-size: 12px;
font-weight: 600;
color: white;
text-shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
}
.comparison-label {
background: rgba(0, 0, 0, 0.3);
padding: 6px 12px;
border-radius: 4px;
backdrop-filter: blur(4px);
}
.result-info {
background: #f9f9f9;
padding: 15px;
border-radius: 6px;
font-size: 13px;
color: #666;
margin-bottom: 10px;
}
.download-btn {
background: #4caf50;
margin-top: 10px;
}
.download-btn:hover {
box-shadow: 0 10px 20px rgba(76, 175, 80, 0.4);
}
.status-badge {
display: inline-block;
padding: 6px 12px;
border-radius: 12px;
font-size: 12px;
font-weight: 600;
margin-top: 10px;
}
.status-badge.success {
background: #d4edda;
color: #155724;
}
.status-badge.error {
background: #f8d7da;
color: #721c24;
}
.status-badge.info {
background: #d1ecf1;
color: #0c5460;
}
.section-title {
color: #333;
font-size: 14px;
font-weight: 600;
margin-bottom: 15px;
text-transform: uppercase;
letter-spacing: 0.5px;
}
</style>
</head>
<body>
<div class="container">
<h1>🚀 超分辨率处理</h1>
<div class="info-box">
ℹ️ 支持图像和视频处理。支持 WebGPU 和 WebGL 后端。
</div>
<div class="form-group">
<label for="universalInput">📁 选择图像或视频</label>
<input
type="file"
id="universalInput"
accept="image/*,video/*"
/>
</div>
<div class="form-group">
<label for="networkSize">⚙️ 处理模式</label>
<select id="networkSize">
<option value="small">小 (快速,质量较低)</option>
<option value="medium" selected>中 (平衡)</option>
<option value="large">大 (最佳质量,时间长)</option>
</select>
</div>
<button id="processBtn" onclick="handleProcess()">
▶️ 开始处理
</button>
<div class="form-group" style="margin-top: 20px">
<label for="videoUrlInput">🌐 在线视频 URL 实时放大</label>
<input
type="text"
id="videoUrlInput"
value="/video.mp4"
placeholder="https://example.com/video.mp4"
style="
width: 100%;
padding: 12px;
border: 2px solid #ddd;
border-radius: 6px;
font-size: 14px;
"
/>
</div>
<button id="playUrlBtn" onclick="handlePlayUrl()">
▶️ 播放并实时放大到下方 Canvas
</button>
<div class="progress-section" id="progressSection">
<div class="section-title">处理进度</div>
<div class="progress-bar">
<div class="progress-fill" id="progressFill"></div>
</div>
<div class="progress-text" id="progressText">准备中...</div>
</div>
<div class="result-section" id="resultSection">
<div class="section-title">处理完成</div>
<div class="image-viewer active" id="comparisonViewer">
<div class="comparison-container" id="comparisonContainer">
<img
id="comparisonEnhanced"
class="comparison-img enhanced"
alt="增强图像"
/>
<img
id="comparisonOriginal"
class="comparison-img original"
alt="原始图像"
/>
<div
class="comparison-handle"
id="comparisonHandle"
></div>
<div class="comparison-labels">
<div class="comparison-label">原始图像</div>
<div class="comparison-label">增强后</div>
</div>
</div>
</div>
<div
id="videoResultSection"
style="display: none; margin-top: 20px"
>
<h3 style="margin-top: 0; color: #333">✓ 视频处理完成</h3>
<div
id="videoComparisonViewer"
style="display: block; margin-bottom: 15px"
>
<div
class="comparison-container"
id="videoComparisonContainer"
>
<video
id="resultVideoEnhanced"
class="comparison-img enhanced"
style="
display: block;
width: 100%;
height: auto;
border-radius: 6px;
background: #000;
"
controls
loop
muted
preload="auto"
>
<source
id="resultVideoSourceEnhanced"
type="video/mp4"
/>
你的浏览器不支持 HTML5 video 标签
</video>
<video
id="resultVideoOriginal"
class="comparison-img original"
style="
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
border-radius: 6px;
background: #000;
"
loop
preload="auto"
>
<source
id="resultVideoSourceOriginal"
type="video/mp4"
/>
你的浏览器不支持 HTML5 video 标签
</video>
<div
class="comparison-handle"
id="videoComparisonHandle"
></div>
<div class="comparison-labels">
<div class="comparison-label">原始视频</div>
<div class="comparison-label">处理后</div>
</div>
</div>
</div>
<div class="result-info">
<strong>文件信息:</strong><br />
原始: <span id="originalVideoSize">-</span> | 处理后:
<span id="resultVideoSize">-</span><br />
后端: <span id="backendVideoInfo">-</span>
</div>
<button class="download-btn" onclick="downloadResult()">
⬇️ 下载处理后的视频
</button>
</div>
<div id="imageResultInfo" style="display: none">
<div class="result-info">
<strong>文件信息:</strong><br />
原始: <span id="originalSize">-</span> | 处理后:
<span id="resultSize">-</span><br />
后端: <span id="backendInfo">-</span>
</div>
<button class="download-btn" onclick="downloadResult()">
⬇️ 下载处理后的图像
</button>
</div>
</div>
<div id="streamSection" style="margin-top: 30px; display: none">
<div class="section-title">实时播放 (URL)</div>
<div class="result-info">
将上方输入框中的在线视频 URL 播放到左侧原始视频,并在右侧
Canvas 上以放大分辨率实时绘制画面,通过中间滑块进行对比。<br />
提示:跨域视频如果没有正确的 CORS 头,可能无法绘制到
Canvas。
</div>
<div
id="streamComparisonViewer"
style="display: block; margin-bottom: 15px"
>
<div
class="comparison-container"
id="streamComparisonContainer"
>
<canvas
id="streamUpscaledCanvas"
class="comparison-img enhanced"
style="
display: block;
width: 100%;
height: auto;
border-radius: 6px;
background: #000;
"
></canvas>
<video
id="streamSourceVideo"
class="comparison-img original"
style="
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
border-radius: 6px;
background: #000;
"
controls
muted
preload="auto"
></video>
<div
class="comparison-handle"
id="streamComparisonHandle"
></div>
<div class="comparison-labels">
<div class="comparison-label">原始视频</div>
<div class="comparison-label">实时放大</div>
</div>
</div>
</div>
</div>
</div>
<script src="upscaler.js"></script>
<script>
let currentResultBlob = null;
let currentOriginalUrl = null;
let isComparisonDragging = false;
let currentMode = "image";
// 创建一个可配置的 Upscaler 实例
const upscaler = new Upscaler({
networkSize: "medium",
weightsBaseUrl: "/assets/websr/weights",
workerUrl: "/assets/websr/worker/main.js",
});
function updateComparisonPosition(element, rect, x) {
x = Math.max(0, Math.min(x, rect.width));
const percentage = (x / rect.width) * 100;
return { x, percentage };
}
function applyMaskToElement(element, percentage) {
const maskGradient = `linear-gradient(to right, black 0%, black ${percentage}%, transparent ${percentage}%, transparent 100%)`;
element.style.maskImage = maskGradient;
element.style.webkitMaskImage = maskGradient;
}
function setupComparisonDrag(container, handle, originalElement) {
function updatePosition(e) {
if (!isComparisonDragging) return;
const rect = container.getBoundingClientRect();
let x = e.clientX - rect.left;
if (e.touches) {
x = e.touches[0].clientX - rect.left;
}
const { percentage } = updateComparisonPosition(
originalElement,
rect,
x,
);
handle.style.left = x + "px";
applyMaskToElement(originalElement, percentage);
}
handle.addEventListener("mousedown", () => {
isComparisonDragging = true;
container.style.cursor = "grabbing";
});
document.addEventListener("mousemove", updatePosition);
document.addEventListener("mouseup", () => {
isComparisonDragging = false;
container.style.cursor = "default";
});
handle.addEventListener("touchstart", () => {
isComparisonDragging = true;
});
document.addEventListener("touchmove", updatePosition);
document.addEventListener("touchend", () => {
isComparisonDragging = false;
});
container.addEventListener("click", (e) => {
const rect = container.getBoundingClientRect();
const x = e.clientX - rect.left;
const { percentage } = updateComparisonPosition(
originalElement,
rect,
x,
);
handle.style.left = x + "px";
applyMaskToElement(originalElement, percentage);
});
applyMaskToElement(originalElement, 50);
handle.style.left = container.offsetWidth / 2 + "px";
}
function initComparisonDrag() {
const container = document.getElementById(
"comparisonContainer",
);
const handle = document.getElementById("comparisonHandle");
const originalImg =
document.getElementById("comparisonOriginal");
setupComparisonDrag(container, handle, originalImg);
}
function initVideoComparisonDrag() {
const container = document.getElementById(
"videoComparisonContainer",
);
const handle = document.getElementById("videoComparisonHandle");
const originalVideo = document.getElementById(
"resultVideoOriginal",
);
const enhancedVideo = document.getElementById(
"resultVideoEnhanced",
);
enhancedVideo.addEventListener("play", () =>
originalVideo.play(),
);
enhancedVideo.addEventListener("pause", () =>
originalVideo.pause(),
);
enhancedVideo.addEventListener("timeupdate", () => {
if (
Math.abs(
originalVideo.currentTime -
enhancedVideo.currentTime,
) > 0.1
) {
originalVideo.currentTime = enhancedVideo.currentTime;
}
});
enhancedVideo.addEventListener("seeking", () => {
originalVideo.currentTime = enhancedVideo.currentTime;
});
setupComparisonDrag(container, handle, originalVideo);
enhancedVideo.autoplay = true;
enhancedVideo.play().catch(() => {});
}
function initStreamComparisonDrag() {
const container = document.getElementById(
"streamComparisonContainer",
);
const handle = document.getElementById(
"streamComparisonHandle",
);
const originalVideo =
document.getElementById("streamSourceVideo");
if (!container || !handle || !originalVideo) return;
setupComparisonDrag(container, handle, originalVideo);
}
async function handleProcess() {
const file = document.getElementById("universalInput").files[0];
if (!file) {
alert("请先选择图像或视频");
return;
}
const isVideo = file.type.startsWith("video/");
const isImage = file.type.startsWith("image/");
if (!isVideo && !isImage) {
alert("不支持的文件类型,请选择图像或视频");
return;
}
currentMode = isVideo ? "video" : "image";
const networkSize =
document.getElementById("networkSize").value;
const processBtn = document.getElementById("processBtn");
const progressSection =
document.getElementById("progressSection");
const resultSection = document.getElementById("resultSection");
const videoResultSection =
document.getElementById("videoResultSection");
processBtn.disabled = true;
progressSection.classList.add("active");
resultSection.classList.remove("active");
if (videoResultSection)
videoResultSection.style.display = "none";
document.getElementById("progressText").textContent =
"初始化中...";
document.getElementById("progressFill").style.width = "0%";
try {
upscaler.onProgress((progress) => {
document.getElementById("progressFill").style.width =
progress + "%";
document.getElementById("progressText").textContent =
`处理中... ${Math.round(progress)}%`;
});
console.log("开始处理:", file.name, "模式:", networkSize);
if (currentMode === "video") {
console.log("🎬 处理视频文件...");
const resultBlob = await upscaler.upscaleVideo(
file,
networkSize,
);
currentResultBlob = resultBlob;
const videoUrl = URL.createObjectURL(resultBlob);
const originalVideoUrl = URL.createObjectURL(file);
currentOriginalUrl = originalVideoUrl;
document.getElementById(
"resultVideoSourceEnhanced",
).src = videoUrl;
document.getElementById(
"resultVideoSourceEnhanced",
).type = "video/mp4";
document.getElementById("resultVideoEnhanced").load();
document.getElementById(
"resultVideoSourceOriginal",
).src = originalVideoUrl;
document.getElementById(
"resultVideoSourceOriginal",
).type = "video/mp4";
document.getElementById("resultVideoOriginal").load();
document.getElementById(
"originalVideoSize",
).textContent = formatBytes(file.size);
document.getElementById("resultVideoSize").textContent =
formatBytes(resultBlob.size);
document.getElementById(
"backendVideoInfo",
).textContent = Upscaler.getBackendType();
progressSection.classList.remove("active");
resultSection.classList.add("active");
if (videoResultSection) {
videoResultSection.style.display = "block";
setTimeout(() => {
initVideoComparisonDrag();
console.log("✓ 视频对比拖动已初始化");
}, 500);
}
console.log("✓ 视频处理完成!");
} else {
console.log("📷 处理图像文件...");
currentOriginalUrl = URL.createObjectURL(file);
const resultBlob = await upscaler.upscaleImage(
file,
networkSize,
);
currentResultBlob = resultBlob;
const previewUrl = URL.createObjectURL(resultBlob);
const enhancedImg =
document.getElementById("comparisonEnhanced");
const originalImg =
document.getElementById("comparisonOriginal");
if (originalImg) {
originalImg.onload = () => {
console.log(
"[Image loaded] Initializing comparison drag",
);
initComparisonDrag();
};
}
if (enhancedImg) enhancedImg.src = previewUrl;
if (originalImg) originalImg.src = currentOriginalUrl;
document.getElementById("originalSize").textContent =
formatBytes(file.size);
document.getElementById("resultSize").textContent =
formatBytes(resultBlob.size);
document.getElementById("backendInfo").textContent =
Upscaler.getBackendType();
progressSection.classList.remove("active");
resultSection.classList.add("active");
const imageResultInfo =
document.getElementById("imageResultInfo");
if (imageResultInfo)
imageResultInfo.style.display = "block";
console.log("✓ 图像处理完成!");
}
} catch (error) {
console.error("✗ 处理错误:", error);
alert("处理失败:\n" + error.message);
progressSection.classList.remove("active");
} finally {
processBtn.disabled = false;
}
}
function downloadResult() {
if (!currentResultBlob) {
alert("没有可下载的结果");
return;
}
const url = URL.createObjectURL(currentResultBlob);
const link = document.createElement("a");
link.href = url;
link.download = `upscaled_${Date.now()}.${currentMode === "video" ? "mp4" : "png"}`;
link.click();
URL.revokeObjectURL(url);
}
function formatBytes(bytes) {
if (bytes === 0) return "0 Bytes";
const k = 1024;
const sizes = ["Bytes", "KB", "MB", "GB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return (
Math.round((bytes / Math.pow(k, i)) * 100) / 100 +
" " +
sizes[i]
);
}
async function handlePlayUrl() {
const urlInput = document.getElementById("videoUrlInput");
const url = urlInput.value.trim();
if (!url) {
alert("请先输入视频 URL");
return;
}
console.log("[URL] play clicked", url);
if (!Upscaler.isSupported()) {
alert(
"当前环境不支持 Upscaler(需要 Worker + OffscreenCanvas)",
);
return;
}
const streamSection = document.getElementById("streamSection");
const videoEl = document.getElementById("streamSourceVideo");
const canvasEl = document.getElementById(
"streamUpscaledCanvas",
);
const networkSize =
document.getElementById("networkSize").value;
try {
if (typeof upscaler.stopRealtimeUpscale === "function") {
upscaler.stopRealtimeUpscale();
}
videoEl.pause();
videoEl.removeAttribute("src");
videoEl.load();
// 不强制 crossOrigin,避免同源资源因缺少 CORS 头加载失败
videoEl.src = url;
streamSection.style.display = "block";
await new Promise((resolve, reject) => {
if (videoEl.readyState >= 1) {
console.log("[URL] video metadata already ready", {
width: videoEl.videoWidth,
height: videoEl.videoHeight,
readyState: videoEl.readyState,
});
resolve();
return;
}
const onLoaded = () => {
cleanup();
console.log("[URL] loadedmetadata", {
width: videoEl.videoWidth,
height: videoEl.videoHeight,
readyState: videoEl.readyState,
});
resolve();
};
const onError = () => {
cleanup();
console.error("[URL] video error event");
reject(new Error("无法加载视频元数据"));
};
const cleanup = () => {
videoEl.removeEventListener(
"loadedmetadata",
onLoaded,
);
videoEl.removeEventListener("error", onError);
};
videoEl.addEventListener("loadedmetadata", onLoaded);
videoEl.addEventListener("error", onError);
});
initStreamComparisonDrag();
await upscaler.startRealtimeUpscale(
videoEl,
canvasEl,
networkSize,
);
console.log(
"[URL] realtime upscaler started, try play video",
);
await videoEl.play().catch(() => {});
} catch (err) {
console.error("URL 实时播放失败:", err);
alert("无法播放或处理该视频:\n" + err.message);
}
}
console.log("✓ Upscaler API 已就绪!");
console.log(
"支持的模式:",
upscaler.getSupportedNetworks().join(", "),
);
console.log("后端类型:", Upscaler.getBackendType());
</script>
</body>
</html>
+615
View File
@@ -0,0 +1,615 @@
class Upscaler {
static DEFAULT_TIMEOUTS = { IMAGE: 300000, VIDEO: 3600000, METADATA: 10000 };
static DEFAULT_DELAYS = { INIT: 500, NETWORK: 300 };
static isSupported() {
try {
const hasWorker = typeof Worker !== "undefined";
const hasOffscreen =
typeof OffscreenCanvas !== "undefined" ||
(typeof document !== "undefined" &&
!!document.createElement("canvas").transferControlToOffscreen);
const hasBlob = typeof Blob !== "undefined";
return hasWorker && hasOffscreen && hasBlob;
} catch {
return false;
}
}
static isVideoSupported() {
try {
return (
typeof VideoEncoder !== "undefined" &&
typeof VideoDecoder !== "undefined"
);
} catch {
return false;
}
}
constructor(options = {}) {
const weightsBaseUrl = options.weightsBaseUrl || "/weights";
this.networks = options.networks || {
small: {
name: "anime4k/cnn-2x-l",
weightsUrl: `${weightsBaseUrl}/cnn-8.json`,
},
medium: {
name: "anime4k/cnn-2x-16",
weightsUrl: `${weightsBaseUrl}/cnn-16.json`,
},
large: {
name: "anime4k/cnn-2x-28",
weightsUrl: `${weightsBaseUrl}/cnn-28.json`,
},
};
this.weightsBaseUrl = weightsBaseUrl;
this.networkSize = options.networkSize || "medium";
this.workerUrl = options.workerUrl || "/worker/main.js";
this.timeouts = {
...Upscaler.DEFAULT_TIMEOUTS,
...(options.timeouts || {}),
};
this.delays = { ...Upscaler.DEFAULT_DELAYS, ...(options.delays || {}) };
this.imageScale =
typeof options.imageScale === "number" && options.imageScale > 0
? options.imageScale
: 2;
this.videoScale =
typeof options.videoScale === "number" && options.videoScale > 0
? options.videoScale
: 2;
this.weightsCache = new Map();
this.workerInstance = null;
this.messageHandlers = {};
this.progressCallback = null;
this.processingType = null;
this.realtimeLoopId = null;
this.realtimeState = null;
this.init();
}
init({ prewarm = true } = {}) {
if (!Upscaler.isSupported()) {
throw new Error("Upscaler is not supported in this environment");
}
if (prewarm) {
this.getWorker().postMessage({ cmd: "isSupported" });
}
return this;
}
getWorker() {
if (!this.workerInstance) {
this.workerInstance = new Worker(this.workerUrl);
this.workerInstance.onmessage = (event) =>
this.handleWorkerMessage(event);
}
return this.workerInstance;
}
static extractProgressValue(data) {
const value = data.data ?? data.progress ?? data.value ?? data.percentage;
return typeof value === "number"
? Math.min(100, Math.max(0, Math.round(value)))
: null;
}
handleBlobResponse(data) {
const blobType =
this.processingType === "video" ? "videoBlob" : "imageBlob";
const handler = this.messageHandlers[blobType];
if (data.data instanceof Blob && handler) {
handler({ [blobType]: data.data });
}
}
requestBlob() {
const cmd =
this.processingType === "video" ? "getVideoBlob" : "getImageBlob";
this.getWorker().postMessage({ cmd });
}
handleWorkerMessage(event) {
const { data } = event;
console.debug("Worker message:", data);
if (!data.cmd) return;
const { cmd } = data;
if (cmd === "progress") {
const progress = Upscaler.extractProgressValue(data);
if (progress !== null && this.progressCallback) {
this.progressCallback(progress);
}
if (this.messageHandlers.progress) this.messageHandlers.progress(data);
} else if (cmd === "finished") {
if (data.data instanceof Blob) {
this.handleBlobResponse(data);
} else {
this.requestBlob();
}
} else if (this.messageHandlers[cmd]) {
this.messageHandlers[cmd](data);
}
}
async loadWeights(networkSize) {
this.validateNetworkSize(networkSize);
if (this.weightsCache.has(networkSize)) {
return this.weightsCache.get(networkSize);
}
const network = this.networks[networkSize];
const response = await fetch(network.weightsUrl);
if (!response.ok)
throw new Error(`Failed to fetch weights: ${response.statusText}`);
const weights = await response.json();
this.weightsCache.set(networkSize, weights);
return weights;
}
static loadImageMetadata(arrayBuffer, mimeType) {
return new Promise((resolve, reject) => {
const blob = new Blob([arrayBuffer], { type: mimeType });
const url = URL.createObjectURL(blob);
const img = new Image();
img.onload = () => {
URL.revokeObjectURL(url);
resolve({ width: img.width, height: img.height });
};
img.onerror = () => {
URL.revokeObjectURL(url);
reject(new Error("Failed to load image"));
};
img.src = url;
});
}
static loadVideoMetadata(file, timeoutMs) {
return new Promise((resolve, reject) => {
const url = URL.createObjectURL(file);
const video = document.createElement("video");
video.src = url;
const timeout = setTimeout(() => reject(new Error("Timeout")), timeoutMs);
video.onloadedmetadata = () => {
clearTimeout(timeout);
URL.revokeObjectURL(url);
resolve({
width: video.videoWidth,
height: video.videoHeight,
duration: video.duration,
});
};
video.onerror = () => {
clearTimeout(timeout);
URL.revokeObjectURL(url);
reject(new Error("Failed to load video"));
};
});
}
createOffscreenCanvas(width, height, scale = 2) {
const factor = typeof scale === "number" && scale > 0 ? scale : 1;
const canvas = document.createElement("canvas");
canvas.width = width * factor;
canvas.height = height * factor;
if (typeof canvas.transferControlToOffscreen !== "function") {
throw new Error("OffscreenCanvas is not supported in this environment");
}
return canvas.transferControlToOffscreen();
}
createBlobPromise(blobType, timeout) {
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
if (this.messageHandlers[blobType]) {
delete this.messageHandlers[blobType];
reject(new Error(`${blobType} processing timeout`));
}
}, timeout);
this.messageHandlers[blobType] = (msg) => {
delete this.messageHandlers[blobType];
clearTimeout(timeoutId);
const blob = msg[blobType] || msg.data;
if (blob instanceof Blob) {
resolve(blob);
} else {
reject(new Error(`Invalid ${blobType} format`));
}
};
});
}
validateNetworkSize(size) {
if (!this.networks[size]) {
throw new Error(
"Invalid networkSize: use one of " +
Object.keys(this.networks).join(", "),
);
}
}
async upscaleImage(imageFile, networkSize, scaleOverride) {
if (!imageFile) throw new Error("Image file is required");
const size = networkSize || this.networkSize;
this.validateNetworkSize(size);
this.processingType = "image";
const arrayBuffer = await imageFile.arrayBuffer();
const mimeType = imageFile.type || "image/jpeg";
const { width, height } = await Upscaler.loadImageMetadata(
arrayBuffer,
mimeType,
);
const weights = await this.loadWeights(size);
const network = this.networks[size];
const scale =
typeof scaleOverride === "number" && scaleOverride > 0
? scaleOverride
: this.imageScale;
const [canvasUp, canvasOrig] = [
this.createOffscreenCanvas(width, height, scale),
this.createOffscreenCanvas(width, height, 1),
];
const blobPromise = this.createBlobPromise(
"imageBlob",
this.timeouts.IMAGE,
);
this.getWorker().postMessage(
{
cmd: "init",
data: {
imageArrayBuffer: arrayBuffer,
imageMimeType: mimeType,
upscaled: canvasUp,
original: canvasOrig,
resolution: { width, height },
network_name: network.name,
weights,
},
},
[canvasUp, canvasOrig],
);
setTimeout(() => {
this.getWorker().postMessage({
cmd: "network",
data: {
name: network.name,
imageArrayBuffer: arrayBuffer,
imageMimeType: mimeType,
weights,
},
});
setTimeout(() => {
this.getWorker().postMessage({
cmd: "getImageBlob",
data: { imageArrayBuffer: arrayBuffer, imageMimeType: mimeType },
});
}, this.delays.NETWORK);
}, this.delays.INIT);
return blobPromise;
}
async upscaleVideo(videoFile, networkSize, scaleOverride) {
if (!videoFile) throw new Error("Video file is required");
const size = networkSize || this.networkSize;
this.validateNetworkSize(size);
if (!Upscaler.isVideoSupported()) {
throw new Error("WebCodecs API not supported. Requires Chrome 94+");
}
this.processingType = "video";
const { width, height, duration } = await Upscaler.loadVideoMetadata(
videoFile,
this.timeouts.METADATA,
);
const weights = await this.loadWeights(size);
const network = this.networks[size];
const scale =
typeof scaleOverride === "number" && scaleOverride > 0
? scaleOverride
: this.videoScale;
const [canvasOut, canvasIn] = [
this.createOffscreenCanvas(width, height, scale),
this.createOffscreenCanvas(width, height, 1),
];
const blobPromise = this.createBlobPromise(
"videoBlob",
this.timeouts.VIDEO,
);
this.getWorker().postMessage(
{
cmd: "process",
file: videoFile,
fileSize: videoFile.size,
duration,
adjustedResolution: {
adjustedInputWidth: width,
adjustedInputHeight: height,
adjustedOutputWidth: width * scale,
adjustedOutputHeight: height * scale,
},
upscaled: canvasOut,
original: canvasIn,
weights,
network_name: network.name,
skipDemuxProgress: true,
},
[canvasOut, canvasIn],
);
return blobPromise;
}
async startRealtimeUpscale(
videoElement,
canvasElement,
networkSize,
scaleOverride,
) {
if (!videoElement || !canvasElement) {
throw new Error("videoElement and canvasElement are required");
}
const size = networkSize || this.networkSize;
this.validateNetworkSize(size);
const network = this.networks[size];
const weights = await this.loadWeights(size);
const scale =
typeof scaleOverride === "number" && scaleOverride > 0
? scaleOverride
: this.videoScale;
const ensureMetadata = () => {
return new Promise((resolve, reject) => {
if (videoElement.readyState >= 1) {
resolve();
return;
}
const onLoaded = () => {
cleanup();
resolve();
};
const onError = () => {
cleanup();
reject(new Error("Failed to load video metadata"));
};
const cleanup = () => {
videoElement.removeEventListener("loadedmetadata", onLoaded);
videoElement.removeEventListener("error", onError);
};
videoElement.addEventListener("loadedmetadata", onLoaded);
videoElement.addEventListener("error", onError);
});
};
await ensureMetadata();
const width = videoElement.videoWidth;
const height = videoElement.videoHeight;
if (!width || !height) {
throw new Error("Invalid video dimensions");
}
const firstInit = !canvasElement._upscalerTransferred;
let offscreen = null;
if (firstInit) {
canvasElement.width = width * scale;
canvasElement.height = height * scale;
if (typeof canvasElement.transferControlToOffscreen !== "function") {
throw new Error("OffscreenCanvas is not supported in this environment");
}
offscreen = canvasElement.transferControlToOffscreen();
canvasElement._upscalerTransferred = true;
}
const captureCanvas = document.createElement("canvas");
captureCanvas.width = width;
captureCanvas.height = height;
const captureCtx = captureCanvas.getContext("2d");
if (this.realtimeLoopId) {
cancelAnimationFrame(this.realtimeLoopId);
this.realtimeLoopId = null;
}
const initPayload = {
cmd: "realtimeInit",
data: {
upscaled: firstInit ? offscreen : null,
resolution: {
width,
height,
scale,
outputWidth: width * scale,
outputHeight: height * scale,
},
network_name: network.name,
weights,
},
};
if (firstInit && offscreen) {
this.getWorker().postMessage(initPayload, [offscreen]);
} else {
this.getWorker().postMessage(initPayload);
}
this.realtimeState = {
running: true,
video: videoElement,
canvas: canvasElement,
scale,
busy: false,
frameIndex: 0,
captureCanvas,
captureCtx,
};
const loop = async () => {
if (!this.realtimeState || !this.realtimeState.running) return;
const state = this.realtimeState;
const v = state.video;
const haveCurrentData = v.HAVE_CURRENT_DATA || 2;
if (
!v.paused &&
!v.ended &&
!state.busy &&
v.readyState >= haveCurrentData &&
v.videoWidth > 0 &&
v.videoHeight > 0
) {
state.busy = true;
try {
const captureCanvas = state.captureCanvas;
const captureCtx = state.captureCtx;
if (!captureCanvas || !captureCtx) {
throw new Error("Missing capture canvas");
}
captureCtx.drawImage(v, 0, 0, captureCanvas.width, captureCanvas.height);
const frame = await createImageBitmap(captureCanvas);
state.frameIndex += 1;
this.getWorker().postMessage(
{
cmd: "realtimeFrame",
frame,
},
[frame],
);
} catch (e) {
console.warn("realtimeFrame error", e);
} finally {
if (this.realtimeState) {
this.realtimeState.busy = false;
}
}
}
this.realtimeLoopId = requestAnimationFrame(loop);
};
this.realtimeLoopId = requestAnimationFrame(loop);
}
stopRealtimeUpscale() {
if (this.realtimeLoopId) {
cancelAnimationFrame(this.realtimeLoopId);
this.realtimeLoopId = null;
}
if (this.realtimeState) {
this.realtimeState.running = false;
this.realtimeState = null;
}
}
async downloadUpscaled(file, networkSize, filename = null) {
const isVideo = file.type.startsWith("video");
const size = networkSize || this.networkSize;
const blob = isVideo
? await this.upscaleVideo(file, size)
: await this.upscaleImage(file, size);
const url = URL.createObjectURL(blob);
const link = document.createElement("a");
link.href = url;
link.download =
filename || `upscaled_${Date.now()}.${isVideo ? "mp4" : "png"}`;
link.click();
URL.revokeObjectURL(url);
return blob;
}
getSupportedNetworks() {
return Object.keys(this.networks);
}
static async getGPUCapability() {
try {
if (navigator.gpu) {
const adapter = await navigator.gpu.requestAdapter();
if (adapter) {
const buffer = adapter.limits.maxStorageBufferBindingSize / 512;
const texture = 8192 * 8192;
return Math.min(buffer, texture);
}
}
const canvas = document.createElement("canvas");
const gl = canvas.getContext("webgl2");
if (gl) return gl.getParameter(gl.MAX_TEXTURE_SIZE) ** 2;
return 8388608;
} catch {
return 8388608;
}
}
static getBackendType() {
try {
if (navigator.gpu) return "webgpu";
if (document.createElement("canvas").getContext("webgl2")) return "webgl";
return "unknown";
} catch {
return "unknown";
}
}
getAppState() {
return {
backend: Upscaler.getBackendType(),
isProcessing: false,
progress: 0,
width: 0,
height: 0,
};
}
onProgress(callback) {
if (typeof callback === "function") {
this.progressCallback = callback;
}
}
dispose() {
this.stopRealtimeUpscale();
if (this.workerInstance) {
this.workerInstance.terminate();
this.workerInstance = null;
}
this.messageHandlers = {};
this.progressCallback = null;
this.processingType = null;
this.weightsCache.clear();
}
}
window.Upscaler = Upscaler;