@viji-dev/core
Version:
Universal execution engine for Viji Creative scenes
1,616 lines (1,614 loc) • 78.6 kB
JavaScript
class VijiCoreError extends Error {
constructor(message, code, context) {
super(message);
this.code = code;
this.context = context;
this.name = "VijiCoreError";
}
}
class IFrameManager {
constructor(hostContainer) {
this.hostContainer = hostContainer;
}
iframe = null;
canvas = null;
offscreenCanvas = null;
isInitialized = false;
scale = 1;
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Phase 7: Interaction event listeners
interactionListeners = /* @__PURE__ */ new Map();
isInteractionEnabled = true;
// Mouse canvas tracking
isMouseInCanvas = false;
// Touch tracking for gesture recognition
activeTouchIds = /* @__PURE__ */ new Set();
/**
* Creates a secure IFrame with proper sandbox attributes
*/
async createSecureIFrame() {
try {
const iframe = document.createElement("iframe");
iframe.sandbox.add("allow-scripts");
iframe.sandbox.add("allow-same-origin");
iframe.style.width = "100%";
iframe.style.height = "100%";
iframe.style.border = "none";
iframe.style.display = "block";
const iframeContent = this.generateIFrameHTML();
const blob = new Blob([iframeContent], { type: "text/html" });
iframe.src = URL.createObjectURL(blob);
this.hostContainer.appendChild(iframe);
await new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new VijiCoreError("IFrame load timeout", "IFRAME_TIMEOUT"));
}, 5e3);
const checkReady = () => {
if (iframe.contentDocument && iframe.contentDocument.readyState === "complete") {
clearTimeout(timeout);
resolve();
}
};
iframe.onload = () => {
if (iframe.contentDocument?.readyState === "complete") {
clearTimeout(timeout);
resolve();
} else {
iframe.contentDocument?.addEventListener("DOMContentLoaded", checkReady);
setTimeout(checkReady, 100);
}
};
iframe.onerror = () => {
clearTimeout(timeout);
reject(new VijiCoreError("IFrame load failed", "IFRAME_LOAD_ERROR"));
};
});
this.iframe = iframe;
return iframe;
} catch (error) {
throw new VijiCoreError(
`Failed to create secure IFrame: ${error}`,
"IFRAME_CREATION_ERROR",
{ error }
);
}
}
/**
* Creates canvas inside the IFrame and returns OffscreenCanvas for WebWorker
*/
async createCanvas() {
if (!this.iframe?.contentWindow) {
throw new VijiCoreError("IFrame not ready for canvas creation", "IFRAME_NOT_READY");
}
try {
const iframeDoc = await this.waitForIFrameDocument();
const canvas = iframeDoc.createElement("canvas");
canvas.id = "viji-canvas";
this.canvas = canvas;
const { width, height } = this.calculateCanvasSize();
canvas.width = width * this.scale;
canvas.height = height * this.scale;
canvas.style.width = "100%";
canvas.style.height = "100%";
canvas.style.display = "block";
const body = iframeDoc.querySelector("body");
if (!body) {
throw new VijiCoreError("IFrame body not found", "IFRAME_BODY_ERROR");
}
body.appendChild(canvas);
this.setupInteractionListeners(canvas, iframeDoc);
const offscreenCanvas = canvas.transferControlToOffscreen();
this.offscreenCanvas = offscreenCanvas;
this.isInitialized = true;
return offscreenCanvas;
} catch (error) {
throw new VijiCoreError(
`Failed to create canvas: ${error}`,
"CANVAS_CREATION_ERROR",
{ error }
);
}
}
/**
* Waits for iframe document to be accessible with retry logic
*/
async waitForIFrameDocument() {
const maxRetries = 15;
const retryDelay = 150;
for (let i = 0; i < maxRetries; i++) {
try {
const iframeDoc = this.iframe?.contentDocument;
if (iframeDoc && (iframeDoc.readyState === "complete" || iframeDoc.readyState === "interactive") && iframeDoc.body && this.iframe?.contentWindow) {
this.debugLog(`IFrame document ready after ${i + 1} attempts`);
return iframeDoc;
}
this.debugLog(`IFrame not ready attempt ${i + 1}/${maxRetries}:`, {
hasDocument: !!iframeDoc,
readyState: iframeDoc?.readyState,
hasBody: !!iframeDoc?.body,
hasWindow: !!this.iframe?.contentWindow
});
await new Promise((resolve) => setTimeout(resolve, retryDelay));
} catch (error) {
this.debugLog(`IFrame access error attempt ${i + 1}/${maxRetries}:`, error);
await new Promise((resolve) => setTimeout(resolve, retryDelay));
}
}
throw new VijiCoreError("Cannot access IFrame document after retries", "IFRAME_ACCESS_ERROR");
}
/**
* Updates canvas resolution setting (scaling handled in worker)
*/
updateScale(scale) {
this.scale = scale;
return this.getEffectiveResolution();
}
/**
* Gets current scale
*/
getScale() {
return this.scale;
}
/**
* Destroys the IFrame and cleans up resources
*/
destroy() {
try {
if (this.iframe) {
if (this.iframe.src.startsWith("blob:")) {
URL.revokeObjectURL(this.iframe.src);
}
this.iframe.remove();
this.iframe = null;
}
this.offscreenCanvas = null;
this.isInitialized = false;
} catch (error) {
console.warn("Error during IFrame cleanup:", error);
}
}
/**
* Checks if IFrame is ready for use
*/
get ready() {
return this.isInitialized && this.iframe !== null && this.offscreenCanvas !== null;
}
/**
* Gets the IFrame element
*/
get element() {
return this.iframe;
}
/**
* Generates the HTML content for the secure IFrame
*/
generateIFrameHTML() {
return `
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Viji Scene Container</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
html, body {
width: 100%;
height: 100%;
overflow: hidden;
background: #000;
}
canvas {
display: block;
width: 100%;
height: 100%;
image-rendering: auto;
}
</style>
</head>
<body>
<!-- Canvas will be created dynamically -->
</body>
</html>
`.trim();
}
/**
* Calculates canvas dimensions based on container size
* Canvas internal size starts as container size, then worker updates it based on resolution
*/
calculateCanvasSize() {
const containerRect = this.hostContainer.getBoundingClientRect();
return {
width: Math.round(containerRect.width),
height: Math.round(containerRect.height)
};
}
/**
* Gets the effective resolution that should be used for rendering
*/
getEffectiveResolution() {
const containerRect = this.hostContainer.getBoundingClientRect();
const containerWidth = Math.round(containerRect.width);
const containerHeight = Math.round(containerRect.height);
const scale = Math.max(0.1, Math.min(1, this.scale));
return {
width: Math.round(containerWidth * scale),
height: Math.round(containerHeight * scale)
};
}
// Phase 7: Interaction Management Methods
/**
* Sets up interaction event listeners on the canvas and document
*/
setupInteractionListeners(canvas, iframeDoc) {
if (!this.isInteractionEnabled) return;
canvas.tabIndex = 0;
canvas.style.outline = "none";
canvas.addEventListener("mousedown", this.handleMouseEvent.bind(this), { passive: false });
canvas.addEventListener("mousemove", this.handleMouseEvent.bind(this), { passive: false });
canvas.addEventListener("mouseup", this.handleMouseEvent.bind(this), { passive: false });
canvas.addEventListener("mouseenter", this.handleMouseEnter.bind(this), { passive: false });
canvas.addEventListener("mouseleave", this.handleMouseLeave.bind(this), { passive: false });
canvas.addEventListener("wheel", this.handleWheelEvent.bind(this), { passive: false });
canvas.addEventListener("contextmenu", (e) => e.preventDefault());
iframeDoc.addEventListener("keydown", this.handleKeyboardEvent.bind(this), { passive: false });
iframeDoc.addEventListener("keyup", this.handleKeyboardEvent.bind(this), { passive: false });
canvas.addEventListener("touchstart", this.handleTouchEvent.bind(this), { passive: false });
canvas.addEventListener("touchmove", this.handleTouchEvent.bind(this), { passive: false });
canvas.addEventListener("touchend", this.handleTouchEvent.bind(this), { passive: false });
canvas.addEventListener("touchcancel", this.handleTouchEvent.bind(this), { passive: false });
canvas.addEventListener("mousedown", () => canvas.focus());
canvas.addEventListener("touchstart", () => canvas.focus());
}
/**
* Handles mouse events and transforms coordinates
*/
handleMouseEvent(event) {
if (!this.canvas || !this.isInteractionEnabled) return;
event.preventDefault();
const rect = this.canvas.getBoundingClientRect();
const x = (event.clientX - rect.left) * (this.canvas.width / rect.width);
const y = (event.clientY - rect.top) * (this.canvas.height / rect.height);
const deltaX = event.movementX || 0;
const deltaY = event.movementY || 0;
const data = {
x,
y,
buttons: event.buttons,
deltaX,
deltaY,
wheelDeltaX: 0,
wheelDeltaY: 0,
isInCanvas: this.isMouseInCanvas,
timestamp: performance.now()
};
this.emitInteractionEvent("mouse-update", data);
}
/**
* Handles mouse enter events
*/
handleMouseEnter(event) {
if (!this.isInteractionEnabled) return;
this.isMouseInCanvas = true;
this.handleMouseEvent(event);
}
/**
* Handles mouse leave events
*/
handleMouseLeave(event) {
if (!this.isInteractionEnabled) return;
this.isMouseInCanvas = false;
this.handleMouseEvent(event);
}
/**
* Handles wheel events
*/
handleWheelEvent(event) {
if (!this.canvas || !this.isInteractionEnabled) return;
event.preventDefault();
const rect = this.canvas.getBoundingClientRect();
const x = (event.clientX - rect.left) * (this.canvas.width / rect.width);
const y = (event.clientY - rect.top) * (this.canvas.height / rect.height);
const data = {
x,
y,
buttons: event.buttons,
deltaX: 0,
deltaY: 0,
wheelDeltaX: event.deltaX,
wheelDeltaY: event.deltaY,
timestamp: performance.now()
};
this.emitInteractionEvent("mouse-update", data);
}
/**
* Handles keyboard events
*/
handleKeyboardEvent(event) {
if (!this.isInteractionEnabled) return;
const allowedKeys = ["Tab", "F1", "F2", "F3", "F4", "F5", "F11", "F12"];
if (!allowedKeys.includes(event.key)) {
event.preventDefault();
}
const data = {
type: event.type,
key: event.key,
code: event.code,
shiftKey: event.shiftKey,
ctrlKey: event.ctrlKey,
altKey: event.altKey,
metaKey: event.metaKey,
timestamp: performance.now()
};
this.emitInteractionEvent("keyboard-update", data);
}
/**
* Handles touch events and tracks multi-touch
*/
handleTouchEvent(event) {
if (!this.canvas || !this.isInteractionEnabled) return;
event.preventDefault();
const rect = this.canvas.getBoundingClientRect();
const scaleX = this.canvas.width / rect.width;
const scaleY = this.canvas.height / rect.height;
const touches = Array.from(event.touches).map((touch) => ({
identifier: touch.identifier,
clientX: (touch.clientX - rect.left) * scaleX,
clientY: (touch.clientY - rect.top) * scaleY,
pressure: touch.pressure || 0,
radiusX: touch.radiusX || 10,
radiusY: touch.radiusY || 10,
rotationAngle: touch.rotationAngle || 0,
force: touch.force || touch.pressure || 0
}));
if (event.type === "touchstart") {
for (const touch of event.changedTouches) {
this.activeTouchIds.add(touch.identifier);
}
} else if (event.type === "touchend" || event.type === "touchcancel") {
for (const touch of event.changedTouches) {
this.activeTouchIds.delete(touch.identifier);
}
}
const data = {
type: event.type,
touches,
timestamp: performance.now()
};
this.emitInteractionEvent("touch-update", data);
}
/**
* Emits an interaction event to registered listeners
*/
emitInteractionEvent(eventType, data) {
const listener = this.interactionListeners.get(eventType);
if (listener) {
listener(data);
}
}
/**
* Registers an interaction event listener
*/
onInteractionEvent(eventType, listener) {
this.interactionListeners.set(eventType, listener);
}
/**
* Removes an interaction event listener
*/
offInteractionEvent(eventType) {
this.interactionListeners.delete(eventType);
}
/**
* Enables or disables interaction event capture
*/
setInteractionEnabled(enabled) {
this.isInteractionEnabled = enabled;
}
/**
* Gets the canvas element (for coordinate calculations)
*/
getCanvas() {
return this.canvas;
}
}
function WorkerWrapper(options) {
return new Worker(
"" + new URL("assets/viji.worker-BKsgIT1d.js", import.meta.url).href,
{
name: options?.name
}
);
}
class WorkerManager {
constructor(sceneCode, offscreenCanvas) {
this.sceneCode = sceneCode;
this.offscreenCanvas = offscreenCanvas;
}
worker = null;
messageId = 0;
pendingMessages = /* @__PURE__ */ new Map();
messageHandlers = /* @__PURE__ */ new Map();
isInitialized = false;
/**
* Creates and initializes the WebWorker with artist code
*/
async createWorker() {
try {
this.worker = new WorkerWrapper();
this.setupMessageHandling();
this.postMessage("set-scene-code", { sceneCode: this.sceneCode });
await this.initializeWorker();
this.isInitialized = true;
return this.worker;
} catch (error) {
throw new VijiCoreError(
`Failed to create worker: ${error}`,
"WORKER_CREATION_ERROR",
{ error }
);
}
}
/**
* Sends a message to the worker and returns a promise for the response
*/
async sendMessage(type, data, timeout = 5e3) {
if (!this.worker) {
throw new VijiCoreError("Worker not initialized", "WORKER_NOT_READY");
}
const id = `msg_${++this.messageId}`;
const message = {
type,
id,
timestamp: Date.now(),
data
};
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
this.pendingMessages.delete(id);
reject(new VijiCoreError(`Message timeout: ${type}`, "MESSAGE_TIMEOUT"));
}, timeout);
this.pendingMessages.set(id, {
resolve,
reject,
timeout: timeoutId
});
this.worker.postMessage(message);
});
}
postMessage(type, data, transfer) {
if (!this.worker) {
console.warn("Attempted to post message to uninitialized worker");
return;
}
const message = {
type,
id: `fire_${++this.messageId}`,
timestamp: Date.now(),
data
};
if (transfer && transfer.length > 0) {
this.worker.postMessage(message, transfer);
} else {
this.worker.postMessage(message);
}
}
/**
* Registers a handler for worker messages
*/
onMessage(type, handler) {
this.messageHandlers.set(type, handler);
}
/**
* Removes a message handler
*/
offMessage(type) {
this.messageHandlers.delete(type);
}
/**
* Terminates the worker and cleans up resources
*/
destroy() {
try {
this.pendingMessages.forEach(({ timeout, reject }) => {
clearTimeout(timeout);
reject(new VijiCoreError("Worker destroyed", "WORKER_DESTROYED"));
});
this.pendingMessages.clear();
this.messageHandlers.clear();
if (this.worker) {
this.worker.terminate();
this.worker = null;
}
this.isInitialized = false;
} catch (error) {
console.warn("Error during worker cleanup:", error);
}
}
/**
* Checks if worker is ready for use
*/
get ready() {
return this.isInitialized && this.worker !== null;
}
/**
* Gets the worker instance
*/
get instance() {
return this.worker;
}
/**
* Sets up message handling for worker communication
*/
setupMessageHandling() {
if (!this.worker) return;
this.worker.onmessage = (event) => {
const message = event.data;
if (this.pendingMessages.has(message.id)) {
const pending = this.pendingMessages.get(message.id);
clearTimeout(pending.timeout);
this.pendingMessages.delete(message.id);
if (message.type === "error") {
pending.reject(new VijiCoreError(
message.data?.message || "Worker error",
message.data?.code || "WORKER_ERROR",
message.data
));
} else {
pending.resolve(message.data);
}
return;
}
const handler = this.messageHandlers.get(message.type);
if (handler) {
try {
handler(message.data);
} catch (error) {
console.error(`Error in message handler for ${message.type}:`, error);
}
}
};
this.worker.onerror = (error) => {
console.error("Worker error:", error);
this.pendingMessages.forEach(({ timeout, reject }) => {
clearTimeout(timeout);
reject(new VijiCoreError("Worker error", "WORKER_ERROR", error));
});
this.pendingMessages.clear();
};
}
/**
* Initializes the worker with canvas and basic setup
*/
async initializeWorker() {
if (!this.worker) {
throw new VijiCoreError("Worker not created", "WORKER_NOT_CREATED");
}
const id = `msg_${++this.messageId}`;
const message = {
type: "init",
id,
timestamp: Date.now(),
data: {
canvas: this.offscreenCanvas
}
};
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
this.pendingMessages.delete(id);
reject(new VijiCoreError("Canvas transfer timeout", "CANVAS_TRANSFER_TIMEOUT"));
}, 1e4);
this.pendingMessages.set(id, {
resolve,
reject,
timeout: timeoutId
});
this.worker.postMessage(message, [this.offscreenCanvas]);
});
}
}
class InteractionManager {
// Mouse state
mouseState = {
x: 0,
y: 0,
isInCanvas: false,
isPressed: false,
leftButton: false,
rightButton: false,
middleButton: false,
velocity: { x: 0, y: 0 },
deltaX: 0,
deltaY: 0,
wheelDelta: 0,
wheelX: 0,
wheelY: 0,
wasPressed: false,
wasReleased: false,
wasMoved: false
};
// Mouse velocity tracking
mouseVelocityHistory = [];
// Keyboard state
keyboardState = {
isPressed: (key) => this.activeKeys.has(key.toLowerCase()),
wasPressed: (key) => this.pressedThisFrame.has(key.toLowerCase()),
wasReleased: (key) => this.releasedThisFrame.has(key.toLowerCase()),
activeKeys: /* @__PURE__ */ new Set(),
pressedThisFrame: /* @__PURE__ */ new Set(),
releasedThisFrame: /* @__PURE__ */ new Set(),
lastKeyPressed: "",
lastKeyReleased: "",
shift: false,
ctrl: false,
alt: false,
meta: false
};
activeKeys = /* @__PURE__ */ new Set();
pressedThisFrame = /* @__PURE__ */ new Set();
releasedThisFrame = /* @__PURE__ */ new Set();
// Touch state
touchState = {
points: [],
count: 0,
started: [],
moved: [],
ended: [],
primary: null,
gestures: {
isPinching: false,
isRotating: false,
isPanning: false,
isTapping: false,
pinchScale: 1,
pinchDelta: 0,
rotationAngle: 0,
rotationDelta: 0,
panDelta: { x: 0, y: 0 },
tapCount: 0,
lastTapTime: 0,
tapPosition: null
}
};
activeTouches = /* @__PURE__ */ new Map();
gestureState = {
initialDistance: 0,
initialAngle: 0,
lastPinchScale: 1,
lastRotationAngle: 0,
panStartPosition: { x: 0, y: 0 },
tapStartTime: 0,
tapCount: 0,
lastTapTime: 0
};
constructor() {
}
/**
* Processes mouse update from the host
*/
updateMouse(data) {
const canvasX = data.x;
const canvasY = data.y;
const deltaX = canvasX - this.mouseState.x;
const deltaY = canvasY - this.mouseState.y;
this.updateMouseVelocity(deltaX, deltaY, data.timestamp);
const prevPressed = this.mouseState.isPressed;
const currentPressed = data.buttons > 0;
this.mouseState.wasPressed = !prevPressed && currentPressed;
this.mouseState.wasReleased = prevPressed && !currentPressed;
this.mouseState.wasMoved = deltaX !== 0 || deltaY !== 0;
this.mouseState.x = canvasX;
this.mouseState.y = canvasY;
this.mouseState.deltaX = deltaX;
this.mouseState.deltaY = deltaY;
this.mouseState.isPressed = currentPressed;
this.mouseState.leftButton = (data.buttons & 1) !== 0;
this.mouseState.rightButton = (data.buttons & 2) !== 0;
this.mouseState.middleButton = (data.buttons & 4) !== 0;
this.mouseState.isInCanvas = data.isInCanvas !== void 0 ? data.isInCanvas : true;
this.mouseState.wheelDelta = data.wheelDeltaY;
this.mouseState.wheelX = data.wheelDeltaX;
this.mouseState.wheelY = data.wheelDeltaY;
}
/**
* Updates mouse velocity with smoothing
*/
updateMouseVelocity(deltaX, deltaY, timestamp) {
this.mouseVelocityHistory.push({ x: deltaX, y: deltaY, time: timestamp });
const cutoff = timestamp - 100;
this.mouseVelocityHistory = this.mouseVelocityHistory.filter((sample) => sample.time > cutoff);
if (this.mouseVelocityHistory.length > 1) {
const recent = this.mouseVelocityHistory.slice(-5);
const avgX = recent.reduce((sum, s) => sum + s.x, 0) / recent.length;
const avgY = recent.reduce((sum, s) => sum + s.y, 0) / recent.length;
this.mouseState.velocity.x = avgX;
this.mouseState.velocity.y = avgY;
} else {
this.mouseState.velocity.x = deltaX;
this.mouseState.velocity.y = deltaY;
}
}
/**
* Processes keyboard update from the host
*/
updateKeyboard(data) {
const key = data.key.toLowerCase();
if (data.type === "keydown") {
if (!this.activeKeys.has(key)) {
this.activeKeys.add(key);
this.pressedThisFrame.add(key);
this.keyboardState.lastKeyPressed = data.key;
}
} else if (data.type === "keyup") {
this.activeKeys.delete(key);
this.releasedThisFrame.add(key);
this.keyboardState.lastKeyReleased = data.key;
}
this.keyboardState.shift = data.shiftKey;
this.keyboardState.ctrl = data.ctrlKey;
this.keyboardState.alt = data.altKey;
this.keyboardState.meta = data.metaKey;
}
/**
* Processes touch update from the host
*/
updateTouch(data) {
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
if (data.type === "touchstart") {
this.processTouchStart(data.touches, data.timestamp);
} else if (data.type === "touchmove") {
this.processTouchMove(data.touches, data.timestamp);
} else if (data.type === "touchend" || data.type === "touchcancel") {
this.processTouchEnd(data.touches, data.timestamp);
}
this.touchState.points = Array.from(this.activeTouches.values());
this.touchState.count = this.touchState.points.length;
this.touchState.primary = this.touchState.points[0] || null;
this.updateGestures();
}
/**
* Processes touch start events
*/
processTouchStart(touches, timestamp) {
for (const touch of touches) {
const touchPoint = this.createTouchPoint(touch, timestamp, true);
this.activeTouches.set(touch.identifier, touchPoint);
this.touchState.started.push(touchPoint);
}
if (this.touchState.count === 1) {
this.gestureState.tapStartTime = timestamp;
const touch = this.touchState.points[0];
this.touchState.gestures.tapPosition = { x: touch.x, y: touch.y };
}
}
/**
* Processes touch move events
*/
processTouchMove(touches, timestamp) {
for (const touch of touches) {
const existing = this.activeTouches.get(touch.identifier);
if (existing) {
const updated = this.createTouchPoint(touch, timestamp, false, existing);
this.activeTouches.set(touch.identifier, updated);
this.touchState.moved.push(updated);
}
}
}
/**
* Processes touch end events
*/
processTouchEnd(touches, timestamp) {
for (const touch of touches) {
const existing = this.activeTouches.get(touch.identifier);
if (existing) {
const ended = { ...existing, isEnding: true, isActive: false };
this.touchState.ended.push(ended);
this.activeTouches.delete(touch.identifier);
}
}
if (this.touchState.count === 0 && this.gestureState.tapStartTime > 0) {
const tapDuration = timestamp - this.gestureState.tapStartTime;
if (tapDuration < 300) {
this.handleTap(timestamp);
}
this.gestureState.tapStartTime = 0;
}
}
/**
* Creates a touch point from raw touch data
*/
createTouchPoint(touch, timestamp, isNew, previous) {
const x = touch.clientX;
const y = touch.clientY;
const deltaX = previous ? x - previous.x : 0;
const deltaY = previous ? y - previous.y : 0;
const timeDelta = previous ? timestamp - previous.timestamp : 16;
const velocityX = timeDelta > 0 ? deltaX / timeDelta * 1e3 : 0;
const velocityY = timeDelta > 0 ? deltaY / timeDelta * 1e3 : 0;
return {
id: touch.identifier,
x,
y,
pressure: touch.pressure || 0,
radius: Math.max(touch.radiusX || 0, touch.radiusY || 0),
radiusX: touch.radiusX || 0,
radiusY: touch.radiusY || 0,
rotationAngle: touch.rotationAngle || 0,
force: touch.force || touch.pressure || 0,
deltaX,
deltaY,
velocity: { x: velocityX, y: velocityY },
isNew,
isActive: true,
isEnding: false
};
}
/**
* Updates gesture recognition
*/
updateGestures() {
const touches = this.touchState.points;
const gestures = this.touchState.gestures;
if (touches.length === 2) {
const touch1 = touches[0];
const touch2 = touches[1];
const distance = Math.sqrt(
Math.pow(touch2.x - touch1.x, 2) + Math.pow(touch2.y - touch1.y, 2)
);
const angle = Math.atan2(touch2.y - touch1.y, touch2.x - touch1.x);
if (this.gestureState.initialDistance === 0) {
this.gestureState.initialDistance = distance;
this.gestureState.initialAngle = angle;
this.gestureState.lastPinchScale = 1;
this.gestureState.lastRotationAngle = 0;
}
const scale = distance / this.gestureState.initialDistance;
const scaleDelta = scale - this.gestureState.lastPinchScale;
gestures.isPinching = Math.abs(scaleDelta) > 0.01;
gestures.pinchScale = scale;
gestures.pinchDelta = scaleDelta;
this.gestureState.lastPinchScale = scale;
const rotationAngle = angle - this.gestureState.initialAngle;
const rotationDelta = rotationAngle - this.gestureState.lastRotationAngle;
gestures.isRotating = Math.abs(rotationDelta) > 0.02;
gestures.rotationAngle = rotationAngle;
gestures.rotationDelta = rotationDelta;
this.gestureState.lastRotationAngle = rotationAngle;
} else {
this.gestureState.initialDistance = 0;
gestures.isPinching = false;
gestures.isRotating = false;
gestures.pinchDelta = 0;
gestures.rotationDelta = 0;
}
if (touches.length > 0) {
const primaryTouch = touches[0];
if (this.gestureState.panStartPosition.x === 0) {
this.gestureState.panStartPosition = { x: primaryTouch.x, y: primaryTouch.y };
}
const panDeltaX = primaryTouch.x - this.gestureState.panStartPosition.x;
const panDeltaY = primaryTouch.y - this.gestureState.panStartPosition.y;
const panDistance = Math.sqrt(panDeltaX * panDeltaX + panDeltaY * panDeltaY);
gestures.isPanning = panDistance > 10;
gestures.panDelta = { x: panDeltaX, y: panDeltaY };
} else {
this.gestureState.panStartPosition = { x: 0, y: 0 };
gestures.isPanning = false;
gestures.panDelta = { x: 0, y: 0 };
}
}
/**
* Handles tap gesture detection
*/
handleTap(timestamp) {
const timeSinceLastTap = timestamp - this.gestureState.lastTapTime;
if (timeSinceLastTap < 300) {
this.gestureState.tapCount++;
} else {
this.gestureState.tapCount = 1;
}
this.touchState.gestures.tapCount = this.gestureState.tapCount;
this.touchState.gestures.lastTapTime = timestamp;
this.touchState.gestures.isTapping = true;
this.gestureState.lastTapTime = timestamp;
}
/**
* Called at the start of each frame to reset frame-based events
*/
frameStart() {
this.mouseState.wasPressed = false;
this.mouseState.wasReleased = false;
this.mouseState.wasMoved = false;
this.mouseState.wheelDelta = 0;
this.mouseState.wheelX = 0;
this.mouseState.wheelY = 0;
this.pressedThisFrame.clear();
this.releasedThisFrame.clear();
this.touchState.gestures.isTapping = false;
this.touchState.gestures.pinchDelta = 0;
this.touchState.gestures.rotationDelta = 0;
}
/**
* Get current mouse state (read-only)
*/
getMouseState() {
return this.mouseState;
}
/**
* Get current keyboard state (read-only)
*/
getKeyboardState() {
return this.keyboardState;
}
/**
* Get current touch state (read-only)
*/
getTouchState() {
return this.touchState;
}
/**
* Cleanup resources
*/
destroy() {
this.mouseVelocityHistory.length = 0;
this.activeTouches.clear();
this.activeKeys.clear();
this.pressedThisFrame.clear();
this.releasedThisFrame.clear();
}
}
class AudioSystem {
// Audio context and analysis nodes
audioContext = null;
analyser = null;
mediaStreamSource = null;
currentStream = null;
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Analysis configuration (good balance, leaning towards quality)
fftSize = 2048;
// Good balance for quality vs performance
smoothingTimeConstant = 0.8;
// Smooth but responsive
// Analysis data arrays
frequencyData = null;
timeDomainData = null;
// Audio analysis state (host-side state)
audioState = {
isConnected: false,
volume: {
rms: 0,
peak: 0
},
bands: {
bass: 0,
mid: 0,
treble: 0,
subBass: 0,
lowMid: 0,
highMid: 0,
presence: 0,
brilliance: 0
}
};
// Analysis loop
analysisLoopId = null;
isAnalysisRunning = false;
// Callback to send results to worker
sendAnalysisResults = null;
constructor(sendAnalysisResultsCallback) {
this.handleAudioStreamUpdate = this.handleAudioStreamUpdate.bind(this);
this.performAnalysis = this.performAnalysis.bind(this);
this.sendAnalysisResults = sendAnalysisResultsCallback || null;
}
/**
* Get the current audio analysis state (for host-side usage)
*/
getAudioState() {
return { ...this.audioState };
}
/**
* Handle audio stream update (called from VijiCore)
*/
handleAudioStreamUpdate(data) {
try {
if (data.audioStream) {
this.setAudioStream(data.audioStream);
} else {
this.disconnectAudioStream();
}
if (data.analysisConfig) {
this.updateAnalysisConfig(data.analysisConfig);
}
} catch (error) {
console.error("Error handling audio stream update:", error);
this.audioState.isConnected = false;
this.sendAnalysisResultsToWorker();
}
}
/**
* Set the audio stream for analysis
*/
async setAudioStream(audioStream) {
this.disconnectAudioStream();
const audioTracks = audioStream.getAudioTracks();
if (audioTracks.length === 0) {
console.warn("No audio tracks in provided stream");
this.audioState.isConnected = false;
this.sendAnalysisResultsToWorker();
return;
}
try {
if (!this.audioContext) {
this.audioContext = new AudioContext();
if (this.audioContext.state === "suspended") {
await this.audioContext.resume();
}
}
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = this.fftSize;
this.analyser.smoothingTimeConstant = this.smoothingTimeConstant;
this.mediaStreamSource = this.audioContext.createMediaStreamSource(audioStream);
this.mediaStreamSource.connect(this.analyser);
const bufferLength = this.analyser.frequencyBinCount;
this.frequencyData = new Uint8Array(bufferLength);
this.timeDomainData = new Uint8Array(bufferLength);
this.currentStream = audioStream;
this.audioState.isConnected = true;
this.startAnalysisLoop();
this.debugLog("Audio stream connected successfully (host-side)", {
sampleRate: this.audioContext.sampleRate,
fftSize: this.fftSize,
bufferLength
});
} catch (error) {
console.error("Failed to set up audio analysis:", error);
this.audioState.isConnected = false;
this.disconnectAudioStream();
}
this.sendAnalysisResultsToWorker();
}
/**
* Disconnect current audio stream and clean up resources
*/
disconnectAudioStream() {
this.stopAnalysisLoop();
if (this.mediaStreamSource) {
this.mediaStreamSource.disconnect();
this.mediaStreamSource = null;
}
if (this.analyser) {
this.analyser.disconnect();
this.analyser = null;
}
this.frequencyData = null;
this.timeDomainData = null;
this.currentStream = null;
this.audioState.isConnected = false;
this.resetAudioValues();
this.sendAnalysisResultsToWorker();
this.debugLog("Audio stream disconnected (host-side)");
}
/**
* Update analysis configuration
*/
updateAnalysisConfig(config) {
let needsReconnect = false;
if (config.fftSize && config.fftSize !== this.fftSize) {
this.fftSize = config.fftSize;
needsReconnect = true;
}
if (config.smoothing !== void 0) {
this.smoothingTimeConstant = config.smoothing;
if (this.analyser) {
this.analyser.smoothingTimeConstant = this.smoothingTimeConstant;
}
}
if (needsReconnect && this.currentStream) {
const stream = this.currentStream;
this.setAudioStream(stream);
}
}
/**
* Start the audio analysis loop
*/
startAnalysisLoop() {
if (this.isAnalysisRunning) return;
this.isAnalysisRunning = true;
this.performAnalysis();
}
/**
* Stop the audio analysis loop
*/
stopAnalysisLoop() {
this.isAnalysisRunning = false;
if (this.analysisLoopId !== null) {
cancelAnimationFrame(this.analysisLoopId);
this.analysisLoopId = null;
}
}
/**
* Perform audio analysis (called every frame)
*/
performAnalysis() {
if (!this.isAnalysisRunning || !this.analyser || !this.frequencyData || !this.timeDomainData) {
return;
}
this.analyser.getByteFrequencyData(this.frequencyData);
this.analyser.getByteTimeDomainData(this.timeDomainData);
this.calculateVolumeMetrics();
this.calculateFrequencyBands();
this.sendAnalysisResultsToWorker();
this.analysisLoopId = requestAnimationFrame(() => this.performAnalysis());
}
/**
* Calculate RMS and peak volume from time domain data
*/
calculateVolumeMetrics() {
if (!this.timeDomainData) return;
let rmsSum = 0;
let peak = 0;
for (let i = 0; i < this.timeDomainData.length; i++) {
const sample = (this.timeDomainData[i] - 128) / 128;
rmsSum += sample * sample;
const absValue = Math.abs(sample);
if (absValue > peak) {
peak = absValue;
}
}
const rms = Math.sqrt(rmsSum / this.timeDomainData.length);
this.audioState.volume.rms = rms;
this.audioState.volume.peak = peak;
}
/**
* Calculate frequency band values from frequency data
*/
calculateFrequencyBands() {
if (!this.frequencyData || !this.audioContext) return;
const nyquist = this.audioContext.sampleRate / 2;
const binCount = this.frequencyData.length;
const bands = {
subBass: { min: 20, max: 60 },
// Sub-bass
bass: { min: 60, max: 250 },
// Bass
lowMid: { min: 250, max: 500 },
// Low midrange
mid: { min: 500, max: 2e3 },
// Midrange
highMid: { min: 2e3, max: 4e3 },
// High midrange
presence: { min: 4e3, max: 6e3 },
// Presence
brilliance: { min: 6e3, max: 2e4 },
// Brilliance
treble: { min: 2e3, max: 2e4 }
// Treble (combined high frequencies)
};
for (const [bandName, range] of Object.entries(bands)) {
const startBin = Math.floor(range.min / nyquist * binCount);
const endBin = Math.min(Math.floor(range.max / nyquist * binCount), binCount - 1);
let sum = 0;
let count = 0;
for (let i = startBin; i <= endBin; i++) {
sum += this.frequencyData[i];
count++;
}
const average = count > 0 ? sum / count : 0;
this.audioState.bands[bandName] = average / 255;
}
}
/**
* Send analysis results to worker
*/
sendAnalysisResultsToWorker() {
if (this.sendAnalysisResults) {
const frequencyData = this.frequencyData ? new Uint8Array(this.frequencyData) : new Uint8Array(0);
this.sendAnalysisResults({
type: "audio-analysis-update",
data: {
...this.audioState,
frequencyData,
// For getFrequencyData() access
timestamp: performance.now()
}
});
}
}
/**
* Reset audio values to defaults
*/
resetAudioValues() {
this.audioState.volume.rms = 0;
this.audioState.volume.peak = 0;
for (const band in this.audioState.bands) {
this.audioState.bands[band] = 0;
}
}
/**
* Reset all audio state (called when destroying)
*/
resetAudioState() {
this.disconnectAudioStream();
if (this.audioContext && this.audioContext.state !== "closed") {
this.audioContext.close();
this.audioContext = null;
}
this.resetAudioValues();
}
/**
* Get current analysis configuration
*/
getAnalysisConfig() {
return {
fftSize: this.fftSize,
smoothing: this.smoothingTimeConstant
};
}
}
class VideoCoordinator {
// Video elements for MediaStream processing
videoElement = null;
canvas = null;
ctx = null;
// Note: currentStream was removed as it was unused
// Transfer coordination
transferLoopId = null;
isTransferRunning = false;
lastTransferTime = 0;
transferInterval = 1e3 / 30;
// Transfer at 30 FPS
// Video state (lightweight - main state is in worker)
coordinatorState = {
isConnected: false,
sourceType: "",
frameWidth: 0,
frameHeight: 0
};
// Track if OffscreenCanvas has been sent to worker
hasTransferredCanvas = false;
// Callback to send data to worker
sendToWorker = null;
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
constructor(sendToWorkerCallback) {
this.handleVideoStreamUpdate = this.handleVideoStreamUpdate.bind(this);
this.transferVideoFrame = this.transferVideoFrame.bind(this);
this.sendToWorker = sendToWorkerCallback || null;
}
/**
* Get the current video coordinator state (for host-side usage)
*/
getCoordinatorState() {
return { ...this.coordinatorState };
}
/**
* Handle video stream update (called from VijiCore)
*/
handleVideoStreamUpdate(data) {
try {
if (data.videoStream) {
this.setVideoStream(data.videoStream);
} else {
this.disconnectVideoStream();
}
if (data.targetFrameRate || data.cvConfig) {
this.sendConfigurationToWorker({
...data.targetFrameRate && { targetFrameRate: data.targetFrameRate },
...data.cvConfig && { cvConfig: data.cvConfig },
timestamp: data.timestamp
});
}
} catch (error) {
console.error("Error handling video stream update:", error);
this.coordinatorState.isConnected = false;
this.sendDisconnectionToWorker();
}
}
/**
* Set the video stream for processing
*/
async setVideoStream(videoStream) {
this.disconnectVideoStream();
const videoTracks = videoStream.getVideoTracks();
if (videoTracks.length === 0) {
console.warn("No video tracks in provided stream");
this.coordinatorState.isConnected = false;
this.sendDisconnectionToWorker();
return;
}
try {
this.videoElement = document.createElement("video");
this.videoElement.autoplay = true;
this.videoElement.muted = true;
this.videoElement.playsInline = true;
this.canvas = document.createElement("canvas");
this.ctx = this.canvas.getContext("2d");
if (!this.ctx) {
throw new Error("Failed to get 2D context from canvas");
}
this.videoElement.srcObject = videoStream;
await new Promise((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error("Video metadata load timeout")), 5e3);
this.videoElement.addEventListener("loadedmetadata", async () => {
clearTimeout(timeout);
try {
await this.videoElement.play();
this.debugLog("✅ Video element is now playing:", {
videoWidth: this.videoElement.videoWidth,
videoHeight: this.videoElement.videoHeight,
readyState: this.videoElement.readyState,
paused: this.videoElement.paused
});
resolve();
} catch (playError) {
console.error("🔴 Failed to start video playback:", playError);
reject(playError);
}
}, { once: true });
this.videoElement.addEventListener("error", (e) => {
clearTimeout(timeout);
reject(new Error(`Video load error: ${e}`));
}, { once: true });
});
this.canvas.width = this.videoElement.videoWidth;
this.canvas.height = this.videoElement.videoHeight;
this.coordinatorState.isConnected = true;
this.coordinatorState.frameWidth = this.videoElement.videoWidth;
this.coordinatorState.frameHeight = this.videoElement.videoHeight;
this.coordinatorState.sourceType = "MediaStream";
await this.transferOffscreenCanvasToWorker(this.videoElement.videoWidth, this.videoElement.videoHeight);
this.startTransferLoop();
this.debugLog("Video stream connected successfully (host-side coordinator)", {
width: this.videoElement.videoWidth,
height: this.videoElement.videoHeight
});
} catch (error) {
console.error("Failed to set up video coordination:", error);
this.coordinatorState.isConnected = false;
this.disconnectVideoStream();
}
}
/**
* ✅ CORRECT: Transfer OffscreenCanvas to worker BEFORE getting context
*/
async transferOffscreenCanvasToWorker(width, height) {
if (this.hasTransferredCanvas) {
this.sendConfigurationToWorker({
width,
height,
timestamp: performance.now()
});
return;
}
try {
const offscreenCanvas = new OffscreenCanvas(width, height);
if (this.sendToWorker) {
this.sendToWorker({
type: "video-canvas-setup",
data: {
offscreenCanvas,
width,
height,
timestamp: performance.now()
}
}, [offscreenCanvas]);
this.hasTransferredCanvas = true;
this.debugLog("✅ OffscreenCanvas transferred to worker (correct approach)", {
width,
height
});
}
} catch (error) {
console.error("Failed to transfer OffscreenCanvas to worker:", error);
throw error;
}
}
/**
* Disconnect current video stream and clean up resources
*/
disconnectVideoStream() {
this.stopTransferLoop();
if (this.videoElement) {
this.videoElement.pause();
this.videoElement.srcObject = null;
this.videoElement = null;
}
if (this.canvas) {
this.canvas = null;
this.ctx = null;
}
this.coordinatorState.isConnected = false;
this.coordinatorState.frameWidth = 0;
this.coordinatorState.frameHeight = 0;
this.coordinatorState.sourceType = "";
this.hasTransferredCanvas = false;
this.sendDisconnectionToWorker();
this.debugLog("Video stream disconnected (host-side coordinator)");
}
/**
* Start the video frame transfer loop
*/
startTransferLoop() {
if (this.isTransferRunning) return;
this.isTransferRunning = true;
this.lastTransferTime = performance.now();
this.transferVideoFrame();
}
/**
* Stop the video frame transfer loop
*/
stopTransferLoop() {
this.isTransferRunning = false;
if (this.transferLoopId !== null) {
cancelAnimationFrame(this.transferLoopId);
this.transferLoopId = null;
}
}
/**
* Transfer video frame to worker using ImageBitmap (for worker to draw on its OffscreenCanvas)
*/
transferVideoFrame() {
if (!this.isTransferRunning || !this.videoElement || !this.canvas || !this.ctx) {
if (!this.isTransferRunning) {
this.debugLog("🔴 Transfer loop stopped");
}
return;
}
const currentTime = performance.now();
const deltaTime = currentTime - this.lastTransferTime;
if (deltaTime >= this.transferInterval) {
if (Math.random() < 0.01) {
this.debugLog(`🔄 Transfer loop tick: ${deltaTime.toFixed(1)}ms since last frame`);
}
this.transferFrameToWorker().catch((error) => {
console.error("🔴 Error transferring video frame to worker:", error);
});
this.lastTransferTime = currentTime;
}
this.transferLoopId = requestAnimationFrame(() => this.transferVideoFrame());
}
/**
* Async frame transfer using ImageBitmap (for worker to draw)
*/
async transferFrameToWorker() {
if (!this.videoElement || !this.canvas || !this.ctx) {
console.warn("🔴 Frame transfer called but missing elements:", {
hasVideo: !!this.videoElement,
hasCanvas: !!this.canvas,
hasCtx: !!this.ctx
});
return;
}
try {
if (this.videoElement.readyState < 2) {
console.warn("🔴 Video not ready for frame capture, readyState:", this.videoElement.readyState);
return;
}
if (this.videoElement.videoWidth === 0 || this.videoElement.videoHeight === 0) {
console.warn("🔴 Video has no dimensions:", {
width: this.videoElement.videoWidth,
height: this.videoElement.videoHeight
});
return;
}
this.ctx.drawImage(this.videoElement, 0, 0, this.canvas.width, this.canvas.height);
const imageBitmap = await createImageBitmap(this.canvas);
if (Math.random() < 0.01) {
this.debugLog("✅ Frame captured and ImageBitmap created:", {
videoDimensions: `${this.videoElement.videoWidth}x${this.videoElement.videoHeight}`,
canvasDimensions: `${this.canvas.width}x${this.canvas.height}`,
bitmapDimensions: `${imageBitmap.width}x${imageBitmap.height}`
});
}
this.sendFrameToWorker(imageBitmap);
} catch (error) {
console.error("🔴 Failed to create ImageBitmap:", error);
}
}
/**
* Send ImageBitmap frame to worker (for worker to draw on its OffscreenCanvas)
*/
sendFrameToWorker(imageBitmap) {
if (this.sendToWorker) {
this.sendToWorker({
type: "video-frame-update",
data: {
imageBitmap,
timestamp: performance.now()
}
}, [imageBitmap]);
}
}
/**
* Send configuration updates to worker
*/
sendConfigurationToWorker(config) {
if (this.sendToWorker) {
this.sendToWorker({
type: "video-config-update",
data: config
});
}
}
/**
* Send disconnection notification to worker
*/
sendDisconnectionToWorker() {
if (this.sendToWorker) {
this.sendToWorker({
type: "video-config-update",
data: {
disconnect: true,
timestamp: performance.now()
}
});
}
}
/**
* Handle image file as video source
*/
async setImageSource(imageFile) {
try {
const img = new Image();
const url = URL.createObjectURL(imageFile);
await new Promise((resolve, reject) => {
img.onload = () => {
URL.revokeObjectURL(url);
resolve();
};
img.onerror = () => {
URL.revokeObjectURL(url);
reject(new Error("Failed to load image"));
};
img.src = url;
});
this.disconnectVi