@viji-dev/core
Version:
Universal execution engine for Viji Creative scenes
1,429 lines (1,428 loc) โข 48.9 kB
JavaScript
(function() {
"use strict";
class ParameterSystem {
// Parameter system for Phase 2 (new object-based approach)
parameterDefinitions = /* @__PURE__ */ new Map();
parameterGroups = /* @__PURE__ */ new Map();
parameterValues = /* @__PURE__ */ new Map();
parameterObjects = /* @__PURE__ */ new Map();
// Maps parameter names to their objects
parametersDefined = false;
initialValuesSynced = false;
// Track if initial values have been synced from host
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Message posting callback
postMessageCallback;
constructor(postMessageCallback) {
this.postMessageCallback = postMessageCallback;
}
// Parameter helper function implementations (return parameter objects)
createSliderParameter(defaultValue, config) {
const paramName = config.label;
const sliderObject = {
value: defaultValue,
min: config.min ?? 0,
max: config.max ?? 100,
step: config.step ?? 1,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "slider",
defaultValue,
label: sliderObject.label,
description: sliderObject.description,
group: sliderObject.group,
category: sliderObject.category,
config: {
min: sliderObject.min,
max: sliderObject.max,
step: sliderObject.step
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, sliderObject);
return sliderObject;
}
createColorParameter(defaultValue, config) {
const paramName = config.label;
const colorObject = {
value: defaultValue,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "color",
defaultValue,
label: colorObject.label,
description: colorObject.description,
group: colorObject.group,
category: colorObject.category
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, colorObject);
return colorObject;
}
createToggleParameter(defaultValue, config) {
const paramName = config.label;
const toggleObject = {
value: defaultValue,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "toggle",
defaultValue,
label: toggleObject.label,
description: toggleObject.description,
group: toggleObject.group,
category: toggleObject.category
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, toggleObject);
return toggleObject;
}
createSelectParameter(defaultValue, config) {
const paramName = config.label;
const selectObject = {
value: defaultValue,
options: config.options,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "select",
defaultValue,
label: selectObject.label,
description: selectObject.description,
group: selectObject.group,
category: selectObject.category,
config: {
options: selectObject.options
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, selectObject);
return selectObject;
}
createTextParameter(defaultValue, config) {
const paramName = config.label;
const textObject = {
value: defaultValue,
maxLength: config.maxLength ?? 1e3,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "text",
defaultValue,
label: textObject.label,
description: textObject.description,
group: textObject.group,
category: textObject.category,
config: {
maxLength: textObject.maxLength
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, textObject);
return textObject;
}
createNumberParameter(defaultValue, config) {
const paramName = config.label;
const numberObject = {
value: defaultValue,
min: config.min ?? 0,
max: config.max ?? 100,
step: config.step ?? 1,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "number",
defaultValue,
label: numberObject.label,
description: numberObject.description,
group: numberObject.group,
category: numberObject.category,
config: {
min: numberObject.min,
max: numberObject.max,
step: numberObject.step
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, numberObject);
return numberObject;
}
storeParameterDefinition(name, definition) {
this.parameterDefinitions.set(name, definition);
this.parameterValues.set(name, definition.defaultValue);
}
updateParameterValue(name, value) {
const definition = this.parameterDefinitions.get(name);
if (!definition) {
console.warn(`Unknown parameter: ${name}. Available parameters:`, Array.from(this.parameterDefinitions.keys()));
return false;
}
if (!this.validateParameterValue(name, value, definition)) {
console.warn(`Validation failed for parameter ${name} = ${value}`);
return false;
}
const currentValue = this.parameterValues.get(name);
const isInitialSync = !this.initialValuesSynced;
if (currentValue === value && !isInitialSync) {
return false;
}
this.parameterValues.set(name, value);
const parameterObject = this.parameterObjects.get(name);
if (parameterObject) {
parameterObject.value = value;
}
return true;
}
validateParameterValue(name, value, definition) {
if (definition.validate && !definition.validate(value)) {
console.error(`Custom validation failed for parameter '${name}': ${value}`);
return false;
}
switch (definition.type) {
case "slider":
case "number":
if (typeof value !== "number" || isNaN(value)) {
console.error(`Parameter '${name}' must be a number, got: ${value}`);
return false;
}
if (definition.config?.min !== void 0 && value < definition.config.min) {
console.error(`Parameter '${name}' value ${value} is below minimum ${definition.config.min}`);
return false;
}
if (definition.config?.max !== void 0 && value > definition.config.max) {
console.error(`Parameter '${name}' value ${value} is above maximum ${definition.config.max}`);
return false;
}
break;
case "color":
if (typeof value !== "string" || !/^#[0-9A-Fa-f]{6}$/.test(value)) {
console.error(`Parameter '${name}' must be a valid hex color, got: ${value}`);
return false;
}
break;
case "toggle":
if (typeof value !== "boolean") {
console.error(`Parameter '${name}' must be a boolean, got: ${value}`);
return false;
}
break;
case "select":
if (!definition.config?.options || !definition.config.options.includes(value)) {
console.error(`Parameter '${name}' value ${value} is not in options: ${definition.config?.options}`);
return false;
}
break;
case "text":
if (typeof value !== "string") {
console.error(`Parameter '${name}' must be a string, got: ${value}`);
return false;
}
if (definition.config?.maxLength && value.length > definition.config.maxLength) {
console.error(`Parameter '${name}' text too long: ${value.length} > ${definition.config.maxLength}`);
return false;
}
break;
}
return true;
}
// Reset parameter state (called when loading new scene)
resetParameterState() {
this.parametersDefined = false;
this.initialValuesSynced = false;
this.parameterDefinitions.clear();
this.parameterGroups.clear();
this.parameterValues.clear();
this.parameterObjects.clear();
}
// Send all parameters (from helper functions) to host
sendAllParametersToHost() {
if (this.parametersDefined || this.parameterDefinitions.size === 0) {
return;
}
try {
const groups = /* @__PURE__ */ new Map();
for (const [paramName, paramDef] of this.parameterDefinitions) {
const groupName = paramDef.group || "general";
if (!groups.has(groupName)) {
const category = paramDef.category || "general";
groups.set(groupName, {
groupName,
category,
parameters: {}
});
}
const group = groups.get(groupName);
group.parameters[paramName] = paramDef;
}
this.parametersDefined = true;
this.postMessageCallback("parameters-defined", {
groups: Array.from(groups.values()),
timestamp: performance.now()
});
this.debugLog(`All parameters sent to host: ${this.parameterDefinitions.size} parameters in ${groups.size} groups`);
} catch (error) {
this.postMessageCallback("parameter-validation-error", {
message: `Failed to send parameters to host: ${error.message}`,
code: "PARAMETER_SENDING_ERROR"
});
}
}
// Mark initial values as synced
markInitialValuesSynced() {
this.initialValuesSynced = true;
}
// Get parameter count for performance reporting
getParameterCount() {
return this.parameterDefinitions.size;
}
}
class InteractionSystem {
// Interaction enabled state
isEnabled = true;
// Mouse interaction state
mouseState = {
x: 0,
y: 0,
isInCanvas: false,
isPressed: false,
leftButton: false,
rightButton: false,
middleButton: false,
velocity: { x: 0, y: 0 },
deltaX: 0,
deltaY: 0,
wheelDelta: 0,
wheelX: 0,
wheelY: 0,
wasPressed: false,
wasReleased: false,
wasMoved: false
};
// Keyboard interaction state
keyboardState = {
isPressed: (key) => this.keyboardState.activeKeys.has(key.toLowerCase()),
wasPressed: (key) => this.keyboardState.pressedThisFrame.has(key.toLowerCase()),
wasReleased: (key) => this.keyboardState.releasedThisFrame.has(key.toLowerCase()),
activeKeys: /* @__PURE__ */ new Set(),
pressedThisFrame: /* @__PURE__ */ new Set(),
releasedThisFrame: /* @__PURE__ */ new Set(),
lastKeyPressed: "",
lastKeyReleased: "",
shift: false,
ctrl: false,
alt: false,
meta: false
};
// Touch interaction state
touchState = {
points: [],
count: 0,
started: [],
moved: [],
ended: [],
primary: null,
gestures: {
isPinching: false,
isRotating: false,
isPanning: false,
isTapping: false,
pinchScale: 1,
pinchDelta: 0,
rotationAngle: 0,
rotationDelta: 0,
panDelta: { x: 0, y: 0 },
tapCount: 0,
lastTapTime: 0,
tapPosition: null
}
};
constructor() {
this.handleMouseUpdate = this.handleMouseUpdate.bind(this);
this.handleKeyboardUpdate = this.handleKeyboardUpdate.bind(this);
this.handleTouchUpdate = this.handleTouchUpdate.bind(this);
this.frameStart = this.frameStart.bind(this);
}
/**
* Get the interaction APIs for inclusion in the viji object
*/
getInteractionAPIs() {
return {
mouse: this.mouseState,
keyboard: this.keyboardState,
touches: this.touchState
};
}
/**
* Called at the start of each frame to reset frame-based events
*/
frameStart() {
this.mouseState.wasPressed = false;
this.mouseState.wasReleased = false;
this.mouseState.wasMoved = false;
this.mouseState.wheelDelta = 0;
this.mouseState.wheelX = 0;
this.mouseState.wheelY = 0;
this.keyboardState.pressedThisFrame.clear();
this.keyboardState.releasedThisFrame.clear();
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
this.touchState.gestures.isTapping = false;
this.touchState.gestures.pinchDelta = 0;
this.touchState.gestures.rotationDelta = 0;
}
/**
* Handle mouse update messages from the host
*/
handleMouseUpdate(data) {
if (!this.isEnabled) return;
this.mouseState.x = data.x;
this.mouseState.y = data.y;
this.mouseState.isInCanvas = data.isInCanvas !== void 0 ? data.isInCanvas : true;
this.mouseState.leftButton = (data.buttons & 1) !== 0;
this.mouseState.rightButton = (data.buttons & 2) !== 0;
this.mouseState.middleButton = (data.buttons & 4) !== 0;
this.mouseState.isPressed = data.buttons > 0;
this.mouseState.deltaX = data.deltaX || 0;
this.mouseState.deltaY = data.deltaY || 0;
this.mouseState.wheelDelta = data.wheelDeltaY || 0;
this.mouseState.wheelX = data.wheelDeltaX || 0;
this.mouseState.wheelY = data.wheelDeltaY || 0;
this.mouseState.velocity.x = data.deltaX || 0;
this.mouseState.velocity.y = data.deltaY || 0;
this.mouseState.wasPressed = data.wasPressed || false;
this.mouseState.wasReleased = data.wasReleased || false;
this.mouseState.wasMoved = data.deltaX !== 0 || data.deltaY !== 0;
}
/**
* Handle keyboard update messages from the host
*/
handleKeyboardUpdate(data) {
if (!this.isEnabled) return;
const key = data.key.toLowerCase();
if (data.type === "keydown") {
if (!this.keyboardState.activeKeys.has(key)) {
this.keyboardState.activeKeys.add(key);
this.keyboardState.pressedThisFrame.add(key);
this.keyboardState.lastKeyPressed = data.key;
}
} else if (data.type === "keyup") {
this.keyboardState.activeKeys.delete(key);
this.keyboardState.releasedThisFrame.add(key);
this.keyboardState.lastKeyReleased = data.key;
}
this.keyboardState.shift = data.shiftKey;
this.keyboardState.ctrl = data.ctrlKey;
this.keyboardState.alt = data.altKey;
this.keyboardState.meta = data.metaKey;
}
/**
* Handle touch update messages from the host
*/
handleTouchUpdate(data) {
if (!this.isEnabled) return;
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
const touches = data.touches.map((touch) => ({
id: touch.identifier,
x: touch.clientX,
y: touch.clientY,
pressure: touch.pressure || 0,
radius: Math.max(touch.radiusX || 0, touch.radiusY || 0),
radiusX: touch.radiusX || 0,
radiusY: touch.radiusY || 0,
rotationAngle: touch.rotationAngle || 0,
force: touch.force || touch.pressure || 0,
deltaX: 0,
// Could be calculated if we track previous positions
deltaY: 0,
velocity: { x: 0, y: 0 },
// Could be calculated if we track movement
isNew: data.type === "touchstart",
isActive: true,
isEnding: data.type === "touchend" || data.type === "touchcancel"
}));
this.touchState.points = touches;
this.touchState.count = touches.length;
this.touchState.primary = touches[0] || null;
if (data.type === "touchstart") {
this.touchState.started = touches;
} else if (data.type === "touchmove") {
this.touchState.moved = touches;
} else if (data.type === "touchend" || data.type === "touchcancel") {
this.touchState.ended = touches;
}
this.touchState.gestures = {
isPinching: false,
isRotating: false,
isPanning: false,
isTapping: false,
pinchScale: 1,
pinchDelta: 0,
rotationAngle: 0,
rotationDelta: 0,
panDelta: { x: 0, y: 0 },
tapCount: 0,
lastTapTime: 0,
tapPosition: null
};
}
/**
* Reset all interaction state (called when loading new scene)
*/
resetInteractionState() {
Object.assign(this.mouseState, {
x: 0,
y: 0,
isInCanvas: false,
isPressed: false,
leftButton: false,
rightButton: false,
middleButton: false,
velocity: { x: 0, y: 0 },
deltaX: 0,
deltaY: 0,
wheelDelta: 0,
wheelX: 0,
wheelY: 0,
wasPressed: false,
wasReleased: false,
wasMoved: false
});
this.keyboardState.activeKeys.clear();
this.keyboardState.pressedThisFrame.clear();
this.keyboardState.releasedThisFrame.clear();
this.keyboardState.lastKeyPressed = "";
this.keyboardState.lastKeyReleased = "";
this.keyboardState.shift = false;
this.keyboardState.ctrl = false;
this.keyboardState.alt = false;
this.keyboardState.meta = false;
this.touchState.points = [];
this.touchState.count = 0;
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
this.touchState.primary = null;
Object.assign(this.touchState.gestures, {
isPinching: false,
isRotating: false,
isPanning: false,
isTapping: false,
pinchScale: 1,
pinchDelta: 0,
rotationAngle: 0,
rotationDelta: 0,
panDelta: { x: 0, y: 0 },
tapCount: 0,
lastTapTime: 0,
tapPosition: null
});
}
/**
* Enable or disable interaction processing
*/
setInteractionEnabled(enabled) {
this.isEnabled = enabled;
if (!enabled) {
this.resetInteractionStates();
}
}
/**
* Get current interaction enabled state
*/
getInteractionEnabled() {
return this.isEnabled;
}
/**
* Reset all interaction states to default values
*/
resetInteractionStates() {
this.mouseState.x = 0;
this.mouseState.y = 0;
this.mouseState.isInCanvas = false;
this.mouseState.isPressed = false;
this.mouseState.leftButton = false;
this.mouseState.rightButton = false;
this.mouseState.middleButton = false;
this.mouseState.velocity.x = 0;
this.mouseState.velocity.y = 0;
this.mouseState.deltaX = 0;
this.mouseState.deltaY = 0;
this.mouseState.wheelDelta = 0;
this.mouseState.wheelX = 0;
this.mouseState.wheelY = 0;
this.mouseState.wasPressed = false;
this.mouseState.wasReleased = false;
this.mouseState.wasMoved = false;
this.keyboardState.activeKeys.clear();
this.keyboardState.pressedThisFrame.clear();
this.keyboardState.releasedThisFrame.clear();
this.keyboardState.lastKeyPressed = "";
this.keyboardState.lastKeyReleased = "";
this.keyboardState.shift = false;
this.keyboardState.ctrl = false;
this.keyboardState.alt = false;
this.keyboardState.meta = false;
this.touchState.points = [];
this.touchState.count = 0;
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
this.touchState.primary = null;
this.touchState.gestures.isPinching = false;
this.touchState.gestures.isRotating = false;
this.touchState.gestures.isPanning = false;
this.touchState.gestures.isTapping = false;
this.touchState.gestures.pinchScale = 1;
this.touchState.gestures.pinchDelta = 0;
this.touchState.gestures.rotationAngle = 0;
this.touchState.gestures.rotationDelta = 0;
this.touchState.gestures.panDelta = { x: 0, y: 0 };
this.touchState.gestures.tapCount = 0;
this.touchState.gestures.lastTapTime = 0;
this.touchState.gestures.tapPosition = null;
}
}
class VideoSystem {
// โ
CORRECT: Worker-owned OffscreenCanvas (transferred from host)
offscreenCanvas = null;
ctx = null;
gl = null;
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Frame processing configuration
targetFrameRate = 30;
// Default target FPS for video processing
lastFrameTime = 0;
frameInterval = 1e3 / this.targetFrameRate;
// ms between frames
// Processing state
hasLoggedFirstFrame = false;
frameCount = 0;
// Video state for artist API
videoState = {
isConnected: false,
currentFrame: null,
frameWidth: 0,
frameHeight: 0,
frameRate: 0,
frameData: null
};
// Phase 11 preparation - CV processing placeholder
cvFeatures = {
faceDetection: false,
handTracking: false,
bodySegmentation: false
};
cvResults = {
faces: [],
hands: [],
bodySegmentation: null
};
constructor() {
}
/**
* Get the video API for inclusion in the viji object
*/
getVideoAPI() {
return {
isConnected: this.videoState.isConnected,
currentFrame: this.videoState.currentFrame,
frameWidth: this.videoState.frameWidth,
frameHeight: this.videoState.frameHeight,
frameRate: this.videoState.frameRate,
getFrameData: () => this.videoState.frameData,
faces: this.cvResults.faces,
hands: this.cvResults.hands
};
}
/**
* โ
CORRECT: Receive OffscreenCanvas transfer from host
*/
handleCanvasSetup(data) {
try {
this.disconnectVideo();
this.offscreenCanvas = data.offscreenCanvas;
this.ctx = this.offscreenCanvas.getContext("2d", {
willReadFrequently: true
// Optimize for frequent getImageData calls
});
if (!this.ctx) {
throw new Error("Failed to get 2D context from transferred OffscreenCanvas");
}
try {
this.gl = this.offscreenCanvas.getContext("webgl2") || this.offscreenCanvas.getContext("webgl");
} catch (e) {
this.debugLog("WebGL not available, using 2D context only");
}
this.videoState.isConnected = true;
this.videoState.currentFrame = this.offscreenCanvas;
this.videoState.frameWidth = data.width;
this.videoState.frameHeight = data.height;
this.frameCount = 0;
this.hasLoggedFirstFrame = false;
this.debugLog("โ
OffscreenCanvas received and setup completed (worker-side)", {
width: data.width,
height: data.height,
hasWebGL: !!this.gl,
targetFrameRate: this.targetFrameRate
});
this.debugLog("๐ฌ CORRECT OffscreenCanvas approach - Worker has full GPU access!");
} catch (error) {
console.error("Failed to setup OffscreenCanvas in worker:", error);
this.disconnectVideo();
}
}
/**
* โ
CORRECT: Receive ImageBitmap frame and draw to worker's OffscreenCanvas
*/
handleFrameUpdate(data) {
if (!this.offscreenCanvas || !this.ctx) {
console.warn("๐ด Received frame but OffscreenCanvas not setup");
return;
}
try {
if (this.frameCount % 150 === 0) {
this.debugLog("โ
Worker received ImageBitmap frame:", {
bitmapSize: `${data.imageBitmap.width}x${data.imageBitmap.height}`,
canvasSize: `${this.offscreenCanvas.width}x${this.offscreenCanvas.height}`,
frameCount: this.frameCount,
timestamp: data.timestamp
});
}
this.ctx.drawImage(data.imageBitmap, 0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height);
this.processCurrentFrame(data.timestamp);
data.imageBitmap.close();
this.frameCount++;
} catch (error) {
console.error("๐ด Error processing video frame (worker-side):", error);
}
}
/**
* Process current frame (called when new frame is drawn)
*/
processCurrentFrame(timestamp) {
if (!this.offscreenCanvas || !this.ctx) {
return;
}
try {
this.videoState.frameData = this.ctx.getImageData(
0,
0,
this.offscreenCanvas.width,
this.offscreenCanvas.height
);
const deltaTime = timestamp - this.lastFrameTime;
this.videoState.frameRate = deltaTime > 0 ? 1e3 / deltaTime : 0;
if (!this.hasLoggedFirstFrame) {
this.debugLog(`๐ฏ Worker-side OffscreenCanvas processing active: ${this.videoState.frameRate.toFixed(1)} FPS (${this.offscreenCanvas.width}x${this.offscreenCanvas.height})`);
this.debugLog("โ
Full GPU access available for custom effects and CV analysis");
this.hasLoggedFirstFrame = true;
}
this.performCVAnalysis();
this.lastFrameTime = timestamp;
} catch (error) {
console.error("Error processing video frame (worker-side):", error);
}
}
/**
* Handle video configuration updates (including disconnection and resize)
*/
handleVideoConfigUpdate(data) {
try {
if (data.disconnect) {
this.disconnectVideo();
return;
}
if (data.width && data.height && this.offscreenCanvas) {
this.resizeCanvas(data.width, data.height);
}
if (data.targetFrameRate) {
this.updateProcessingConfig(data.targetFrameRate);
}
if (data.cvConfig) {
this.updateCVConfig(data.cvConfig);
}
} catch (error) {
console.error("Error handling video config update:", error);
}
}
/**
* Resize the OffscreenCanvas (when video dimensions change)
*/
resizeCanvas(width, height) {
if (!this.offscreenCanvas) return;
try {
this.offscreenCanvas.width = width;
this.offscreenCanvas.height = height;
this.videoState.frameWidth = width;
this.videoState.frameHeight = height;
if (this.gl) {
this.gl.viewport(0, 0, width, height);
}
this.debugLog(`๐ OffscreenCanvas resized to ${width}x${height} (worker-side)`);
} catch (error) {
console.error("Error resizing OffscreenCanvas:", error);
}
}
/**
* Disconnect video and clean up resources
*/
disconnectVideo() {
if (this.offscreenCanvas && this.ctx) {
this.ctx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height);
this.debugLog("๐งน Cleared OffscreenCanvas on disconnect");
}
this.offscreenCanvas = null;
this.ctx = null;
this.gl = null;
this.videoState.isConnected = false;
this.videoState.currentFrame = null;
this.videoState.frameWidth = 0;
this.videoState.frameHeight = 0;
this.videoState.frameRate = 0;
this.videoState.frameData = null;
this.resetCVResults();
this.hasLoggedFirstFrame = false;
this.frameCount = 0;
this.debugLog("Video disconnected (worker-side)");
}
/**
* Update video processing configuration
*/
updateProcessingConfig(targetFrameRate) {
this.targetFrameRate = Math.max(1, Math.min(60, targetFrameRate));
this.frameInterval = 1e3 / this.targetFrameRate;
this.debugLog(`Video processing frame rate updated to ${this.targetFrameRate} FPS (worker-side)`);
}
/**
* Phase 11 preparation - Update CV configuration
*/
updateCVConfig(cvConfig) {
this.cvFeatures = {
faceDetection: cvConfig.faceDetection || false,
handTracking: cvConfig.handTracking || false,
bodySegmentation: cvConfig.bodySegmentation || false
};
this.debugLog("CV configuration updated (Phase 11 preparation, worker-side):", this.cvFeatures);
}
/**
* Phase 11 preparation - Perform computer vision analysis
*/
performCVAnalysis() {
if (this.cvFeatures.faceDetection) ;
if (this.cvFeatures.handTracking) ;
if (this.cvFeatures.bodySegmentation) ;
}
/**
* Reset CV results
*/
resetCVResults() {
this.cvResults = {
faces: [],
hands: [],
bodySegmentation: null
};
}
/**
* Reset all video state (called when loading new scene)
*/
resetVideoState() {
this.disconnectVideo();
this.resetCVResults();
}
/**
* Get current processing configuration
*/
getProcessingConfig() {
return {
targetFrameRate: this.targetFrameRate,
frameInterval: this.frameInterval,
frameCount: this.frameCount
};
}
/**
* Get WebGL context for advanced effects (if available)
*/
getWebGLContext() {
return this.gl;
}
/**
* โ
WORKER API: Artists can access the OffscreenCanvas directly for custom effects
*/
getCanvasForArtistEffects() {
return this.offscreenCanvas;
}
}
class VijiWorkerRuntime {
canvas = null;
ctx = null;
gl = null;
isRunning = false;
frameCount = 0;
lastTime = 0;
startTime = 0;
frameRateMode = "full";
skipNextFrame = false;
screenRefreshRate = 60;
// Will be detected
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
if (this.videoSystem) this.videoSystem.setDebugMode(enabled);
if (this.parameterSystem && "setDebugMode" in this.parameterSystem) {
this.parameterSystem.setDebugMode(enabled);
}
if (this.interactionSystem && "setDebugMode" in this.interactionSystem) {
this.interactionSystem.setDebugMode(enabled);
}
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Effective refresh rate tracking
effectiveFrameTimes = [];
lastEffectiveRateReport = 0;
effectiveRateReportInterval = 1e3;
// Report every 1 second
// Parameter system
parameterSystem;
// Interaction system (Phase 7)
interactionSystem;
// Video system (Phase 10) - worker-side video processing
videoSystem;
// Audio state (Phase 5) - receives analysis results from host
audioState = {
isConnected: false,
volume: { rms: 0, peak: 0 },
bands: {
bass: 0,
mid: 0,
treble: 0,
subBass: 0,
lowMid: 0,
highMid: 0,
presence: 0,
brilliance: 0
},
frequencyData: new Uint8Array(0)
};
// Video state is now managed by the worker-side VideoSystem
// Artist API object
viji = {
// Canvas (will be set during init)
canvas: null,
ctx: null,
gl: null,
width: 0,
height: 0,
pixelRatio: 1,
// Timing
time: 0,
deltaTime: 0,
frameCount: 0,
fps: 60,
// Audio API (Phase 5) - will be set in constructor
audio: {},
video: {
isConnected: false,
currentFrame: null,
frameWidth: 0,
frameHeight: 0,
frameRate: 0,
getFrameData: () => null,
faces: [],
hands: []
},
// Interaction APIs will be added during construction
mouse: {},
keyboard: {},
touches: {},
// Parameter helper functions (return parameter objects) - delegate to parameter system
slider: (defaultValue, config) => {
return this.parameterSystem.createSliderParameter(defaultValue, config);
},
color: (defaultValue, config) => {
return this.parameterSystem.createColorParameter(defaultValue, config);
},
toggle: (defaultValue, config) => {
return this.parameterSystem.createToggleParameter(defaultValue, config);
},
select: (defaultValue, config) => {
return this.parameterSystem.createSelectParameter(defaultValue, config);
},
text: (defaultValue, config) => {
return this.parameterSystem.createTextParameter(defaultValue, config);
},
number: (defaultValue, config) => {
return this.parameterSystem.createNumberParameter(defaultValue, config);
},
// Context selection
useContext: (type) => {
if (type === "2d") {
if (!this.ctx && this.canvas) {
this.ctx = this.canvas.getContext("2d");
this.viji.ctx = this.ctx;
}
return this.ctx;
} else if (type === "webgl") {
if (!this.gl && this.canvas) {
this.gl = this.canvas.getContext("webgl2") || this.canvas.getContext("webgl");
this.viji.gl = this.gl;
if (this.gl) {
this.gl.viewport(0, 0, this.viji.width, this.viji.height);
}
}
return this.gl;
}
return null;
}
};
constructor() {
this.parameterSystem = new ParameterSystem((type, data) => {
this.postMessage(type, data);
});
this.interactionSystem = new InteractionSystem();
this.videoSystem = new VideoSystem();
Object.assign(this.viji, this.interactionSystem.getInteractionAPIs());
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
this.viji.audio = {
...this.audioState,
getFrequencyData: () => this.audioState.frequencyData
};
this.setupMessageHandling();
}
// Reset parameter state (called when loading new scene)
resetParameterState() {
this.parameterSystem.resetParameterState();
this.interactionSystem.resetInteractionState();
this.audioState = {
isConnected: false,
volume: { rms: 0, peak: 0 },
bands: {
bass: 0,
mid: 0,
treble: 0,
subBass: 0,
lowMid: 0,
highMid: 0,
presence: 0,
brilliance: 0
},
frequencyData: new Uint8Array(0)
};
this.viji.audio = {
...this.audioState,
getFrequencyData: () => this.audioState.frequencyData
};
this.videoSystem.resetVideoState();
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
// Send all parameters (from helper functions) to host
sendAllParametersToHost() {
this.parameterSystem.sendAllParametersToHost();
}
setupMessageHandling() {
self.onmessage = (event) => {
const message = event.data;
switch (message.type) {
case "init":
this.handleInit(message);
break;
case "frame-rate-update":
this.handleFrameRateUpdate(message);
break;
case "refresh-rate-update":
this.handleRefreshRateUpdate(message);
break;
case "resolution-update":
this.handleResolutionUpdate(message);
break;
case "set-scene-code":
this.handleSetSceneCode(message);
break;
case "debug-mode":
this.setDebugMode(message.data.enabled);
break;
case "parameter-update":
this.handleParameterUpdate(message);
break;
case "parameter-batch-update":
this.handleParameterBatchUpdate(message);
break;
case "stream-update":
this.handleStreamUpdate(message);
break;
case "audio-analysis-update":
this.handleAudioAnalysisUpdate(message);
break;
case "video-canvas-setup":
this.handleVideoCanvasSetup(message);
break;
case "video-frame-update":
this.handleVideoFrameUpdate(message);
break;
case "video-config-update":
this.handleVideoConfigUpdate(message);
break;
case "mouse-update":
this.handleMouseUpdate(message);
break;
case "keyboard-update":
this.handleKeyboardUpdate(message);
break;
case "touch-update":
this.handleTouchUpdate(message);
break;
case "interaction-enabled":
this.handleInteractionEnabled(message);
break;
case "performance-update":
this.handlePerformanceUpdate(message);
break;
case "capture-frame":
this.handleCaptureFrame(message);
break;
}
};
}
handleInit(message) {
try {
this.canvas = message.data.canvas;
this.viji.canvas = this.canvas;
this.viji.width = this.canvas.width;
this.viji.height = this.canvas.height;
this.startRenderLoop();
this.postMessage("ready", {
id: message.id,
canvasSize: { width: this.canvas.width, height: this.canvas.height }
});
} catch (error) {
this.postMessage("error", {
id: message.id,
message: error.message,
code: "INIT_ERROR"
});
}
}
handleFrameRateUpdate(message) {
if (message.data && message.data.mode) {
this.frameRateMode = message.data.mode;
this.debugLog("Frame rate mode updated to:", message.data.mode);
}
}
handleRefreshRateUpdate(message) {
if (message.data && message.data.screenRefreshRate) {
this.screenRefreshRate = message.data.screenRefreshRate;
this.debugLog("Screen refresh rate updated to:", message.data.screenRefreshRate + "Hz");
}
}
trackEffectiveFrameTime(currentTime) {
this.effectiveFrameTimes.push(currentTime);
if (this.effectiveFrameTimes.length > 60) {
this.effectiveFrameTimes.shift();
}
}
reportEffectiveRefreshRate(currentTime) {
if (currentTime - this.lastEffectiveRateReport >= this.effectiveRateReportInterval) {
if (this.effectiveFrameTimes.length >= 2) {
const totalTime = this.effectiveFrameTimes[this.effectiveFrameTimes.length - 1] - this.effectiveFrameTimes[0];
const frameCount = this.effectiveFrameTimes.length - 1;
const effectiveRefreshRate = Math.round(frameCount / totalTime * 1e3);
this.postMessage("performance-update", {
effectiveRefreshRate,
frameRateMode: this.frameRateMode,
screenRefreshRate: this.screenRefreshRate,
parameterCount: this.parameterSystem.getParameterCount()
});
}
this.lastEffectiveRateReport = currentTime;
}
}
handleResolutionUpdate(message) {
if (message.data) {
if (this.canvas) {
this.canvas.width = Math.round(message.data.effectiveWidth);
this.canvas.height = Math.round(message.data.effectiveHeight);
}
this.viji.width = Math.round(message.data.effectiveWidth);
this.viji.height = Math.round(message.data.effectiveHeight);
if (this.gl) {
this.gl.viewport(0, 0, this.viji.width, this.viji.height);
}
this.debugLog("Canvas resolution updated to:", this.viji.width + "x" + this.viji.height);
}
}
handleParameterUpdate(message) {
if (message.data && message.data.name !== void 0 && message.data.value !== void 0) {
this.parameterSystem.updateParameterValue(message.data.name, message.data.value);
}
}
handleParameterBatchUpdate(message) {
if (message.data && message.data.updates) {
for (const update of message.data.updates) {
this.parameterSystem.updateParameterValue(update.name, update.value);
}
this.parameterSystem.markInitialValuesSynced();
this.debugLog("Parameter system initialized successfully");
}
}
handleStreamUpdate(message) {
this.debugLog("Stream update:", message.data);
}
handleAudioAnalysisUpdate(message) {
this.audioState = {
isConnected: message.data.isConnected,
volume: message.data.volume,
bands: message.data.bands,
frequencyData: message.data.frequencyData
};
this.viji.audio = {
...this.audioState,
getFrequencyData: () => this.audioState.frequencyData
};
}
handleVideoCanvasSetup(message) {
this.videoSystem.handleCanvasSetup({
offscreenCanvas: message.data.offscreenCanvas,
width: message.data.width,
height: message.data.height,
timestamp: message.data.timestamp
});
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
handleVideoFrameUpdate(message) {
this.videoSystem.handleFrameUpdate({
imageBitmap: message.data.imageBitmap,
timestamp: message.data.timestamp
});
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
handleVideoConfigUpdate(message) {
this.videoSystem.handleVideoConfigUpdate({
...message.data.targetFrameRate && { targetFrameRate: message.data.targetFrameRate },
...message.data.cvConfig && { cvConfig: message.data.cvConfig },
...message.data.width && { width: message.data.width },
...message.data.height && { height: message.data.height },
...message.data.disconnect && { disconnect: message.data.disconnect },
timestamp: message.data.timestamp
});
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
handlePerformanceUpdate(message) {
this.debugLog("Performance update:", message.data);
}
/**
* Handle capture-frame request from host.
* Produces an ArrayBuffer (image bytes) to send back as transferable.
*/
async handleCaptureFrame(message) {
try {
if (!this.canvas) {
throw new Error("Canvas not initialized");
}
const mimeType = message.data.type || "image/jpeg";
const srcWidth = this.canvas.width;
const srcHeight = this.canvas.height;
let targetWidth = srcWidth;
let targetHeight = srcHeight;
if (typeof message.data.resolution === "number") {
const scale = message.data.resolution > 0 ? message.data.resolution : 1;
targetWidth = Math.max(1, Math.floor(srcWidth * scale));
targetHeight = Math.max(1, Math.floor(srcHeight * scale));
} else if (message.data.resolution && typeof message.data.resolution === "object") {
targetWidth = Math.max(1, Math.floor(message.data.resolution.width));
targetHeight = Math.max(1, Math.floor(message.data.resolution.height));
}
const srcAspect = srcWidth / srcHeight;
const dstAspect = targetWidth / targetHeight;
let sx = 0;
let sy = 0;
let sWidth = srcWidth;
let sHeight = srcHeight;
if (Math.abs(srcAspect - dstAspect) > 1e-6) {
if (dstAspect > srcAspect) {
sHeight = Math.floor(srcWidth / dstAspect);
sy = Math.floor((srcHeight - sHeight) / 2);
} else {
sWidth = Math.floor(srcHeight * dstAspect);
sx = Math.floor((srcWidth - sWidth) / 2);
}
}
const temp = new OffscreenCanvas(targetWidth, targetHeight);
const tctx = temp.getContext("2d");
if (!tctx) throw new Error("Failed to get 2D context");
tctx.drawImage(this.canvas, sx, sy, sWidth, sHeight, 0, 0, targetWidth, targetHeight);
const blob = await temp.convertToBlob({ type: mimeType });
const arrayBuffer = await blob.arrayBuffer();
self.postMessage({
type: "capture-frame-result",
id: message.id,
timestamp: Date.now(),
data: {
mimeType,
buffer: arrayBuffer,
width: targetWidth,
height: targetHeight
}
}, [arrayBuffer]);
} catch (error) {
this.postMessage("error", {
id: message.id,
message: error.message,
code: "CAPTURE_FRAME_ERROR"
});
}
}
handleSetSceneCode(message) {
if (message.data && message.data.sceneCode) {
self.setSceneCode(message.data.sceneCode);
}
}
startRenderLoop() {
this.isRunning = true;
this.startTime = performance.now();
this.lastTime = this.startTime;
this.renderFrame();
}
renderFrame() {
if (!this.isRunning) return;
const currentTime = performance.now();
this.interactionSystem.frameStart();
this.viji.fps = this.frameRateMode === "full" ? this.screenRefreshRate : this.screenRefreshRate / 2;
let shouldRender = true;
if (this.frameRateMode === "half") {
shouldRender = !this.skipNextFrame;
this.skipNextFrame = !this.skipNextFrame;
}
if (shouldRender) {
this.viji.deltaTime = (currentTime - this.lastTime) / 1e3;
this.viji.time = (currentTime - this.startTime) / 1e3;
this.viji.frameCount = ++this.frameCount;
this.trackEffectiveFrameTime(currentTime);
this.lastTime = currentTime;
try {
const renderFunction2 = self.renderFunction;
if (renderFunction2 && typeof renderFunction2 === "function") {
renderFunction2(this.viji);
}
} catch (error) {
console.error("Render error:", error);
this.postMessage("error", {
message: error.message,
code: "RENDER_ERROR",
stack: error.stack
});
}
}
this.reportEffectiveRefreshRate(currentTime);
requestAnimationFrame(() => this.renderFrame());
}
postMessage(type, data) {
self.postMessage({
type,
id: data?.id || `${type}_${Date.now()}`,
timestamp: Date.now(),
data
});
}
// Phase 7: Interaction Message Handlers (delegated to InteractionSystem)
handleMouseUpdate(message) {
this.interactionSystem.handleMouseUpdate(message.data);
}
handleKeyboardUpdate(message) {
this.interactionSystem.handleKeyboardUpdate(message.data);
}
handleTouchUpdate(message) {
this.interactionSystem.handleTouchUpdate(message.data);
}
handleInteractionEnabled(message) {
this.interactionSystem.setInteractionEnabled(message.data.enabled);
}
}
const runtime = new VijiWorkerRuntime();
let renderFunction = null;
function setSceneCode(sceneCode) {
try {
runtime.resetParameterState();
const functionBody = sceneCode + '\nif (typeof render === "function") {\n return render;\n}\nthrow new Error("Scene code must define a render function");';
const sceneFunction = new Function("viji", functionBody);
renderFunction = sceneFunction(runtime.viji);
self.renderFunction = renderFunction;
runtime.sendAllParametersToHost();
} catch (error) {
console.error("Failed to load scene code:", error);
self.postMessage({
type: "error",
id: `scene_error_${Date.now()}`,
timestamp: Date.now(),
data: {
message: `Scene code error: ${error.message}`,
code: "SCENE_CODE_ERROR"
}
});
}
}
self.setSceneCode = setSceneCode;
})();
//# sourceMappingURL=viji.worker-BKsgIT1d.js.map