UNPKG

@viji-dev/core

Version:

Universal execution engine for Viji Creative scenes

1,429 lines (1,428 loc) โ€ข 48.9 kB
(function() { "use strict"; class ParameterSystem { // Parameter system for Phase 2 (new object-based approach) parameterDefinitions = /* @__PURE__ */ new Map(); parameterGroups = /* @__PURE__ */ new Map(); parameterValues = /* @__PURE__ */ new Map(); parameterObjects = /* @__PURE__ */ new Map(); // Maps parameter names to their objects parametersDefined = false; initialValuesSynced = false; // Track if initial values have been synced from host // Debug logging control debugMode = false; /** * Enable or disable debug logging */ setDebugMode(enabled) { this.debugMode = enabled; } /** * Debug logging helper */ debugLog(message, ...args) { if (this.debugMode) { console.log(message, ...args); } } // Message posting callback postMessageCallback; constructor(postMessageCallback) { this.postMessageCallback = postMessageCallback; } // Parameter helper function implementations (return parameter objects) createSliderParameter(defaultValue, config) { const paramName = config.label; const sliderObject = { value: defaultValue, min: config.min ?? 0, max: config.max ?? 100, step: config.step ?? 1, label: config.label, description: config.description ?? "", group: config.group ?? "general", category: config.category ?? "general" }; const definition = { type: "slider", defaultValue, label: sliderObject.label, description: sliderObject.description, group: sliderObject.group, category: sliderObject.category, config: { min: sliderObject.min, max: sliderObject.max, step: sliderObject.step } }; this.storeParameterDefinition(paramName, definition); this.parameterObjects.set(paramName, sliderObject); return sliderObject; } createColorParameter(defaultValue, config) { const paramName = config.label; const colorObject = { value: defaultValue, label: config.label, description: config.description ?? "", group: config.group ?? "general", category: config.category ?? "general" }; const definition = { type: "color", defaultValue, label: colorObject.label, description: colorObject.description, group: colorObject.group, category: colorObject.category }; this.storeParameterDefinition(paramName, definition); this.parameterObjects.set(paramName, colorObject); return colorObject; } createToggleParameter(defaultValue, config) { const paramName = config.label; const toggleObject = { value: defaultValue, label: config.label, description: config.description ?? "", group: config.group ?? "general", category: config.category ?? "general" }; const definition = { type: "toggle", defaultValue, label: toggleObject.label, description: toggleObject.description, group: toggleObject.group, category: toggleObject.category }; this.storeParameterDefinition(paramName, definition); this.parameterObjects.set(paramName, toggleObject); return toggleObject; } createSelectParameter(defaultValue, config) { const paramName = config.label; const selectObject = { value: defaultValue, options: config.options, label: config.label, description: config.description ?? "", group: config.group ?? "general", category: config.category ?? "general" }; const definition = { type: "select", defaultValue, label: selectObject.label, description: selectObject.description, group: selectObject.group, category: selectObject.category, config: { options: selectObject.options } }; this.storeParameterDefinition(paramName, definition); this.parameterObjects.set(paramName, selectObject); return selectObject; } createTextParameter(defaultValue, config) { const paramName = config.label; const textObject = { value: defaultValue, maxLength: config.maxLength ?? 1e3, label: config.label, description: config.description ?? "", group: config.group ?? "general", category: config.category ?? "general" }; const definition = { type: "text", defaultValue, label: textObject.label, description: textObject.description, group: textObject.group, category: textObject.category, config: { maxLength: textObject.maxLength } }; this.storeParameterDefinition(paramName, definition); this.parameterObjects.set(paramName, textObject); return textObject; } createNumberParameter(defaultValue, config) { const paramName = config.label; const numberObject = { value: defaultValue, min: config.min ?? 0, max: config.max ?? 100, step: config.step ?? 1, label: config.label, description: config.description ?? "", group: config.group ?? "general", category: config.category ?? "general" }; const definition = { type: "number", defaultValue, label: numberObject.label, description: numberObject.description, group: numberObject.group, category: numberObject.category, config: { min: numberObject.min, max: numberObject.max, step: numberObject.step } }; this.storeParameterDefinition(paramName, definition); this.parameterObjects.set(paramName, numberObject); return numberObject; } storeParameterDefinition(name, definition) { this.parameterDefinitions.set(name, definition); this.parameterValues.set(name, definition.defaultValue); } updateParameterValue(name, value) { const definition = this.parameterDefinitions.get(name); if (!definition) { console.warn(`Unknown parameter: ${name}. Available parameters:`, Array.from(this.parameterDefinitions.keys())); return false; } if (!this.validateParameterValue(name, value, definition)) { console.warn(`Validation failed for parameter ${name} = ${value}`); return false; } const currentValue = this.parameterValues.get(name); const isInitialSync = !this.initialValuesSynced; if (currentValue === value && !isInitialSync) { return false; } this.parameterValues.set(name, value); const parameterObject = this.parameterObjects.get(name); if (parameterObject) { parameterObject.value = value; } return true; } validateParameterValue(name, value, definition) { if (definition.validate && !definition.validate(value)) { console.error(`Custom validation failed for parameter '${name}': ${value}`); return false; } switch (definition.type) { case "slider": case "number": if (typeof value !== "number" || isNaN(value)) { console.error(`Parameter '${name}' must be a number, got: ${value}`); return false; } if (definition.config?.min !== void 0 && value < definition.config.min) { console.error(`Parameter '${name}' value ${value} is below minimum ${definition.config.min}`); return false; } if (definition.config?.max !== void 0 && value > definition.config.max) { console.error(`Parameter '${name}' value ${value} is above maximum ${definition.config.max}`); return false; } break; case "color": if (typeof value !== "string" || !/^#[0-9A-Fa-f]{6}$/.test(value)) { console.error(`Parameter '${name}' must be a valid hex color, got: ${value}`); return false; } break; case "toggle": if (typeof value !== "boolean") { console.error(`Parameter '${name}' must be a boolean, got: ${value}`); return false; } break; case "select": if (!definition.config?.options || !definition.config.options.includes(value)) { console.error(`Parameter '${name}' value ${value} is not in options: ${definition.config?.options}`); return false; } break; case "text": if (typeof value !== "string") { console.error(`Parameter '${name}' must be a string, got: ${value}`); return false; } if (definition.config?.maxLength && value.length > definition.config.maxLength) { console.error(`Parameter '${name}' text too long: ${value.length} > ${definition.config.maxLength}`); return false; } break; } return true; } // Reset parameter state (called when loading new scene) resetParameterState() { this.parametersDefined = false; this.initialValuesSynced = false; this.parameterDefinitions.clear(); this.parameterGroups.clear(); this.parameterValues.clear(); this.parameterObjects.clear(); } // Send all parameters (from helper functions) to host sendAllParametersToHost() { if (this.parametersDefined || this.parameterDefinitions.size === 0) { return; } try { const groups = /* @__PURE__ */ new Map(); for (const [paramName, paramDef] of this.parameterDefinitions) { const groupName = paramDef.group || "general"; if (!groups.has(groupName)) { const category = paramDef.category || "general"; groups.set(groupName, { groupName, category, parameters: {} }); } const group = groups.get(groupName); group.parameters[paramName] = paramDef; } this.parametersDefined = true; this.postMessageCallback("parameters-defined", { groups: Array.from(groups.values()), timestamp: performance.now() }); this.debugLog(`All parameters sent to host: ${this.parameterDefinitions.size} parameters in ${groups.size} groups`); } catch (error) { this.postMessageCallback("parameter-validation-error", { message: `Failed to send parameters to host: ${error.message}`, code: "PARAMETER_SENDING_ERROR" }); } } // Mark initial values as synced markInitialValuesSynced() { this.initialValuesSynced = true; } // Get parameter count for performance reporting getParameterCount() { return this.parameterDefinitions.size; } } class InteractionSystem { // Interaction enabled state isEnabled = true; // Mouse interaction state mouseState = { x: 0, y: 0, isInCanvas: false, isPressed: false, leftButton: false, rightButton: false, middleButton: false, velocity: { x: 0, y: 0 }, deltaX: 0, deltaY: 0, wheelDelta: 0, wheelX: 0, wheelY: 0, wasPressed: false, wasReleased: false, wasMoved: false }; // Keyboard interaction state keyboardState = { isPressed: (key) => this.keyboardState.activeKeys.has(key.toLowerCase()), wasPressed: (key) => this.keyboardState.pressedThisFrame.has(key.toLowerCase()), wasReleased: (key) => this.keyboardState.releasedThisFrame.has(key.toLowerCase()), activeKeys: /* @__PURE__ */ new Set(), pressedThisFrame: /* @__PURE__ */ new Set(), releasedThisFrame: /* @__PURE__ */ new Set(), lastKeyPressed: "", lastKeyReleased: "", shift: false, ctrl: false, alt: false, meta: false }; // Touch interaction state touchState = { points: [], count: 0, started: [], moved: [], ended: [], primary: null, gestures: { isPinching: false, isRotating: false, isPanning: false, isTapping: false, pinchScale: 1, pinchDelta: 0, rotationAngle: 0, rotationDelta: 0, panDelta: { x: 0, y: 0 }, tapCount: 0, lastTapTime: 0, tapPosition: null } }; constructor() { this.handleMouseUpdate = this.handleMouseUpdate.bind(this); this.handleKeyboardUpdate = this.handleKeyboardUpdate.bind(this); this.handleTouchUpdate = this.handleTouchUpdate.bind(this); this.frameStart = this.frameStart.bind(this); } /** * Get the interaction APIs for inclusion in the viji object */ getInteractionAPIs() { return { mouse: this.mouseState, keyboard: this.keyboardState, touches: this.touchState }; } /** * Called at the start of each frame to reset frame-based events */ frameStart() { this.mouseState.wasPressed = false; this.mouseState.wasReleased = false; this.mouseState.wasMoved = false; this.mouseState.wheelDelta = 0; this.mouseState.wheelX = 0; this.mouseState.wheelY = 0; this.keyboardState.pressedThisFrame.clear(); this.keyboardState.releasedThisFrame.clear(); this.touchState.started = []; this.touchState.moved = []; this.touchState.ended = []; this.touchState.gestures.isTapping = false; this.touchState.gestures.pinchDelta = 0; this.touchState.gestures.rotationDelta = 0; } /** * Handle mouse update messages from the host */ handleMouseUpdate(data) { if (!this.isEnabled) return; this.mouseState.x = data.x; this.mouseState.y = data.y; this.mouseState.isInCanvas = data.isInCanvas !== void 0 ? data.isInCanvas : true; this.mouseState.leftButton = (data.buttons & 1) !== 0; this.mouseState.rightButton = (data.buttons & 2) !== 0; this.mouseState.middleButton = (data.buttons & 4) !== 0; this.mouseState.isPressed = data.buttons > 0; this.mouseState.deltaX = data.deltaX || 0; this.mouseState.deltaY = data.deltaY || 0; this.mouseState.wheelDelta = data.wheelDeltaY || 0; this.mouseState.wheelX = data.wheelDeltaX || 0; this.mouseState.wheelY = data.wheelDeltaY || 0; this.mouseState.velocity.x = data.deltaX || 0; this.mouseState.velocity.y = data.deltaY || 0; this.mouseState.wasPressed = data.wasPressed || false; this.mouseState.wasReleased = data.wasReleased || false; this.mouseState.wasMoved = data.deltaX !== 0 || data.deltaY !== 0; } /** * Handle keyboard update messages from the host */ handleKeyboardUpdate(data) { if (!this.isEnabled) return; const key = data.key.toLowerCase(); if (data.type === "keydown") { if (!this.keyboardState.activeKeys.has(key)) { this.keyboardState.activeKeys.add(key); this.keyboardState.pressedThisFrame.add(key); this.keyboardState.lastKeyPressed = data.key; } } else if (data.type === "keyup") { this.keyboardState.activeKeys.delete(key); this.keyboardState.releasedThisFrame.add(key); this.keyboardState.lastKeyReleased = data.key; } this.keyboardState.shift = data.shiftKey; this.keyboardState.ctrl = data.ctrlKey; this.keyboardState.alt = data.altKey; this.keyboardState.meta = data.metaKey; } /** * Handle touch update messages from the host */ handleTouchUpdate(data) { if (!this.isEnabled) return; this.touchState.started = []; this.touchState.moved = []; this.touchState.ended = []; const touches = data.touches.map((touch) => ({ id: touch.identifier, x: touch.clientX, y: touch.clientY, pressure: touch.pressure || 0, radius: Math.max(touch.radiusX || 0, touch.radiusY || 0), radiusX: touch.radiusX || 0, radiusY: touch.radiusY || 0, rotationAngle: touch.rotationAngle || 0, force: touch.force || touch.pressure || 0, deltaX: 0, // Could be calculated if we track previous positions deltaY: 0, velocity: { x: 0, y: 0 }, // Could be calculated if we track movement isNew: data.type === "touchstart", isActive: true, isEnding: data.type === "touchend" || data.type === "touchcancel" })); this.touchState.points = touches; this.touchState.count = touches.length; this.touchState.primary = touches[0] || null; if (data.type === "touchstart") { this.touchState.started = touches; } else if (data.type === "touchmove") { this.touchState.moved = touches; } else if (data.type === "touchend" || data.type === "touchcancel") { this.touchState.ended = touches; } this.touchState.gestures = { isPinching: false, isRotating: false, isPanning: false, isTapping: false, pinchScale: 1, pinchDelta: 0, rotationAngle: 0, rotationDelta: 0, panDelta: { x: 0, y: 0 }, tapCount: 0, lastTapTime: 0, tapPosition: null }; } /** * Reset all interaction state (called when loading new scene) */ resetInteractionState() { Object.assign(this.mouseState, { x: 0, y: 0, isInCanvas: false, isPressed: false, leftButton: false, rightButton: false, middleButton: false, velocity: { x: 0, y: 0 }, deltaX: 0, deltaY: 0, wheelDelta: 0, wheelX: 0, wheelY: 0, wasPressed: false, wasReleased: false, wasMoved: false }); this.keyboardState.activeKeys.clear(); this.keyboardState.pressedThisFrame.clear(); this.keyboardState.releasedThisFrame.clear(); this.keyboardState.lastKeyPressed = ""; this.keyboardState.lastKeyReleased = ""; this.keyboardState.shift = false; this.keyboardState.ctrl = false; this.keyboardState.alt = false; this.keyboardState.meta = false; this.touchState.points = []; this.touchState.count = 0; this.touchState.started = []; this.touchState.moved = []; this.touchState.ended = []; this.touchState.primary = null; Object.assign(this.touchState.gestures, { isPinching: false, isRotating: false, isPanning: false, isTapping: false, pinchScale: 1, pinchDelta: 0, rotationAngle: 0, rotationDelta: 0, panDelta: { x: 0, y: 0 }, tapCount: 0, lastTapTime: 0, tapPosition: null }); } /** * Enable or disable interaction processing */ setInteractionEnabled(enabled) { this.isEnabled = enabled; if (!enabled) { this.resetInteractionStates(); } } /** * Get current interaction enabled state */ getInteractionEnabled() { return this.isEnabled; } /** * Reset all interaction states to default values */ resetInteractionStates() { this.mouseState.x = 0; this.mouseState.y = 0; this.mouseState.isInCanvas = false; this.mouseState.isPressed = false; this.mouseState.leftButton = false; this.mouseState.rightButton = false; this.mouseState.middleButton = false; this.mouseState.velocity.x = 0; this.mouseState.velocity.y = 0; this.mouseState.deltaX = 0; this.mouseState.deltaY = 0; this.mouseState.wheelDelta = 0; this.mouseState.wheelX = 0; this.mouseState.wheelY = 0; this.mouseState.wasPressed = false; this.mouseState.wasReleased = false; this.mouseState.wasMoved = false; this.keyboardState.activeKeys.clear(); this.keyboardState.pressedThisFrame.clear(); this.keyboardState.releasedThisFrame.clear(); this.keyboardState.lastKeyPressed = ""; this.keyboardState.lastKeyReleased = ""; this.keyboardState.shift = false; this.keyboardState.ctrl = false; this.keyboardState.alt = false; this.keyboardState.meta = false; this.touchState.points = []; this.touchState.count = 0; this.touchState.started = []; this.touchState.moved = []; this.touchState.ended = []; this.touchState.primary = null; this.touchState.gestures.isPinching = false; this.touchState.gestures.isRotating = false; this.touchState.gestures.isPanning = false; this.touchState.gestures.isTapping = false; this.touchState.gestures.pinchScale = 1; this.touchState.gestures.pinchDelta = 0; this.touchState.gestures.rotationAngle = 0; this.touchState.gestures.rotationDelta = 0; this.touchState.gestures.panDelta = { x: 0, y: 0 }; this.touchState.gestures.tapCount = 0; this.touchState.gestures.lastTapTime = 0; this.touchState.gestures.tapPosition = null; } } class VideoSystem { // โœ… CORRECT: Worker-owned OffscreenCanvas (transferred from host) offscreenCanvas = null; ctx = null; gl = null; // Debug logging control debugMode = false; /** * Enable or disable debug logging */ setDebugMode(enabled) { this.debugMode = enabled; } /** * Debug logging helper */ debugLog(message, ...args) { if (this.debugMode) { console.log(message, ...args); } } // Frame processing configuration targetFrameRate = 30; // Default target FPS for video processing lastFrameTime = 0; frameInterval = 1e3 / this.targetFrameRate; // ms between frames // Processing state hasLoggedFirstFrame = false; frameCount = 0; // Video state for artist API videoState = { isConnected: false, currentFrame: null, frameWidth: 0, frameHeight: 0, frameRate: 0, frameData: null }; // Phase 11 preparation - CV processing placeholder cvFeatures = { faceDetection: false, handTracking: false, bodySegmentation: false }; cvResults = { faces: [], hands: [], bodySegmentation: null }; constructor() { } /** * Get the video API for inclusion in the viji object */ getVideoAPI() { return { isConnected: this.videoState.isConnected, currentFrame: this.videoState.currentFrame, frameWidth: this.videoState.frameWidth, frameHeight: this.videoState.frameHeight, frameRate: this.videoState.frameRate, getFrameData: () => this.videoState.frameData, faces: this.cvResults.faces, hands: this.cvResults.hands }; } /** * โœ… CORRECT: Receive OffscreenCanvas transfer from host */ handleCanvasSetup(data) { try { this.disconnectVideo(); this.offscreenCanvas = data.offscreenCanvas; this.ctx = this.offscreenCanvas.getContext("2d", { willReadFrequently: true // Optimize for frequent getImageData calls }); if (!this.ctx) { throw new Error("Failed to get 2D context from transferred OffscreenCanvas"); } try { this.gl = this.offscreenCanvas.getContext("webgl2") || this.offscreenCanvas.getContext("webgl"); } catch (e) { this.debugLog("WebGL not available, using 2D context only"); } this.videoState.isConnected = true; this.videoState.currentFrame = this.offscreenCanvas; this.videoState.frameWidth = data.width; this.videoState.frameHeight = data.height; this.frameCount = 0; this.hasLoggedFirstFrame = false; this.debugLog("โœ… OffscreenCanvas received and setup completed (worker-side)", { width: data.width, height: data.height, hasWebGL: !!this.gl, targetFrameRate: this.targetFrameRate }); this.debugLog("๐ŸŽฌ CORRECT OffscreenCanvas approach - Worker has full GPU access!"); } catch (error) { console.error("Failed to setup OffscreenCanvas in worker:", error); this.disconnectVideo(); } } /** * โœ… CORRECT: Receive ImageBitmap frame and draw to worker's OffscreenCanvas */ handleFrameUpdate(data) { if (!this.offscreenCanvas || !this.ctx) { console.warn("๐Ÿ”ด Received frame but OffscreenCanvas not setup"); return; } try { if (this.frameCount % 150 === 0) { this.debugLog("โœ… Worker received ImageBitmap frame:", { bitmapSize: `${data.imageBitmap.width}x${data.imageBitmap.height}`, canvasSize: `${this.offscreenCanvas.width}x${this.offscreenCanvas.height}`, frameCount: this.frameCount, timestamp: data.timestamp }); } this.ctx.drawImage(data.imageBitmap, 0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height); this.processCurrentFrame(data.timestamp); data.imageBitmap.close(); this.frameCount++; } catch (error) { console.error("๐Ÿ”ด Error processing video frame (worker-side):", error); } } /** * Process current frame (called when new frame is drawn) */ processCurrentFrame(timestamp) { if (!this.offscreenCanvas || !this.ctx) { return; } try { this.videoState.frameData = this.ctx.getImageData( 0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height ); const deltaTime = timestamp - this.lastFrameTime; this.videoState.frameRate = deltaTime > 0 ? 1e3 / deltaTime : 0; if (!this.hasLoggedFirstFrame) { this.debugLog(`๐ŸŽฏ Worker-side OffscreenCanvas processing active: ${this.videoState.frameRate.toFixed(1)} FPS (${this.offscreenCanvas.width}x${this.offscreenCanvas.height})`); this.debugLog("โœ… Full GPU access available for custom effects and CV analysis"); this.hasLoggedFirstFrame = true; } this.performCVAnalysis(); this.lastFrameTime = timestamp; } catch (error) { console.error("Error processing video frame (worker-side):", error); } } /** * Handle video configuration updates (including disconnection and resize) */ handleVideoConfigUpdate(data) { try { if (data.disconnect) { this.disconnectVideo(); return; } if (data.width && data.height && this.offscreenCanvas) { this.resizeCanvas(data.width, data.height); } if (data.targetFrameRate) { this.updateProcessingConfig(data.targetFrameRate); } if (data.cvConfig) { this.updateCVConfig(data.cvConfig); } } catch (error) { console.error("Error handling video config update:", error); } } /** * Resize the OffscreenCanvas (when video dimensions change) */ resizeCanvas(width, height) { if (!this.offscreenCanvas) return; try { this.offscreenCanvas.width = width; this.offscreenCanvas.height = height; this.videoState.frameWidth = width; this.videoState.frameHeight = height; if (this.gl) { this.gl.viewport(0, 0, width, height); } this.debugLog(`๐Ÿ“ OffscreenCanvas resized to ${width}x${height} (worker-side)`); } catch (error) { console.error("Error resizing OffscreenCanvas:", error); } } /** * Disconnect video and clean up resources */ disconnectVideo() { if (this.offscreenCanvas && this.ctx) { this.ctx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height); this.debugLog("๐Ÿงน Cleared OffscreenCanvas on disconnect"); } this.offscreenCanvas = null; this.ctx = null; this.gl = null; this.videoState.isConnected = false; this.videoState.currentFrame = null; this.videoState.frameWidth = 0; this.videoState.frameHeight = 0; this.videoState.frameRate = 0; this.videoState.frameData = null; this.resetCVResults(); this.hasLoggedFirstFrame = false; this.frameCount = 0; this.debugLog("Video disconnected (worker-side)"); } /** * Update video processing configuration */ updateProcessingConfig(targetFrameRate) { this.targetFrameRate = Math.max(1, Math.min(60, targetFrameRate)); this.frameInterval = 1e3 / this.targetFrameRate; this.debugLog(`Video processing frame rate updated to ${this.targetFrameRate} FPS (worker-side)`); } /** * Phase 11 preparation - Update CV configuration */ updateCVConfig(cvConfig) { this.cvFeatures = { faceDetection: cvConfig.faceDetection || false, handTracking: cvConfig.handTracking || false, bodySegmentation: cvConfig.bodySegmentation || false }; this.debugLog("CV configuration updated (Phase 11 preparation, worker-side):", this.cvFeatures); } /** * Phase 11 preparation - Perform computer vision analysis */ performCVAnalysis() { if (this.cvFeatures.faceDetection) ; if (this.cvFeatures.handTracking) ; if (this.cvFeatures.bodySegmentation) ; } /** * Reset CV results */ resetCVResults() { this.cvResults = { faces: [], hands: [], bodySegmentation: null }; } /** * Reset all video state (called when loading new scene) */ resetVideoState() { this.disconnectVideo(); this.resetCVResults(); } /** * Get current processing configuration */ getProcessingConfig() { return { targetFrameRate: this.targetFrameRate, frameInterval: this.frameInterval, frameCount: this.frameCount }; } /** * Get WebGL context for advanced effects (if available) */ getWebGLContext() { return this.gl; } /** * โœ… WORKER API: Artists can access the OffscreenCanvas directly for custom effects */ getCanvasForArtistEffects() { return this.offscreenCanvas; } } class VijiWorkerRuntime { canvas = null; ctx = null; gl = null; isRunning = false; frameCount = 0; lastTime = 0; startTime = 0; frameRateMode = "full"; skipNextFrame = false; screenRefreshRate = 60; // Will be detected // Debug logging control debugMode = false; /** * Enable or disable debug logging */ setDebugMode(enabled) { this.debugMode = enabled; if (this.videoSystem) this.videoSystem.setDebugMode(enabled); if (this.parameterSystem && "setDebugMode" in this.parameterSystem) { this.parameterSystem.setDebugMode(enabled); } if (this.interactionSystem && "setDebugMode" in this.interactionSystem) { this.interactionSystem.setDebugMode(enabled); } } /** * Debug logging helper */ debugLog(message, ...args) { if (this.debugMode) { console.log(message, ...args); } } // Effective refresh rate tracking effectiveFrameTimes = []; lastEffectiveRateReport = 0; effectiveRateReportInterval = 1e3; // Report every 1 second // Parameter system parameterSystem; // Interaction system (Phase 7) interactionSystem; // Video system (Phase 10) - worker-side video processing videoSystem; // Audio state (Phase 5) - receives analysis results from host audioState = { isConnected: false, volume: { rms: 0, peak: 0 }, bands: { bass: 0, mid: 0, treble: 0, subBass: 0, lowMid: 0, highMid: 0, presence: 0, brilliance: 0 }, frequencyData: new Uint8Array(0) }; // Video state is now managed by the worker-side VideoSystem // Artist API object viji = { // Canvas (will be set during init) canvas: null, ctx: null, gl: null, width: 0, height: 0, pixelRatio: 1, // Timing time: 0, deltaTime: 0, frameCount: 0, fps: 60, // Audio API (Phase 5) - will be set in constructor audio: {}, video: { isConnected: false, currentFrame: null, frameWidth: 0, frameHeight: 0, frameRate: 0, getFrameData: () => null, faces: [], hands: [] }, // Interaction APIs will be added during construction mouse: {}, keyboard: {}, touches: {}, // Parameter helper functions (return parameter objects) - delegate to parameter system slider: (defaultValue, config) => { return this.parameterSystem.createSliderParameter(defaultValue, config); }, color: (defaultValue, config) => { return this.parameterSystem.createColorParameter(defaultValue, config); }, toggle: (defaultValue, config) => { return this.parameterSystem.createToggleParameter(defaultValue, config); }, select: (defaultValue, config) => { return this.parameterSystem.createSelectParameter(defaultValue, config); }, text: (defaultValue, config) => { return this.parameterSystem.createTextParameter(defaultValue, config); }, number: (defaultValue, config) => { return this.parameterSystem.createNumberParameter(defaultValue, config); }, // Context selection useContext: (type) => { if (type === "2d") { if (!this.ctx && this.canvas) { this.ctx = this.canvas.getContext("2d"); this.viji.ctx = this.ctx; } return this.ctx; } else if (type === "webgl") { if (!this.gl && this.canvas) { this.gl = this.canvas.getContext("webgl2") || this.canvas.getContext("webgl"); this.viji.gl = this.gl; if (this.gl) { this.gl.viewport(0, 0, this.viji.width, this.viji.height); } } return this.gl; } return null; } }; constructor() { this.parameterSystem = new ParameterSystem((type, data) => { this.postMessage(type, data); }); this.interactionSystem = new InteractionSystem(); this.videoSystem = new VideoSystem(); Object.assign(this.viji, this.interactionSystem.getInteractionAPIs()); Object.assign(this.viji.video, this.videoSystem.getVideoAPI()); this.viji.audio = { ...this.audioState, getFrequencyData: () => this.audioState.frequencyData }; this.setupMessageHandling(); } // Reset parameter state (called when loading new scene) resetParameterState() { this.parameterSystem.resetParameterState(); this.interactionSystem.resetInteractionState(); this.audioState = { isConnected: false, volume: { rms: 0, peak: 0 }, bands: { bass: 0, mid: 0, treble: 0, subBass: 0, lowMid: 0, highMid: 0, presence: 0, brilliance: 0 }, frequencyData: new Uint8Array(0) }; this.viji.audio = { ...this.audioState, getFrequencyData: () => this.audioState.frequencyData }; this.videoSystem.resetVideoState(); Object.assign(this.viji.video, this.videoSystem.getVideoAPI()); } // Send all parameters (from helper functions) to host sendAllParametersToHost() { this.parameterSystem.sendAllParametersToHost(); } setupMessageHandling() { self.onmessage = (event) => { const message = event.data; switch (message.type) { case "init": this.handleInit(message); break; case "frame-rate-update": this.handleFrameRateUpdate(message); break; case "refresh-rate-update": this.handleRefreshRateUpdate(message); break; case "resolution-update": this.handleResolutionUpdate(message); break; case "set-scene-code": this.handleSetSceneCode(message); break; case "debug-mode": this.setDebugMode(message.data.enabled); break; case "parameter-update": this.handleParameterUpdate(message); break; case "parameter-batch-update": this.handleParameterBatchUpdate(message); break; case "stream-update": this.handleStreamUpdate(message); break; case "audio-analysis-update": this.handleAudioAnalysisUpdate(message); break; case "video-canvas-setup": this.handleVideoCanvasSetup(message); break; case "video-frame-update": this.handleVideoFrameUpdate(message); break; case "video-config-update": this.handleVideoConfigUpdate(message); break; case "mouse-update": this.handleMouseUpdate(message); break; case "keyboard-update": this.handleKeyboardUpdate(message); break; case "touch-update": this.handleTouchUpdate(message); break; case "interaction-enabled": this.handleInteractionEnabled(message); break; case "performance-update": this.handlePerformanceUpdate(message); break; case "capture-frame": this.handleCaptureFrame(message); break; } }; } handleInit(message) { try { this.canvas = message.data.canvas; this.viji.canvas = this.canvas; this.viji.width = this.canvas.width; this.viji.height = this.canvas.height; this.startRenderLoop(); this.postMessage("ready", { id: message.id, canvasSize: { width: this.canvas.width, height: this.canvas.height } }); } catch (error) { this.postMessage("error", { id: message.id, message: error.message, code: "INIT_ERROR" }); } } handleFrameRateUpdate(message) { if (message.data && message.data.mode) { this.frameRateMode = message.data.mode; this.debugLog("Frame rate mode updated to:", message.data.mode); } } handleRefreshRateUpdate(message) { if (message.data && message.data.screenRefreshRate) { this.screenRefreshRate = message.data.screenRefreshRate; this.debugLog("Screen refresh rate updated to:", message.data.screenRefreshRate + "Hz"); } } trackEffectiveFrameTime(currentTime) { this.effectiveFrameTimes.push(currentTime); if (this.effectiveFrameTimes.length > 60) { this.effectiveFrameTimes.shift(); } } reportEffectiveRefreshRate(currentTime) { if (currentTime - this.lastEffectiveRateReport >= this.effectiveRateReportInterval) { if (this.effectiveFrameTimes.length >= 2) { const totalTime = this.effectiveFrameTimes[this.effectiveFrameTimes.length - 1] - this.effectiveFrameTimes[0]; const frameCount = this.effectiveFrameTimes.length - 1; const effectiveRefreshRate = Math.round(frameCount / totalTime * 1e3); this.postMessage("performance-update", { effectiveRefreshRate, frameRateMode: this.frameRateMode, screenRefreshRate: this.screenRefreshRate, parameterCount: this.parameterSystem.getParameterCount() }); } this.lastEffectiveRateReport = currentTime; } } handleResolutionUpdate(message) { if (message.data) { if (this.canvas) { this.canvas.width = Math.round(message.data.effectiveWidth); this.canvas.height = Math.round(message.data.effectiveHeight); } this.viji.width = Math.round(message.data.effectiveWidth); this.viji.height = Math.round(message.data.effectiveHeight); if (this.gl) { this.gl.viewport(0, 0, this.viji.width, this.viji.height); } this.debugLog("Canvas resolution updated to:", this.viji.width + "x" + this.viji.height); } } handleParameterUpdate(message) { if (message.data && message.data.name !== void 0 && message.data.value !== void 0) { this.parameterSystem.updateParameterValue(message.data.name, message.data.value); } } handleParameterBatchUpdate(message) { if (message.data && message.data.updates) { for (const update of message.data.updates) { this.parameterSystem.updateParameterValue(update.name, update.value); } this.parameterSystem.markInitialValuesSynced(); this.debugLog("Parameter system initialized successfully"); } } handleStreamUpdate(message) { this.debugLog("Stream update:", message.data); } handleAudioAnalysisUpdate(message) { this.audioState = { isConnected: message.data.isConnected, volume: message.data.volume, bands: message.data.bands, frequencyData: message.data.frequencyData }; this.viji.audio = { ...this.audioState, getFrequencyData: () => this.audioState.frequencyData }; } handleVideoCanvasSetup(message) { this.videoSystem.handleCanvasSetup({ offscreenCanvas: message.data.offscreenCanvas, width: message.data.width, height: message.data.height, timestamp: message.data.timestamp }); Object.assign(this.viji.video, this.videoSystem.getVideoAPI()); } handleVideoFrameUpdate(message) { this.videoSystem.handleFrameUpdate({ imageBitmap: message.data.imageBitmap, timestamp: message.data.timestamp }); Object.assign(this.viji.video, this.videoSystem.getVideoAPI()); } handleVideoConfigUpdate(message) { this.videoSystem.handleVideoConfigUpdate({ ...message.data.targetFrameRate && { targetFrameRate: message.data.targetFrameRate }, ...message.data.cvConfig && { cvConfig: message.data.cvConfig }, ...message.data.width && { width: message.data.width }, ...message.data.height && { height: message.data.height }, ...message.data.disconnect && { disconnect: message.data.disconnect }, timestamp: message.data.timestamp }); Object.assign(this.viji.video, this.videoSystem.getVideoAPI()); } handlePerformanceUpdate(message) { this.debugLog("Performance update:", message.data); } /** * Handle capture-frame request from host. * Produces an ArrayBuffer (image bytes) to send back as transferable. */ async handleCaptureFrame(message) { try { if (!this.canvas) { throw new Error("Canvas not initialized"); } const mimeType = message.data.type || "image/jpeg"; const srcWidth = this.canvas.width; const srcHeight = this.canvas.height; let targetWidth = srcWidth; let targetHeight = srcHeight; if (typeof message.data.resolution === "number") { const scale = message.data.resolution > 0 ? message.data.resolution : 1; targetWidth = Math.max(1, Math.floor(srcWidth * scale)); targetHeight = Math.max(1, Math.floor(srcHeight * scale)); } else if (message.data.resolution && typeof message.data.resolution === "object") { targetWidth = Math.max(1, Math.floor(message.data.resolution.width)); targetHeight = Math.max(1, Math.floor(message.data.resolution.height)); } const srcAspect = srcWidth / srcHeight; const dstAspect = targetWidth / targetHeight; let sx = 0; let sy = 0; let sWidth = srcWidth; let sHeight = srcHeight; if (Math.abs(srcAspect - dstAspect) > 1e-6) { if (dstAspect > srcAspect) { sHeight = Math.floor(srcWidth / dstAspect); sy = Math.floor((srcHeight - sHeight) / 2); } else { sWidth = Math.floor(srcHeight * dstAspect); sx = Math.floor((srcWidth - sWidth) / 2); } } const temp = new OffscreenCanvas(targetWidth, targetHeight); const tctx = temp.getContext("2d"); if (!tctx) throw new Error("Failed to get 2D context"); tctx.drawImage(this.canvas, sx, sy, sWidth, sHeight, 0, 0, targetWidth, targetHeight); const blob = await temp.convertToBlob({ type: mimeType }); const arrayBuffer = await blob.arrayBuffer(); self.postMessage({ type: "capture-frame-result", id: message.id, timestamp: Date.now(), data: { mimeType, buffer: arrayBuffer, width: targetWidth, height: targetHeight } }, [arrayBuffer]); } catch (error) { this.postMessage("error", { id: message.id, message: error.message, code: "CAPTURE_FRAME_ERROR" }); } } handleSetSceneCode(message) { if (message.data && message.data.sceneCode) { self.setSceneCode(message.data.sceneCode); } } startRenderLoop() { this.isRunning = true; this.startTime = performance.now(); this.lastTime = this.startTime; this.renderFrame(); } renderFrame() { if (!this.isRunning) return; const currentTime = performance.now(); this.interactionSystem.frameStart(); this.viji.fps = this.frameRateMode === "full" ? this.screenRefreshRate : this.screenRefreshRate / 2; let shouldRender = true; if (this.frameRateMode === "half") { shouldRender = !this.skipNextFrame; this.skipNextFrame = !this.skipNextFrame; } if (shouldRender) { this.viji.deltaTime = (currentTime - this.lastTime) / 1e3; this.viji.time = (currentTime - this.startTime) / 1e3; this.viji.frameCount = ++this.frameCount; this.trackEffectiveFrameTime(currentTime); this.lastTime = currentTime; try { const renderFunction2 = self.renderFunction; if (renderFunction2 && typeof renderFunction2 === "function") { renderFunction2(this.viji); } } catch (error) { console.error("Render error:", error); this.postMessage("error", { message: error.message, code: "RENDER_ERROR", stack: error.stack }); } } this.reportEffectiveRefreshRate(currentTime); requestAnimationFrame(() => this.renderFrame()); } postMessage(type, data) { self.postMessage({ type, id: data?.id || `${type}_${Date.now()}`, timestamp: Date.now(), data }); } // Phase 7: Interaction Message Handlers (delegated to InteractionSystem) handleMouseUpdate(message) { this.interactionSystem.handleMouseUpdate(message.data); } handleKeyboardUpdate(message) { this.interactionSystem.handleKeyboardUpdate(message.data); } handleTouchUpdate(message) { this.interactionSystem.handleTouchUpdate(message.data); } handleInteractionEnabled(message) { this.interactionSystem.setInteractionEnabled(message.data.enabled); } } const runtime = new VijiWorkerRuntime(); let renderFunction = null; function setSceneCode(sceneCode) { try { runtime.resetParameterState(); const functionBody = sceneCode + '\nif (typeof render === "function") {\n return render;\n}\nthrow new Error("Scene code must define a render function");'; const sceneFunction = new Function("viji", functionBody); renderFunction = sceneFunction(runtime.viji); self.renderFunction = renderFunction; runtime.sendAllParametersToHost(); } catch (error) { console.error("Failed to load scene code:", error); self.postMessage({ type: "error", id: `scene_error_${Date.now()}`, timestamp: Date.now(), data: { message: `Scene code error: ${error.message}`, code: "SCENE_CODE_ERROR" } }); } } self.setSceneCode = setSceneCode; })(); //# sourceMappingURL=viji.worker-BKsgIT1d.js.map