@viji-dev/core
Version:
Universal execution engine for Viji Creative scenes
3,550 lines • 123 kB
JavaScript
class ParameterSystem {
// Parameter system for Phase 2 (new object-based approach)
parameterDefinitions = /* @__PURE__ */ new Map();
parameterGroups = /* @__PURE__ */ new Map();
parameterValues = /* @__PURE__ */ new Map();
parameterObjects = /* @__PURE__ */ new Map();
// Maps parameter names to their objects
parametersDefined = false;
initialValuesSynced = false;
// Track if initial values have been synced from host
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Message posting callback
postMessageCallback;
constructor(postMessageCallback) {
this.postMessageCallback = postMessageCallback;
}
// Parameter helper function implementations (return parameter objects)
createSliderParameter(defaultValue, config) {
const paramName = config.label;
const sliderObject = {
value: defaultValue,
min: config.min ?? 0,
max: config.max ?? 100,
step: config.step ?? 1,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "slider",
defaultValue,
label: sliderObject.label,
description: sliderObject.description,
group: sliderObject.group,
category: sliderObject.category,
config: {
min: sliderObject.min,
max: sliderObject.max,
step: sliderObject.step
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, sliderObject);
return sliderObject;
}
createColorParameter(defaultValue, config) {
const paramName = config.label;
const colorObject = {
value: defaultValue,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "color",
defaultValue,
label: colorObject.label,
description: colorObject.description,
group: colorObject.group,
category: colorObject.category
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, colorObject);
return colorObject;
}
createToggleParameter(defaultValue, config) {
const paramName = config.label;
const toggleObject = {
value: defaultValue,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "toggle",
defaultValue,
label: toggleObject.label,
description: toggleObject.description,
group: toggleObject.group,
category: toggleObject.category
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, toggleObject);
return toggleObject;
}
createSelectParameter(defaultValue, config) {
const paramName = config.label;
const selectObject = {
value: defaultValue,
options: config.options,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "select",
defaultValue,
label: selectObject.label,
description: selectObject.description,
group: selectObject.group,
category: selectObject.category,
config: {
options: selectObject.options
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, selectObject);
return selectObject;
}
createTextParameter(defaultValue, config) {
const paramName = config.label;
const textObject = {
value: defaultValue,
maxLength: config.maxLength ?? 1e3,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "text",
defaultValue,
label: textObject.label,
description: textObject.description,
group: textObject.group,
category: textObject.category,
config: {
maxLength: textObject.maxLength
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, textObject);
return textObject;
}
createNumberParameter(defaultValue, config) {
const paramName = config.label;
const numberObject = {
value: defaultValue,
min: config.min ?? 0,
max: config.max ?? 100,
step: config.step ?? 1,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "number",
defaultValue,
label: numberObject.label,
description: numberObject.description,
group: numberObject.group,
category: numberObject.category,
config: {
min: numberObject.min,
max: numberObject.max,
step: numberObject.step
}
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, numberObject);
return numberObject;
}
createImageParameter(defaultValue, config) {
const paramName = config.label;
const imageObject = {
value: defaultValue,
label: config.label,
description: config.description ?? "",
group: config.group ?? "general",
category: config.category ?? "general"
};
const definition = {
type: "image",
defaultValue,
label: imageObject.label,
description: imageObject.description,
group: imageObject.group,
category: imageObject.category
};
this.storeParameterDefinition(paramName, definition);
this.parameterObjects.set(paramName, imageObject);
return imageObject;
}
storeParameterDefinition(name, definition) {
this.parameterDefinitions.set(name, definition);
this.parameterValues.set(name, definition.defaultValue);
}
updateParameterValue(name, value) {
const definition = this.parameterDefinitions.get(name);
if (!definition) {
console.warn(`Unknown parameter: ${name}. Available parameters:`, Array.from(this.parameterDefinitions.keys()));
return false;
}
if (!this.validateParameterValue(name, value, definition)) {
console.warn(`Validation failed for parameter ${name} = ${value}`);
return false;
}
const currentValue = this.parameterValues.get(name);
const isInitialSync = !this.initialValuesSynced;
if (currentValue === value && !isInitialSync) {
return false;
}
this.parameterValues.set(name, value);
const parameterObject = this.parameterObjects.get(name);
if (parameterObject) {
parameterObject.value = value;
}
return true;
}
validateParameterValue(name, value, definition) {
if (definition.validate && !definition.validate(value)) {
console.error(`Custom validation failed for parameter '${name}': ${value}`);
return false;
}
switch (definition.type) {
case "slider":
case "number":
if (typeof value !== "number" || isNaN(value)) {
console.error(`Parameter '${name}' must be a number, got: ${value}`);
return false;
}
if (definition.config?.min !== void 0 && value < definition.config.min) {
console.error(`Parameter '${name}' value ${value} is below minimum ${definition.config.min}`);
return false;
}
if (definition.config?.max !== void 0 && value > definition.config.max) {
console.error(`Parameter '${name}' value ${value} is above maximum ${definition.config.max}`);
return false;
}
break;
case "color":
if (typeof value !== "string" || !/^#[0-9A-Fa-f]{6}$/.test(value)) {
console.error(`Parameter '${name}' must be a valid hex color, got: ${value}`);
return false;
}
break;
case "toggle":
if (typeof value !== "boolean") {
console.error(`Parameter '${name}' must be a boolean, got: ${value}`);
return false;
}
break;
case "select":
if (!definition.config?.options || !definition.config.options.includes(value)) {
console.error(`Parameter '${name}' value ${value} is not in options: ${definition.config?.options}`);
return false;
}
break;
case "text":
if (typeof value !== "string") {
console.error(`Parameter '${name}' must be a string, got: ${value}`);
return false;
}
if (definition.config?.maxLength && value.length > definition.config.maxLength) {
console.error(`Parameter '${name}' text too long: ${value.length} > ${definition.config.maxLength}`);
return false;
}
break;
case "image":
if (value !== null && !(value instanceof ImageBitmap) && !(value instanceof OffscreenCanvas)) {
console.error(`Parameter '${name}' must be null, ImageBitmap, or OffscreenCanvas, got: ${value}`);
return false;
}
break;
}
return true;
}
// Reset parameter state (called when loading new scene)
resetParameterState() {
this.parametersDefined = false;
this.initialValuesSynced = false;
this.parameterDefinitions.clear();
this.parameterGroups.clear();
this.parameterValues.clear();
this.parameterObjects.clear();
}
// Send all parameters (from helper functions) to host
sendAllParametersToHost() {
if (this.parametersDefined || this.parameterDefinitions.size === 0) {
return;
}
try {
const groups = /* @__PURE__ */ new Map();
for (const [paramName, paramDef] of this.parameterDefinitions) {
const groupName = paramDef.group || "general";
if (!groups.has(groupName)) {
const category = paramDef.category || "general";
groups.set(groupName, {
groupName,
category,
parameters: {}
});
}
const group = groups.get(groupName);
group.parameters[paramName] = paramDef;
}
this.parametersDefined = true;
this.postMessageCallback("parameters-defined", {
groups: Array.from(groups.values())
});
this.debugLog(`All parameters sent to host: ${this.parameterDefinitions.size} parameters in ${groups.size} groups`);
} catch (error) {
this.postMessageCallback("parameter-validation-error", {
message: `Failed to send parameters to host: ${error.message}`,
code: "PARAMETER_SENDING_ERROR"
});
}
}
// Mark initial values as synced
markInitialValuesSynced() {
this.initialValuesSynced = true;
}
// Get parameter count for performance reporting
getParameterCount() {
return this.parameterDefinitions.size;
}
// Get all parameter objects (for P5 adapter to add .p5 properties)
getAllParameterObjects() {
return this.parameterObjects;
}
}
class InteractionSystem {
// Interaction enabled state
isEnabled = true;
// Mouse interaction state
mouseState = {
x: 0,
y: 0,
isInCanvas: false,
isPressed: false,
leftButton: false,
rightButton: false,
middleButton: false,
velocity: { x: 0, y: 0 },
deltaX: 0,
deltaY: 0,
wheelDelta: 0,
wheelX: 0,
wheelY: 0,
wasPressed: false,
wasReleased: false,
wasMoved: false
};
// Keyboard interaction state
keyboardState = {
isPressed: (key) => this.keyboardState.activeKeys.has(key.toLowerCase()),
wasPressed: (key) => this.keyboardState.pressedThisFrame.has(key.toLowerCase()),
wasReleased: (key) => this.keyboardState.releasedThisFrame.has(key.toLowerCase()),
activeKeys: /* @__PURE__ */ new Set(),
pressedThisFrame: /* @__PURE__ */ new Set(),
releasedThisFrame: /* @__PURE__ */ new Set(),
lastKeyPressed: "",
lastKeyReleased: "",
shift: false,
ctrl: false,
alt: false,
meta: false
};
// Touch interaction state
touchState = {
points: [],
count: 0,
started: [],
moved: [],
ended: [],
primary: null,
gestures: {
isPinching: false,
isRotating: false,
isPanning: false,
isTapping: false,
pinchScale: 1,
pinchDelta: 0,
rotationAngle: 0,
rotationDelta: 0,
panDelta: { x: 0, y: 0 },
tapCount: 0,
lastTapTime: 0,
tapPosition: null
}
};
constructor() {
this.handleMouseUpdate = this.handleMouseUpdate.bind(this);
this.handleKeyboardUpdate = this.handleKeyboardUpdate.bind(this);
this.handleTouchUpdate = this.handleTouchUpdate.bind(this);
this.frameStart = this.frameStart.bind(this);
}
/**
* Get the interaction APIs for inclusion in the viji object
*/
getInteractionAPIs() {
return {
mouse: this.mouseState,
keyboard: this.keyboardState,
touches: this.touchState
};
}
/**
* Called at the start of each frame to reset frame-based events
*/
frameStart() {
this.mouseState.wasPressed = false;
this.mouseState.wasReleased = false;
this.mouseState.wasMoved = false;
this.mouseState.wheelDelta = 0;
this.mouseState.wheelX = 0;
this.mouseState.wheelY = 0;
this.keyboardState.pressedThisFrame.clear();
this.keyboardState.releasedThisFrame.clear();
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
this.touchState.gestures.isTapping = false;
this.touchState.gestures.pinchDelta = 0;
this.touchState.gestures.rotationDelta = 0;
}
/**
* Handle mouse update messages from the host
*/
handleMouseUpdate(data) {
if (!this.isEnabled) return;
this.mouseState.x = data.x;
this.mouseState.y = data.y;
this.mouseState.isInCanvas = data.isInCanvas !== void 0 ? data.isInCanvas : true;
this.mouseState.leftButton = (data.buttons & 1) !== 0;
this.mouseState.rightButton = (data.buttons & 2) !== 0;
this.mouseState.middleButton = (data.buttons & 4) !== 0;
this.mouseState.isPressed = data.buttons > 0;
this.mouseState.deltaX = data.deltaX || 0;
this.mouseState.deltaY = data.deltaY || 0;
this.mouseState.wheelDelta = data.wheelDeltaY || 0;
this.mouseState.wheelX = data.wheelDeltaX || 0;
this.mouseState.wheelY = data.wheelDeltaY || 0;
this.mouseState.velocity.x = data.deltaX || 0;
this.mouseState.velocity.y = data.deltaY || 0;
this.mouseState.wasPressed = data.wasPressed || false;
this.mouseState.wasReleased = data.wasReleased || false;
this.mouseState.wasMoved = data.deltaX !== 0 || data.deltaY !== 0;
}
/**
* Handle keyboard update messages from the host
*/
handleKeyboardUpdate(data) {
if (!this.isEnabled) return;
const key = data.key.toLowerCase();
if (data.type === "keydown") {
if (!this.keyboardState.activeKeys.has(key)) {
this.keyboardState.activeKeys.add(key);
this.keyboardState.pressedThisFrame.add(key);
this.keyboardState.lastKeyPressed = data.key;
}
} else if (data.type === "keyup") {
this.keyboardState.activeKeys.delete(key);
this.keyboardState.releasedThisFrame.add(key);
this.keyboardState.lastKeyReleased = data.key;
}
this.keyboardState.shift = data.shiftKey;
this.keyboardState.ctrl = data.ctrlKey;
this.keyboardState.alt = data.altKey;
this.keyboardState.meta = data.metaKey;
}
/**
* Handle touch update messages from the host
*/
handleTouchUpdate(data) {
if (!this.isEnabled) return;
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
const touches = data.touches.map((touch) => ({
id: touch.identifier,
x: touch.clientX,
y: touch.clientY,
pressure: touch.pressure || 0,
radius: Math.max(touch.radiusX || 0, touch.radiusY || 0),
radiusX: touch.radiusX || 0,
radiusY: touch.radiusY || 0,
rotationAngle: touch.rotationAngle || 0,
force: touch.force || touch.pressure || 0,
deltaX: 0,
// Could be calculated if we track previous positions
deltaY: 0,
velocity: { x: 0, y: 0 },
// Could be calculated if we track movement
isNew: data.type === "touchstart",
isActive: true,
isEnding: data.type === "touchend" || data.type === "touchcancel"
}));
this.touchState.points = touches;
this.touchState.count = touches.length;
this.touchState.primary = touches[0] || null;
if (data.type === "touchstart") {
this.touchState.started = touches;
} else if (data.type === "touchmove") {
this.touchState.moved = touches;
} else if (data.type === "touchend" || data.type === "touchcancel") {
this.touchState.ended = touches;
}
this.touchState.gestures = {
isPinching: false,
isRotating: false,
isPanning: false,
isTapping: false,
pinchScale: 1,
pinchDelta: 0,
rotationAngle: 0,
rotationDelta: 0,
panDelta: { x: 0, y: 0 },
tapCount: 0,
lastTapTime: 0,
tapPosition: null
};
}
/**
* Reset all interaction state (called when loading new scene)
*/
resetInteractionState() {
Object.assign(this.mouseState, {
x: 0,
y: 0,
isInCanvas: false,
isPressed: false,
leftButton: false,
rightButton: false,
middleButton: false,
velocity: { x: 0, y: 0 },
deltaX: 0,
deltaY: 0,
wheelDelta: 0,
wheelX: 0,
wheelY: 0,
wasPressed: false,
wasReleased: false,
wasMoved: false
});
this.keyboardState.activeKeys.clear();
this.keyboardState.pressedThisFrame.clear();
this.keyboardState.releasedThisFrame.clear();
this.keyboardState.lastKeyPressed = "";
this.keyboardState.lastKeyReleased = "";
this.keyboardState.shift = false;
this.keyboardState.ctrl = false;
this.keyboardState.alt = false;
this.keyboardState.meta = false;
this.touchState.points = [];
this.touchState.count = 0;
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
this.touchState.primary = null;
Object.assign(this.touchState.gestures, {
isPinching: false,
isRotating: false,
isPanning: false,
isTapping: false,
pinchScale: 1,
pinchDelta: 0,
rotationAngle: 0,
rotationDelta: 0,
panDelta: { x: 0, y: 0 },
tapCount: 0,
lastTapTime: 0,
tapPosition: null
});
}
/**
* Enable or disable interaction processing
*/
setInteractionEnabled(enabled) {
this.isEnabled = enabled;
if (!enabled) {
this.resetInteractionStates();
}
}
/**
* Get current interaction enabled state
*/
getInteractionEnabled() {
return this.isEnabled;
}
/**
* Reset all interaction states to default values
*/
resetInteractionStates() {
this.mouseState.x = 0;
this.mouseState.y = 0;
this.mouseState.isInCanvas = false;
this.mouseState.isPressed = false;
this.mouseState.leftButton = false;
this.mouseState.rightButton = false;
this.mouseState.middleButton = false;
this.mouseState.velocity.x = 0;
this.mouseState.velocity.y = 0;
this.mouseState.deltaX = 0;
this.mouseState.deltaY = 0;
this.mouseState.wheelDelta = 0;
this.mouseState.wheelX = 0;
this.mouseState.wheelY = 0;
this.mouseState.wasPressed = false;
this.mouseState.wasReleased = false;
this.mouseState.wasMoved = false;
this.keyboardState.activeKeys.clear();
this.keyboardState.pressedThisFrame.clear();
this.keyboardState.releasedThisFrame.clear();
this.keyboardState.lastKeyPressed = "";
this.keyboardState.lastKeyReleased = "";
this.keyboardState.shift = false;
this.keyboardState.ctrl = false;
this.keyboardState.alt = false;
this.keyboardState.meta = false;
this.touchState.points = [];
this.touchState.count = 0;
this.touchState.started = [];
this.touchState.moved = [];
this.touchState.ended = [];
this.touchState.primary = null;
this.touchState.gestures.isPinching = false;
this.touchState.gestures.isRotating = false;
this.touchState.gestures.isPanning = false;
this.touchState.gestures.isTapping = false;
this.touchState.gestures.pinchScale = 1;
this.touchState.gestures.pinchDelta = 0;
this.touchState.gestures.rotationAngle = 0;
this.touchState.gestures.rotationDelta = 0;
this.touchState.gestures.panDelta = { x: 0, y: 0 };
this.touchState.gestures.tapCount = 0;
this.touchState.gestures.lastTapTime = 0;
this.touchState.gestures.tapPosition = null;
}
}
class CVSystem {
// MediaPipe Tasks Vision worker
cvWorker = null;
workerRestartCount = 0;
maxWorkerRestarts = 3;
workerLastRestart = 0;
workerRestartCooldown = 5e3;
// 5 seconds
// Feature activation state
activeFeatures = /* @__PURE__ */ new Set();
pendingFeatures = /* @__PURE__ */ new Set();
// Features to restore after restart
// CV Results cache (for non-blocking processing)
results = {
faces: [],
hands: [],
pose: null,
segmentation: null
};
// Processing state and performance tracking
processing = false;
cvFrameCounter = 0;
cvFrameRateMode = "quarter";
// Default: 1/4 scene rate
sceneTargetFPS = 60;
// Will be updated from scene processing rate
processingStartTime = 0;
processingTimes = [];
// CV Frame Rate Tracking (similar to main core)
cvFrameTimes = [];
lastCVFrameTime = 0;
actualCVFPS = 0;
debugMode = false;
// Debug mode disabled for production
constructor() {
this.debugLog("🔧 CVSystem initialized for MediaPipe Tasks Vision");
}
debugLog(...args) {
if (this.debugMode) {
console.log("🔧 [CVSystem]", ...args);
}
}
setDebugMode(enabled) {
this.debugMode = enabled;
this.debugLog(`Debug mode ${enabled ? "enabled" : "disabled"}`);
}
/**
* Update CV frame rate configuration (called from worker)
*/
updateCVFrameRate(mode, sceneTargetFPS) {
this.cvFrameRateMode = mode;
this.sceneTargetFPS = sceneTargetFPS;
this.debugLog(`CV frame rate updated: mode=${mode}, sceneTargetFPS=${sceneTargetFPS}`);
}
/**
* Initialize MediaPipe Tasks Vision worker
*/
async ensureCVWorker() {
if (this.cvWorker) return;
try {
this.debugLog("🔧 Creating MediaPipe Tasks Vision worker...");
const workerUrl = "/dist/assets/cv-tasks.worker.js";
this.cvWorker = new Worker(workerUrl);
this.cvWorker.addEventListener("message", (evt) => {
const msg = evt.data;
this.debugLog(`📨 [CV Worker -> CVSystem] ${msg.type}`, msg.success ? msg.data : msg.error);
});
this.cvWorker.onerror = (err) => {
this.debugLog("❌ CV worker error event", err.message);
this.handleWorkerFailure(`Worker error: ${err.message}`);
};
this.cvWorker.onmessageerror = (err) => {
this.debugLog("❌ CV worker message error", err);
this.handleWorkerFailure("Worker message error");
};
await this.postToCV("init", {});
this.debugLog("✅ CV worker initialized");
} catch (error) {
this.debugLog("❌ Failed to initialize CV worker:", error);
throw error;
}
}
/**
* Handle worker failure and attempt restart
*/
async handleWorkerFailure(reason) {
this.debugLog(`⚠️ CV Worker failure: ${reason}`);
if (this.cvWorker) {
this.cvWorker.terminate();
this.cvWorker = null;
}
const now = Date.now();
if (this.workerRestartCount >= this.maxWorkerRestarts) {
this.debugLog("❌ Max worker restarts exceeded, giving up");
return;
}
if (now - this.workerLastRestart < this.workerRestartCooldown) {
this.debugLog("⏱️ Worker restart cooldown active, skipping restart");
return;
}
this.pendingFeatures = new Set(this.activeFeatures);
this.activeFeatures.clear();
try {
this.workerRestartCount++;
this.workerLastRestart = now;
this.debugLog(`🔄 Restarting CV worker (attempt ${this.workerRestartCount}/${this.maxWorkerRestarts})`);
await this.ensureCVWorker();
if (this.pendingFeatures.size > 0) {
const featuresToRestore = Array.from(this.pendingFeatures);
this.debugLog(`🔄 Restoring features: [${featuresToRestore.join(", ")}]`);
try {
await this.postToCV("config", { features: featuresToRestore });
this.activeFeatures = new Set(this.pendingFeatures);
this.debugLog("✅ Features restored successfully");
} catch (error) {
this.debugLog("❌ Failed to restore features:", error);
}
this.pendingFeatures.clear();
}
this.debugLog("✅ CV worker restarted successfully");
} catch (error) {
this.debugLog("❌ Failed to restart CV worker:", error);
}
}
/**
* Send message to CV worker and wait for response
*/
postToCV(type, data, transfer) {
return new Promise((resolve, reject) => {
if (!this.cvWorker) {
return reject(new Error("CV worker not initialized"));
}
const timeout = setTimeout(() => {
this.debugLog(`⏱️ [CV Worker] timeout for ${type}`);
if (type === "config") {
this.handleWorkerFailure(`Timeout for ${type} message`);
}
reject(new Error(`CV worker timeout for ${type}`));
}, 5e3);
const onMessage = (ev) => {
const msg = ev.data;
if (msg.type === "result") {
clearTimeout(timeout);
this.cvWorker.removeEventListener("message", onMessage);
if (msg.success) {
this.debugLog(`✅ [CV Worker] response for ${type}`, msg.data);
resolve(msg.data);
} else {
this.debugLog(`❌ [CV Worker] error response for ${type}`, msg.error);
if (msg.restartRequired) {
this.handleWorkerFailure(`Worker reported restart required: ${msg.error}`);
}
reject(new Error(msg.error || "CV worker error"));
}
}
};
this.cvWorker.addEventListener("message", onMessage);
const message = {
type,
...data
};
this.debugLog(`📤 [CVSystem -> CV Worker] ${type}`, data);
this.cvWorker.postMessage(message, transfer || []);
});
}
/**
* Enable face detection feature (bounding boxes only)
*/
async enableFaceDetection() {
if (this.activeFeatures.has("faceDetection")) return;
try {
this.debugLog("🔧 Enabling face detection...");
if (this.activeFeatures.size >= 3 && !this.checkWebGLContextAvailability()) {
this.debugLog("⚠️ Warning: WebGL contexts may be running low. Consider disabling unused CV features.");
}
await this.ensureCVWorker();
const newFeatures = Array.from(this.activeFeatures).concat(["faceDetection"]);
await this.postToCV("config", { features: newFeatures });
this.activeFeatures.add("faceDetection");
this.debugLog("✅ Face detection enabled");
} catch (error) {
this.debugLog("❌ Failed to enable face detection:", error);
throw error;
}
}
/**
* Disable face detection and cleanup
*/
async disableFaceDetection() {
if (!this.activeFeatures.has("faceDetection")) return;
try {
this.debugLog("🔧 Disabling face detection...");
this.activeFeatures.delete("faceDetection");
if (!this.activeFeatures.has("faceMesh")) {
this.results.faces = [];
}
const newFeatures = Array.from(this.activeFeatures);
await this.postToCV("config", { features: newFeatures });
this.debugLog("✅ Face detection disabled and cleaned up");
} catch (error) {
this.debugLog("❌ Failed to disable face detection:", error);
throw error;
}
}
/**
* Enable face mesh feature (468-point facial landmarks)
*/
async enableFaceMesh() {
if (this.activeFeatures.has("faceMesh")) return;
try {
this.debugLog("🔧 Enabling face mesh...");
if (this.activeFeatures.size >= 3 && !this.checkWebGLContextAvailability()) {
this.debugLog("⚠️ Warning: WebGL contexts may be running low. Consider disabling unused CV features.");
}
await this.ensureCVWorker();
const newFeatures = Array.from(this.activeFeatures).concat(["faceMesh"]);
await this.postToCV("config", { features: newFeatures });
this.activeFeatures.add("faceMesh");
this.debugLog("✅ Face mesh enabled");
} catch (error) {
this.debugLog("❌ Failed to enable face mesh:", error);
throw error;
}
}
/**
* Disable face mesh and cleanup
*/
async disableFaceMesh() {
if (!this.activeFeatures.has("faceMesh")) return;
try {
this.debugLog("🔧 Disabling face mesh...");
this.activeFeatures.delete("faceMesh");
if (!this.activeFeatures.has("faceDetection")) {
this.results.faces = [];
}
const newFeatures = Array.from(this.activeFeatures);
await this.postToCV("config", { features: newFeatures });
this.debugLog("✅ Face mesh disabled and cleaned up");
} catch (error) {
this.debugLog("❌ Failed to disable face mesh:", error);
throw error;
}
}
/**
* Enable hand tracking feature
*/
async enableHandTracking() {
if (this.activeFeatures.has("handTracking")) return;
try {
this.debugLog("🔧 Enabling hand tracking...");
await this.ensureCVWorker();
const newFeatures = Array.from(this.activeFeatures).concat(["handTracking"]);
await this.postToCV("config", { features: newFeatures });
this.activeFeatures.add("handTracking");
this.debugLog("✅ Hand tracking enabled");
} catch (error) {
this.debugLog("❌ Failed to enable hand tracking:", error);
throw error;
}
}
/**
* Disable hand tracking and cleanup
*/
async disableHandTracking() {
if (!this.activeFeatures.has("handTracking")) return;
try {
this.debugLog("🔧 Disabling hand tracking...");
this.activeFeatures.delete("handTracking");
this.results.hands = [];
const newFeatures = Array.from(this.activeFeatures);
await this.postToCV("config", { features: newFeatures });
this.debugLog("✅ Hand tracking disabled and cleaned up");
} catch (error) {
this.debugLog("❌ Failed to disable hand tracking:", error);
throw error;
}
}
/**
* Enable pose detection feature
*/
async enablePoseDetection() {
if (this.activeFeatures.has("poseDetection")) return;
try {
this.debugLog("🔧 Enabling pose detection...");
await this.ensureCVWorker();
const newFeatures = Array.from(this.activeFeatures).concat(["poseDetection"]);
await this.postToCV("config", { features: newFeatures });
this.activeFeatures.add("poseDetection");
this.debugLog("✅ Pose detection enabled");
} catch (error) {
this.debugLog("❌ Failed to enable pose detection:", error);
throw error;
}
}
/**
* Disable pose detection and cleanup
*/
async disablePoseDetection() {
if (!this.activeFeatures.has("poseDetection")) return;
try {
this.debugLog("🔧 Disabling pose detection...");
this.activeFeatures.delete("poseDetection");
this.results.pose = null;
const newFeatures = Array.from(this.activeFeatures);
await this.postToCV("config", { features: newFeatures });
this.debugLog("✅ Pose detection disabled and cleaned up");
} catch (error) {
this.debugLog("❌ Failed to disable pose detection:", error);
throw error;
}
}
/**
* Enable body segmentation feature
*/
async enableBodySegmentation() {
if (this.activeFeatures.has("bodySegmentation")) return;
try {
this.debugLog("🔧 Enabling body segmentation...");
await this.ensureCVWorker();
const newFeatures = Array.from(this.activeFeatures).concat(["bodySegmentation"]);
await this.postToCV("config", { features: newFeatures });
this.activeFeatures.add("bodySegmentation");
this.debugLog("✅ Body segmentation enabled");
} catch (error) {
this.debugLog("❌ Failed to enable body segmentation:", error);
throw error;
}
}
/**
* Disable body segmentation and cleanup
*/
async disableBodySegmentation() {
if (!this.activeFeatures.has("bodySegmentation")) return;
try {
this.debugLog("🔧 Disabling body segmentation...");
this.activeFeatures.delete("bodySegmentation");
this.results.segmentation = null;
const newFeatures = Array.from(this.activeFeatures);
await this.postToCV("config", { features: newFeatures });
this.debugLog("✅ Body segmentation disabled and cleaned up");
} catch (error) {
this.debugLog("❌ Failed to disable body segmentation:", error);
throw error;
}
}
/**
* Process video frame with active CV features
*/
async processFrame(bitmap) {
if (this.processing || this.activeFeatures.size === 0) {
return;
}
this.cvFrameCounter++;
const shouldProcess = this.shouldProcessFrame();
if (!shouldProcess) {
return;
}
this.processing = true;
this.processingStartTime = performance.now();
this.trackCVFrameRate();
this.debugLog(`🎬 Processing frame ${this.cvFrameCounter} with features:`, Array.from(this.activeFeatures));
try {
const features = Array.from(this.activeFeatures);
const timestamp = performance.now();
const processPromise = this.postToCV("process", {
bitmap,
timestamp,
features
}, [bitmap]);
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => reject(new Error("CV processing timeout")), 500);
});
const results = await Promise.race([processPromise, timeoutPromise]);
if (results.faces && (this.activeFeatures.has("faceDetection") || this.activeFeatures.has("faceMesh"))) {
this.results.faces = results.faces;
this.debugLog(`📥 Received ${results.faces.length} face results`);
}
if (results.hands && this.activeFeatures.has("handTracking")) {
this.results.hands = results.hands;
this.debugLog(`📥 Received ${results.hands.length} hand results`);
}
if (results.pose && this.activeFeatures.has("poseDetection")) {
this.results.pose = results.pose;
this.debugLog(`📥 Received pose results with ${results.pose.landmarks.length} landmarks`);
}
if (results.segmentation && this.activeFeatures.has("bodySegmentation")) {
this.results.segmentation = results.segmentation;
this.debugLog(`📥 Received segmentation results ${results.segmentation.width}x${results.segmentation.height}`);
}
const processingTime = performance.now() - this.processingStartTime;
this.processingTimes.push(processingTime);
if (this.processingTimes.length > 30) {
this.processingTimes.shift();
}
} catch (error) {
this.debugLog("⚠️ CV processing failed:", error);
} finally {
this.processing = false;
}
}
/**
* Check if current frame should be processed based on CV frame rate mode
*/
shouldProcessFrame() {
const divisor = this.getFrameRateDivisor();
return this.cvFrameCounter % divisor === 0;
}
/**
* Track CV processing frame rate (similar to main core)
*/
trackCVFrameRate() {
const now = performance.now();
if (this.lastCVFrameTime > 0) {
const deltaTime = now - this.lastCVFrameTime;
this.cvFrameTimes.push(deltaTime);
if (this.cvFrameTimes.length > 30) {
this.cvFrameTimes.shift();
}
if (this.cvFrameTimes.length > 0) {
const avgDeltaTime = this.cvFrameTimes.reduce((a, b) => a + b, 0) / this.cvFrameTimes.length;
this.actualCVFPS = Math.round(1e3 / avgDeltaTime);
}
}
this.lastCVFrameTime = now;
}
/**
* Get frame rate divisor based on current mode
*/
getFrameRateDivisor() {
switch (this.cvFrameRateMode) {
case "full":
return 1;
case "half":
return 2;
case "quarter":
return 4;
case "eighth":
return 8;
default:
return 4;
}
}
/**
* Get current CV results
*/
getResults() {
return { ...this.results };
}
/**
* Get processing statistics
*/
getStats() {
const avgProcessingTime = this.processingTimes.length > 0 ? this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length : 0;
const targetFPS = this.sceneTargetFPS / this.getFrameRateDivisor();
return {
activeFeatures: Array.from(this.activeFeatures),
processingTime: avgProcessingTime,
effectiveFPS: targetFPS,
actualFPS: this.actualCVFPS,
// Add actual measured CV FPS
isProcessing: this.processing
};
}
/**
* Check WebGL context availability before enabling features
*/
checkWebGLContextAvailability() {
try {
const canvas = new OffscreenCanvas(1, 1);
const gl = canvas.getContext("webgl");
if (!gl) {
this.debugLog("⚠️ WebGL contexts may be exhausted");
return false;
}
const ext = gl.getExtension("WEBGL_lose_context");
if (ext) ext.loseContext();
return true;
} catch (error) {
this.debugLog("⚠️ WebGL context check failed:", error);
return false;
}
}
/**
* Get CV control interface for artist API
*/
getControlInterface() {
return {
enableFaceDetection: (enabled) => {
if (enabled === false) {
return this.disableFaceDetection();
} else {
return this.enableFaceDetection();
}
},
disableFaceDetection: () => this.disableFaceDetection(),
enableFaceMesh: (enabled) => {
if (enabled === false) {
return this.disableFaceMesh();
} else {
return this.enableFaceMesh();
}
},
disableFaceMesh: () => this.disableFaceMesh(),
enableHandTracking: (enabled) => {
if (enabled === false) {
return this.disableHandTracking();
} else {
return this.enableHandTracking();
}
},
disableHandTracking: () => this.disableHandTracking(),
enablePoseDetection: (enabled) => {
if (enabled === false) {
return this.disablePoseDetection();
} else {
return this.enablePoseDetection();
}
},
disablePoseDetection: () => this.disablePoseDetection(),
enableBodySegmentation: (enabled) => {
if (enabled === false) {
return this.disableBodySegmentation();
} else {
return this.enableBodySegmentation();
}
},
disableBodySegmentation: () => this.disableBodySegmentation(),
getActiveFeatures: () => Array.from(this.activeFeatures),
isProcessing: () => this.processing,
getStats: () => this.getStats(),
getWorkerStatus: () => ({
healthy: !!this.cvWorker,
restartCount: this.workerRestartCount,
maxRestarts: this.maxWorkerRestarts
}),
restartWorker: () => this.handleWorkerFailure("Manual restart requested"),
// WebGL context monitoring
checkWebGLAvailability: () => this.checkWebGLContextAvailability(),
getResourceUsage: () => ({
activeFeatures: this.activeFeatures.size,
estimatedWebGLContexts: this.activeFeatures.size * 2,
// ~2 contexts per feature
webglAvailable: this.checkWebGLContextAvailability()
})
};
}
/**
* Cleanup all CV resources
*/
async cleanup() {
this.debugLog("🔧 Cleaning up CVSystem...");
for (const feature of Array.from(this.activeFeatures)) {
switch (feature) {
case "faceDetection":
await this.disableFaceDetection();
break;
case "handTracking":
await this.disableHandTracking();
break;
case "poseDetection":
await this.disablePoseDetection();
break;
case "bodySegmentation":
await this.disableBodySegmentation();
break;
}
}
if (this.cvWorker) {
this.cvWorker.terminate();
this.cvWorker = null;
}
this.activeFeatures.clear();
this.pendingFeatures.clear();
this.workerRestartCount = 0;
this.results.faces = [];
this.results.hands = [];
this.results.pose = null;
this.results.segmentation = null;
this.processing = false;
this.processingTimes = [];
this.debugLog("✅ CVSystem cleanup complete");
}
}
class VideoSystem {
// ✅ CORRECT: Worker-owned OffscreenCanvas (transferred from host)
offscreenCanvas = null;
ctx = null;
gl = null;
// CV processing helpers
cvScratchCanvas = null;
cvScratchContext = null;
// Debug logging control
debugMode = false;
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
if (this.cvSystem) {
this.cvSystem.setDebugMode(enabled);
}
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Frame processing configuration
targetFrameRate = 30;
// Default target FPS for video processing
lastFrameTime = 0;
frameInterval = 1e3 / this.targetFrameRate;
// ms between frames
// Processing state
hasLoggedFirstFrame = false;
frameCount = 0;
// Video state for artist API
videoState = {
isConnected: false,
currentFrame: null,
frameWidth: 0,
frameHeight: 0,
frameRate: 0,
frameData: null
};
// Phase 11 - CV System Integration
cvSystem;
constructor() {
this.cvSystem = new CVSystem();
}
/**
* Get the video API for inclusion in the viji object
*/
getVideoAPI() {
const cvResults = this.cvSystem.getResults();
return {
isConnected: this.videoState.isConnected,
currentFrame: this.videoState.currentFrame,
frameWidth: this.videoState.frameWidth,
frameHeight: this.videoState.frameHeight,
frameRate: this.videoState.frameRate,
getFrameData: () => this.videoState.frameData,
// CV Results from CVSystem
faces: cvResults.faces,
hands: cvResults.hands,
pose: cvResults.pose,
segmentation: cvResults.segmentation,
// CV Control Interface
cv: this.cvSystem.getControlInterface()
};
}
/**
* ✅ CORRECT: Receive OffscreenCanvas transfer from host
*/
handleCanvasSetup(data) {
try {
this.disconnectVideo();
this.offscreenCanvas = data.offscreenCanvas;
this.ctx = this.offscreenCanvas.getContext("2d", {
willReadFrequently: true
// Optimize for frequent getImageData calls
});
if (!this.ctx) {
throw new Error("Failed to get 2D context from transferred OffscreenCanvas");
}
try {
this.gl = this.offscreenCanvas.getContext("webgl2") || this.offscreenCanvas.getContext("webgl");
} catch (e) {
this.debugLog("WebGL not available, using 2D context only");
}
this.videoState.isConnected = true;
this.videoState.currentFrame = this.offscreenCanvas;
this.videoState.frameWidth = data.width;
this.videoState.frameHeight = data.height;
this.frameCount = 0;
this.hasLoggedFirstFrame = false;
this.debugLog("✅ OffscreenCanvas received and setup completed (worker-side)", {
width: data.width,
height: data.height,
hasWebGL: !!this.gl,
targetFrameRate: this.targetFrameRate
});
this.debugLog("🎬 CORRECT OffscreenCanvas approach - Worker has full GPU access!");
} catch (error) {
console.error("Failed to setup OffscreenCanvas in worker:", error);
this.disconnectVideo();
}
}
/**
* ✅ CORRECT: Receive ImageBitmap frame and draw to worker's OffscreenCanvas
*/
handleFrameUpdate(data) {
if (!this.offscreenCanvas || !this.ctx) {
console.warn("🔴 Received frame but OffscreenCanvas not setup");
return;
}
try {
if (this.frameCount % 150 === 0) {
this.debugLog("✅ Worker received ImageBitmap frame:", {
bitmapSize: `${data.imageBitmap.width}x${data.imageBitmap.height}`,
canvasSize: `${this.offscreenCanvas.width}x${this.offscreenCanvas.height}`,
frameCount: this.frameCount,
timestamp: data.timestamp
});
}
this.ctx.drawImage(data.imageBitmap, 0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height);
this.processCurrentFrame(data.timestamp);
data.imageBitmap.close();
this.frameCount++;
} catch (error) {
console.error("🔴 Error processing video frame (worker-side):", error);
}
}
/**
* Process current frame (called when new frame is drawn)
*/
async processCurrentFrame(timestamp) {
if (!this.offscreenCanvas || !this.ctx) {
return;
}
try {
this.videoState.frameData = this.ctx.getImageData(
0,
0,
this.offscreenCanvas.width,
this.offscreenCanvas.height
);
const deltaTime = timestamp - this.lastFrameTime;
this.videoState.frameRate = deltaTime > 0 ? 1e3 / deltaTime : 0;
if (!this.hasLoggedFirstFrame) {
this.debugLog(`🎯 Worker-side OffscreenCanvas processing active: ${this.videoState.frameRate.toFixed(1)} FPS (${this.offscreenCanvas.width}x${this.offscreenCanvas.height})`);
this.debugLog("✅ Full GPU access available for custom effects and CV analysis");
this.hasLoggedFirstFrame = true;
}
if (this.videoState.frameData) {
try {
const bitmap = await createImageBitmap(this.videoState.frameData);
this.cvSystem.processFrame(bitmap);
} catch (bitmapError) {
this.debugLog("⚠️ createImageBitmap failed – falling back to reusable CV canvas:", bitmapError);
if (!this.cvScratchCanvas || !this.cvScratchContext || this.cvScratchCanvas.width !== this.videoState.frameData.width || this.cvScratchCanvas.height !== this.videoState.frameData.height) {
this.cvScratchCanvas = new OffscreenCanvas(
this.videoState.frameData.width,
this.videoState.frameData.height
);
this.cvScratchContext = this.cvScratchCanvas.getContext("2d");
if (!this.cvScratchContext) {
throw new Error("Failed to get 2D context for CV fallback canvas");
}
}
this.cvScratchContext.putImageData(this.videoState.frameData, 0, 0);
const fallbackBitmap = this.cvScratchCanvas.transferToImageBitmap();
this.cvSystem.processFrame(fallbackBitmap);
}
}
this.lastFrameTime = timestamp;
} catch (error) {
console.error("Error processing video frame (worker-side):", error);
}
}
/**
* Handle video configuration updates (including disconnection and resize)
*/
handleVideoConfigUpdate(data) {
try {
if (data.disconnect) {
this.disconnectVideo();
return;
}
if (data.width && data.height && this.offscreenCanvas) {
this.resizeCanvas(data.width, data.height);
}
if (data.targetFrameRate) {
this.updateProcessingConfig(data.targetFrameRate);
}
if (data.cvFrameRate) {
this.updateCVFrameRate(data.cvFrameRate);
}
} catch (error) {
console.error("Error handling video config update:", error);
}
}
/**
* Resize the OffscreenCanvas (when video dimensions change)
*/
resizeCanvas(width, height) {
if (!this.offscreenCanvas) return;
try {
this.offscreenCanvas.width = width;
this.offscreenCanvas.height = height;
this.videoState.frameWidth = width;
this.videoState.frameHeight = height;
if (this.gl) {
this.gl.viewport(0, 0, width, height);
}
this.debugLog(`📐 OffscreenCanvas resized to ${width}x${height} (worker-side)`);
} catch (error) {
console.error("Error resizing OffscreenCanvas:", error);
}
}
/**
* Disconnect video and clean up resources
*/
disconnectVideo() {
if (this.offscreenCanvas && this.ctx) {
this.ctx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height);
this.debugLog("🧹 Cleared OffscreenCanvas on disconnect");
}
this.offscreenCanvas = null;
this.ctx = null;
this.gl = null;
this.videoState.isConnected = false;
this.videoState.currentFrame = null;
this.videoState.frameWidth = 0;
this.videoState.frameHeight = 0;
this.videoState.frameRate = 0;
this.videoState.frameData = null;
this.cvScratchCanvas = null;
this.cvScratchContext = null;
this.hasLoggedFirstFrame = false;
this.frameCount = 0;
this.debugLog("Video disconnected (worker-side)");
}
/**
* Update video processing configuration
*/
updateProcessingConfig(targetFrameRate) {
this.targetFrameRate = Math.max(1, Math.min(60, targetFrameRate));
this.frameInterval = 1e3 / this.targetFrameRate;
this.debugLog(`Video processing frame rate updated to ${this.targetFrameRate} FPS (worker-side)`);
}
/**
* Phase 11 - Update CV frame rate configuration
*/
updateCVFrameRate(cvFrameRate) {
this.cvSystem.updateCVFrameRate(cvFrameRate.mode, cvFrameRate.sceneTargetFPS);
this.debugLog(`CV frame rate updated to ${cvFrameRate.mode} of ${cvFrameRate.sceneTargetFPS} FPS (worker-side)`);
}
/**
* Reset all video state (called when loading new scene)
*/
resetVideoState() {
this.disconnectVideo();
if (this.cvSystem) {
this.cvSystem.cleanup();
}
}
/**
* Get current processing configuration
*/
getProcessingConfig() {
return {
targetFrameRate: this.targetFrameRate,
frameInterval: this.frameInterval,
frameCount: this.frameCount
};
}
/**
* Get WebGL context for advanced effects (if available)
*/
getWebGLContext() {
return this.gl;
}
/**
* ✅ WORKER API: Artists can access the OffscreenCanvas directly for custom effects
*/
getCanvasForArtistEffects() {
return this.offscreenCanvas;
}
/**
* Get CV processing statistics
*/
getCVStats() {
return this.cvSystem.getStats();
}
}
class P5WorkerAdapter {
constructor(offscreenCanvas, _vijiAPI, sceneCode) {
this.offscreenCanvas = offscreenCanvas;
this.setupFn = sceneCode.setup || null;
this.renderFn = sceneCode.render;
this.installMinimalShims();
}
p5Instance = null;
setupFn = null;
renderFn = null;
p5InternalSetupComplete = false;
artistSetupComplete = false;
p5Class = null;
// Cache for converted P5.Image objects
imageParameterCache = /* @__PURE__ */ new Map();
// Track if P5.js's main canvas has been created
mainCanvasCreated = false;
/**
* Initialize P5 instance after P5.js library is loaded
* This must be called after the P5 class is available
*/
async init() {
try {
const p5Module = await import("https://esm.sh/p5@1.9.4");
this.p5Class = p5Module.default || p5Module;
const setupPromise = new Promise((resolve) => {
new this.p5Class((p) => {
this.p5Instance = p;
p.setup = () => {
p.createCanvas(this.offscreenCanvas.width, this.offscreenCanvas.height);
p.noLoop();
this.p5InternalSetupComplete = true;
resolve();
};
p.draw = () => {
};
setTimeout(() => {
if (p.setup && typeof p.setup === "function") {
try {
p.setup();
} catch (setupError) {
console.error("P5 setup failed:", setupError);
resolve();
}
}
}, 0);
});
});
await setupPromise;
} catch (error) {
console.error("Failed to initialize P5.js:", error);
throw error;
}
}
/**
* Install minimal DOM shims that P5.js needs for rendering
*/
installMinimalShims() {
const self2 = globalThis;
if (typeof self2.document === "undefined") {
const createStyleProxy = () => new Proxy({}, {
get: () => "",
set: () => true
});
const bodyElement = {
style: createStyleProxy(),
appendChild: () => {
},
removeChild: () => {
},
children: [],
childNodes: [],
firstChild: null,
lastChild: null,
parentNode: null,
ownerDocument: void 0,
// Will be set after document is created
setAttribute: () => {
},
getAttribute: () => null,
addEventListener: () => {
},
removeEventListener: () => {
},
tagName: "BODY"
};
self2.document = {
createElement: (tag) => {
if (tag === "canvas") {
let canvas;
if (!this.mainCanvasCreated) {
canvas = this.offscreenCanvas;
this.mainCanvasCreated = true;
} else {
canvas = new OffscreenCanvas(300, 300);
}
canvas.style = createStyleProxy();
canvas.dataset = new Proxy({}, {
get: () => void 0,
set: () => true
});
canvas.classList = {
add: () => {
},
remove: () => {
},
contains: () => false,
toggle: () => false
};
canvas.getBoundingClientRect = () => ({
left: 0,
top: 0,
width: canvas.width,
height: canvas.height
});
return canvas;
}
return {
style: createStyleProxy(),
appendChild: () => {
},
removeChild: () => {
},
setAttribute: () => {
},
getAttribute: () => null,
tagName: tag.toUpperCase(),
addEventListener: () => {
},
removeEventListener: () => {
}
};
},
createElementNS: (_ns, tag) => {
return self2.document.createElement(tag);
},
body: bodyElement,
documentElement: {
style: createStyleProxy(),
children: [],
childNodes: []
},
getElementById: () => null,
querySelector: () => null,
querySelectorAll: () => [],
getElementsByTagName: (tagName) => {
if (tagName.toLowerCase() === "main") {
return [bodyElement];
}
return [];
},
addEventListener: () => {
},
removeEventListener: () => {
},
hasFocus: () => true
// P5.js checks this for accessibility features
};
bodyElement.ownerDocument = self2.document;
}
if (typeof self2.window === "undefined") {
self2.window = {
devicePixelRatio: 1,
innerWidth: this.offscreenCanvas.width,
innerHeight: this.offscreenCanvas.height,
addEventListener: () => {
},
removeEventListener: () => {
},
requestAnimationFrame: (_callback) => {
return 0;
},
cancelAnimationFrame: () => {
},
setTimeout: self2.setTimeout.bind(self2),
clearTimeout: self2.clearTimeout.bind(self2),
setInterval: self2.setInterval.bind(self2),
clearInterval: self2.clearInterval.bind(self2),
performance: self2.performance,
console: self2.console,
Math: self2.Math,
Date: self2.Date,
Array: self2.Array,
Object: self2.Object
};
}
if (typeof self2.navigator === "undefined") {
self2.navigator = {
userAgent: "Viji-Worker-P5",
platform: "Worker",
language: "en-US"
};
}
if (typeof self2.screen === "undefined") {
self2.screen = {
width: this.offscreenCanvas.width,
height: this.offscreenCanvas.height,
availWidth: this.offscreenCanvas.width,
availHeight: this.offscreenCanvas.height,
colorDepth: 24,
pixelDepth: 24
};
}
if (typeof self2.HTMLCanvasElement === "undefined") {
self2.HTMLCanvasElement = function() {
};
Object.setPrototypeOf(OffscreenCanvas.prototype, self2.HTMLCanvasElement.prototype);
}
}
/**
* Convert ImageBitmap to a P5.js-compatible image object (with caching)
* Returns an object that mimics P5.Image structure for P5.js's image() function
*/
getOrCreateP5Image(cacheKey, source) {
if (!this.p5Instance) return null;
const cached = this.imageParameterCache.get(cacheKey);
if (cached && cached.source === source) {
return cached.p5Image;
}
try {
const offscreenCanvas = new OffscreenCanvas(source.width, source.height);
const ctx = offscreenCanvas.getContext("2d");
if (!ctx) {
throw new Error("Failed to get 2d context from OffscreenCanvas");
}
ctx.drawImage(source, 0, 0);
const p5ImageWrapper = {
canvas: offscreenCanvas,
// P5.js looks for img.canvas || img.elt
elt: offscreenCanvas,
// Fallback for compatibility
width: source.width,
// Logical width
height: source.height
// Logical height
};
this.imageParameterCache.set(cacheKey, { source, p5Image: p5ImageWrapper });
return p5ImageWrapper;
} catch (error) {
console.warn("Failed to convert image to P5-compatible object:", error);
return null;
}
}
/**
* Add .p5 property to image parameters for P5.js-specific rendering
* This allows artists to use p5.image() while keeping .value for native canvas API
* @param parameterObjects Map of parameter name to parameter object from ParameterSystem
*/
addP5PropertyToImageParameters(parameterObjects) {
if (!this.p5Instance) return;
const isImageLike = (value) => {
return value instanceof ImageBitmap || value instanceof OffscreenCanvas || value && typeof value === "object" && "width" in value && "height" in value;
};
for (const [name, param] of parameterObjects) {
try {
if (param && typeof param === "object" && "value" in param) {
const value = param.value;
if (value && isImageLike(value) && !("p5" in param)) {
Object.defineProperty(param, "p5", {
get: () => this.getOrCreateP5Image(name, param.value),
enumerable: true,
configurable: true
});
}
}
} catch (error) {
console.warn(`Failed to add .p5 property to parameter '${name}':`, error);
continue;
}
}
}
/**
* Execute one frame of the P5 scene
* Called by Viji's render loop
* @param vijiAPI The Viji API object passed to artist code
* @param parameterObjects Map of parameter objects from ParameterSystem
*/
tick(vijiAPI, parameterObjects) {
if (!this.p5Instance || !this.p5InternalSetupComplete) {
return;
}
try {
this.addP5PropertyToImageParameters(parameterObjects);
if (!this.artistSetupComplete && this.setupFn) {
this.setupFn(vijiAPI, this.p5Instance);
this.artistSetupComplete = true;
}
if (this.p5Instance._setProperty) {
this.p5Instance._setProperty("frameCount", this.p5Instance.frameCount + 1);
}
if (this.renderFn) {
this.renderFn(vijiAPI, this.p5Instance);
}
} catch (error) {
console.error("P5 render error:", error);
throw error;
}
}
/**
* Handle canvas resize
*/
resize(width, height) {
if (!this.p5Instance) return;
this.p5Instance._setProperty("width", width);
this.p5Instance._setProperty("height", height);
this.p5Instance._setProperty("_width", width);
this.p5Instance._setProperty("_height", height);
if (this.p5Instance._renderer) {
this.p5Instance._renderer.width = width;
this.p5Instance._renderer.height = height;
}
if (typeof this.p5Instance.resizeCanvas === "function") {
try {
this.p5Instance.resizeCanvas(width, height, true);
} catch (error) {
console.warn("P5 resize warning:", error);
}
}
}
/**
* Cleanup P5 instance
*/
destroy() {
if (this.p5Instance) {
try {
if (typeof this.p5Instance.remove === "function") {
this.p5Instance.remove();
}
} catch (error) {
console.warn("P5 cleanup warning:", error);
}
this.p5Instance = null;
}
this.setupFn = null;
this.renderFn = null;
this.p5InternalSetupComplete = false;
this.artistSetupComplete = false;
}
}
class ShaderParameterParser {
/**
* Parse all parameter declarations from shader code
*/
static parseParameters(shaderCode) {
const parameters = [];
const lines = shaderCode.split("\n");
for (const line of lines) {
const trimmed = line.trim();
const match = trimmed.match(/\/\/\s*@viji-(\w+):(\w+)\s+(.+)/);
if (match) {
const [, type, uniformName, configStr] = match;
try {
const config = this.parseKeyValuePairs(configStr);
const param = {
type,
uniformName,
label: config.label || uniformName,
default: config.default,
config
};
this.validateParameter(param);
parameters.push(param);
} catch (error) {
console.warn(`Failed to parse shader parameter: ${line}`, error);
}
}
}
return parameters;
}
/**
* Parse key:value pairs from configuration string
*/
static parseKeyValuePairs(configStr) {
const config = {};
const keyValueRegex = /(\w+):((?:"[^"]*"|\[[^\]]*\]|#[0-9a-fA-F]{6}|#[0-9a-fA-F]{3}|[^\s]+))/g;
let match;
while ((match = keyValueRegex.exec(configStr)) !== null) {
const [, key, value] = match;
config[key] = this.parseValue(value);
}
return config;
}
/**
* Parse individual value from string
*/
static parseValue(value) {
if (value.startsWith('"') && value.endsWith('"')) {
return value.slice(1, -1);
}
if (value.startsWith("[") && value.endsWith("]")) {
try {
return JSON.parse(value);
} catch {
const items = value.slice(1, -1).split(",").map((s) => s.trim());
return items.map((item) => {
if (item.startsWith('"') && item.endsWith('"')) {
return item.slice(1, -1);
}
const num2 = parseFloat(item);
return isNaN(num2) ? item : num2;
});
}
}
if (value.startsWith("#")) {
return value;
}
if (value === "true") return true;
if (value === "false") return false;
const num = parseFloat(value);
if (!isNaN(num)) return num;
return value;
}
/**
* Validate parameter definition
*/
static validateParameter(param) {
if (!param.type) {
throw new Error("Parameter type is required");
}
if (!param.uniformName) {
throw new Error("Parameter uniformName is required");
}
if (!param.config.label) {
throw new Error(`Parameter ${param.uniformName} missing required 'label' key`);
}
switch (param.type) {
case "slider":
case "number":
if (param.config.default === void 0) {
throw new Error(`Parameter ${param.uniformName} of type ${param.type} missing required 'default' key`);
}
break;
case "color":
if (param.config.default === void 0) {
throw new Error(`Parameter ${param.uniformName} of type 'color' missing required 'default' key`);
}
if (!param.config.default.startsWith("#")) {
throw new Error(`Parameter ${param.uniformName} of type 'color' default must be hex color (e.g., #ff0000)`);
}
break;
case "toggle":
if (param.config.default === void 0) {
throw new Error(`Parameter ${param.uniformName} of type 'toggle' missing required 'default' key`);
}
if (typeof param.config.default !== "boolean") {
throw new Error(`Parameter ${param.uniformName} of type 'toggle' default must be boolean (true or false)`);
}
break;
case "select":
if (param.config.default === void 0) {
throw new Error(`Parameter ${param.uniformName} of type 'select' missing required 'default' key`);
}
if (!param.config.options || !Array.isArray(param.config.options)) {
throw new Error(`Parameter ${param.uniformName} of type 'select' missing required 'options' key (array)`);
}
break;
case "image":
break;
default:
console.warn(`Unknown parameter type: ${param.type}`);
}
if (param.uniformName.startsWith("u_")) {
console.warn(`Parameter name "${param.uniformName}" uses reserved prefix "u_". Consider renaming to avoid conflicts with built-in uniforms.`);
}
}
/**
* Generate uniform declaration for a parameter
*/
static generateUniformDeclaration(param) {
switch (param.type) {
case "slider":
case "number":
return `uniform float ${param.uniformName};`;
case "color":
return `uniform vec3 ${param.uniformName};`;
case "toggle":
return `uniform bool ${param.uniformName};`;
case "select":
return `uniform int ${param.uniformName};`;
case "image":
return `uniform sampler2D ${param.uniformName};`;
default:
return `// Unknown parameter type: ${param.type}`;
}
}
}
class ShaderWorkerAdapter {
constructor(offscreenCanvas, _vijiAPI, shaderCode) {
this.shaderCode = shaderCode;
this.glslVersion = this.detectGLSLVersion(shaderCode);
this.backbufferEnabled = shaderCode.includes("backbuffer");
if (this.glslVersion === "glsl300") {
const gl = offscreenCanvas.getContext("webgl2");
if (!gl) {
throw new Error("WebGL 2 not supported. Use GLSL ES 1.00 syntax instead.");
}
this.gl = gl;
} else {
const gl = offscreenCanvas.getContext("webgl");
if (!gl) {
throw new Error("WebGL not supported");
}
this.gl = gl;
}
}
gl;
program = null;
uniformLocations = /* @__PURE__ */ new Map();
textureUnits = /* @__PURE__ */ new Map();
nextTextureUnit = 0;
textures = /* @__PURE__ */ new Map();
// Fullscreen quad
quadBuffer = null;
// Parameter definitions
parameters = [];
// GLSL version detection
glslVersion = "glsl100";
// Audio FFT texture
audioFFTTexture = null;
videoTexture = null;
segmentationTexture = null;
// Backbuffer support (ping-pong framebuffers)
backbufferFramebuffer = null;
backbufferTexture = null;
currentFramebuffer = null;
currentTexture = null;
backbufferEnabled = false;
/**
* Initialize the shader adapter
*/
async init() {
try {
this.parameters = ShaderParameterParser.parseParameters(this.shaderCode);
this.createFullscreenQuad();
const processedCode = this.injectUniforms(this.shaderCode);
this.compileAndLinkShader(processedCode);
this.cacheUniformLocations();
this.reserveTextureUnits();
if (this.backbufferEnabled) {
this.createBackbufferFramebuffers();
}
} catch (error) {
console.error("Failed to initialize ShaderWorkerAdapter:", error);
throw error;
}
}
/**
* Detect GLSL version from shader code
*/
detectGLSLVersion(code) {
return code.includes("#version 300") ? "glsl300" : "glsl100";
}
/**
* Create fullscreen quad geometry
*/
createFullscreenQuad() {
const vertices = new Float32Array([
-1,
-1,
// Bottom-left
1,
-1,
// Bottom-right
-1,
1,
// Top-left
1,
1
// Top-right
]);
this.quadBuffer = this.gl.createBuffer();
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.quadBuffer);
this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
}
/**
* Inject built-in and parameter uniforms into shader code
*/
injectUniforms(artistCode) {
let versionLine = "";
let codeWithoutVersion = artistCode;
const lines = artistCode.split("\n");
for (let i = 0; i < lines.length; i++) {
const trimmed = lines[i].trim();
if (trimmed.startsWith("#version")) {
versionLine = trimmed;
lines[i] = "";
codeWithoutVersion = lines.join("\n");
break;
}
}
const injectionPoint = this.findInjectionPoint(codeWithoutVersion);
const builtInUniforms = this.getBuiltInUniforms();
const parameterUniforms = this.parameters.map((p) => ShaderParameterParser.generateUniformDeclaration(p)).join("\n");
const usesFwidth = artistCode.includes("fwidth");
if (usesFwidth && this.glslVersion === "glsl100") {
const ext = this.gl.getExtension("OES_standard_derivatives");
if (!ext) {
console.warn("Shader uses fwidth() but OES_standard_derivatives extension is not supported. Shader may not compile.");
}
}
const parts = [];
if (usesFwidth && this.glslVersion === "glsl100") {
parts.push("#extension GL_OES_standard_derivatives : enable");
}
if (this.glslVersion === "glsl100") {
parts.push("");
parts.push("#ifdef GL_ES");
parts.push("precision mediump float;");
parts.push("#endif");
} else {
parts.push("");
parts.push("precision mediump float;");
}
parts.push("");
parts.push("// ===== VIJI AUTO-INJECTED UNIFORMS =====");
parts.push("// Built-in uniforms (auto-provided)");
parts.push(builtInUniforms);
parts.push("");
parts.push("// Parameter uniforms (from @viji-* declarations)");
parts.push(parameterUniforms);
parts.push("");
parts.push("// ===== ARTIST CODE =====");
const uniformBlock = parts.join("\n");
const codeWithUniforms = codeWithoutVersion.slice(0, injectionPoint) + "\n" + uniformBlock + "\n" + codeWithoutVersion.slice(injectionPoint);
const finalCode = versionLine ? versionLine + "\n" + codeWithUniforms : codeWithUniforms;
console.log("=== INJECTED SHADER CODE (first 50 lines) ===");
console.log(finalCode.split("\n").slice(0, 50).join("\n"));
console.log("=== END INJECTED CODE ===");
return finalCode;
}
/**
* Find where to inject extensions and uniforms
* Extensions must come after #version but before any code
*
* Strategy:
* 1. If #version exists, inject right after it
* 2. Otherwise, skip ALL comments (single and multi-line) and inject before first code
*/
findInjectionPoint(code) {
const lines = code.split("\n");
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
if (line.startsWith("#version")) {
return this.getLineEndPosition(code, i);
}
}
let inMultiLineComment = false;
let firstCodeLine = 0;
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
if (line.includes("/*")) {
inMultiLineComment = true;
}
if (line.includes("*/")) {
inMultiLineComment = false;
firstCodeLine = i + 1;
continue;
}
if (inMultiLineComment) {
continue;
}
if (line === "" || line.startsWith("//")) {
firstCodeLine = i + 1;
continue;
}
break;
}
if (firstCodeLine > 0 && firstCodeLine < lines.length) {
return this.getLineEndPosition(code, firstCodeLine - 1);
}
return 0;
}
/**
* Get byte position of end of line N
*/
getLineEndPosition(code, lineNumber) {
const lines = code.split("\n");
let position = 0;
for (let i = 0; i <= lineNumber && i < lines.length; i++) {
position += lines[i].length + 1;
}
return position;
}
/**
* Get built-in uniform declarations
*/
getBuiltInUniforms() {
return `// Core - Canvas & Timing
uniform vec2 u_resolution; // Canvas width and height in pixels
uniform float u_time; // Elapsed time in seconds since scene start
uniform float u_deltaTime; // Time elapsed since last frame in seconds
uniform int u_frame; // Current frame number
uniform float u_pixelRatio; // Device pixel ratio for high-DPI displays
uniform float u_fps; // Current frames per second
// Mouse API
uniform vec2 u_mouse; // Mouse position in pixels (WebGL coords: bottom-left origin)
uniform bool u_mouseInCanvas; // True if mouse is inside canvas bounds
uniform bool u_mousePressed; // True if any mouse button is pressed
uniform bool u_mouseLeft; // True if left mouse button is pressed
uniform bool u_mouseRight; // True if right mouse button is pressed
uniform bool u_mouseMiddle; // True if middle mouse button is pressed
uniform vec2 u_mouseVelocity; // Mouse movement velocity in pixels per second
// Keyboard API - Common keys
uniform bool u_keySpace; // True if spacebar is pressed
uniform bool u_keyShift; // True if Shift key is pressed
uniform bool u_keyCtrl; // True if Ctrl/Cmd key is pressed
uniform bool u_keyAlt; // True if Alt/Option key is pressed
uniform bool u_keyW; // True if W key is pressed
uniform bool u_keyA; // True if A key is pressed
uniform bool u_keyS; // True if S key is pressed
uniform bool u_keyD; // True if D key is pressed
uniform bool u_keyUp; // True if Up arrow key is pressed
uniform bool u_keyDown; // True if Down arrow key is pressed
uniform bool u_keyLeft; // True if Left arrow key is pressed
uniform bool u_keyRight; // True if Right arrow key is pressed
// Touch API
uniform int u_touchCount; // Number of active touch points (0-5)
uniform vec2 u_touch0; // First touch point position in pixels
uniform vec2 u_touch1; // Second touch point position in pixels
uniform vec2 u_touch2; // Third touch point position in pixels
uniform vec2 u_touch3; // Fourth touch point position in pixels
uniform vec2 u_touch4; // Fifth touch point position in pixels
// Audio
uniform float u_audioVolume; // RMS volume level (0-1)
uniform float u_audioPeak; // Peak volume level (0-1)
uniform float u_audioBass; // Bass frequency band level (0-1)
uniform float u_audioMid; // Mid frequency band level (0-1)
uniform float u_audioTreble; // Treble frequency band level (0-1)
uniform float u_audioSubBass; // Sub-bass frequency band 20-60 Hz (0-1)
uniform float u_audioLowMid; // Low-mid frequency band 250-500 Hz (0-1)
uniform float u_audioHighMid; // High-mid frequency band 2-4 kHz (0-1)
uniform float u_audioPresence; // Presence frequency band 4-6 kHz (0-1)
uniform float u_audioBrilliance; // Brilliance frequency band 6-20 kHz (0-1)
uniform sampler2D u_audioFFT; // FFT texture containing 512 frequency bins
// Video
uniform sampler2D u_video; // Current video frame as texture
uniform vec2 u_videoResolution; // Video frame width and height in pixels
uniform float u_videoFrameRate; // Video frame rate in frames per second
// CV - Face Detection
uniform int u_faceCount; // Number of detected faces (0-1)
uniform vec4 u_face0Bounds; // First face bounding box (x, y, width, height)
uniform vec3 u_face0HeadPose; // First face head rotation (pitch, yaw, roll) in radians
uniform float u_face0Confidence; // First face detection confidence (0-1)
uniform float u_face0Happy; // First face happy expression confidence (0-1)
uniform float u_face0Sad; // First face sad expression confidence (0-1)
uniform float u_face0Angry; // First face angry expression confidence (0-1)
uniform float u_face0Surprised; // First face surprised expression confidence (0-1)
// CV - Hand Tracking
uniform int u_handCount; // Number of detected hands (0-2)
uniform vec3 u_leftHandPalm; // Left hand palm position (x, y, z)
uniform vec3 u_rightHandPalm; // Right hand palm position (x, y, z)
uniform float u_leftHandFist; // Left hand fist gesture confidence (0-1)
uniform float u_leftHandOpen; // Left hand open palm gesture confidence (0-1)
uniform float u_rightHandFist; // Right hand fist gesture confidence (0-1)
uniform float u_rightHandOpen; // Right hand open palm gesture confidence (0-1)
// CV - Pose Detection
uniform bool u_poseDetected; // True if a pose is currently detected
uniform vec2 u_nosePosition; // Nose landmark position in pixels
uniform vec2 u_leftWristPosition; // Left wrist landmark position in pixels
uniform vec2 u_rightWristPosition; // Right wrist landmark position in pixels
uniform vec2 u_leftAnklePosition; // Left ankle landmark position in pixels
uniform vec2 u_rightAnklePosition; // Right ankle landmark position in pixels
// CV - Segmentation
uniform sampler2D u_segmentationMask; // Body segmentation mask texture (0=background, 1=person)
uniform vec2 u_segmentationRes; // Segmentation mask resolution in pixels
// Backbuffer (previous frame feedback)
${this.backbufferEnabled ? "uniform sampler2D backbuffer; // Previous frame texture for feedback effects" : "// backbuffer not enabled"}
`;
}
/**
* Compile and link shader program
*/
compileAndLinkShader(fragmentShaderCode) {
const gl = this.gl;
const vertexShaderCode = this.glslVersion === "glsl300" ? `#version 300 es
precision mediump float;
in vec2 a_position;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
}` : `attribute vec2 a_position;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
}`;
const vertexShader = this.compileShader(gl.VERTEX_SHADER, vertexShaderCode);
const fragmentShader = this.compileShader(gl.FRAGMENT_SHADER, fragmentShaderCode);
const program = gl.createProgram();
if (!program) {
throw new Error("Failed to create WebGL program");
}
gl.attachShader(program, vertexShader);
gl.attachShader(program, fragmentShader);
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
const error = gl.getProgramInfoLog(program);
throw new Error(`Shader program link failed: ${error}`);
}
this.program = program;
gl.useProgram(program);
gl.deleteShader(vertexShader);
gl.deleteShader(fragmentShader);
}
/**
* Compile a shader
*/
compileShader(type, source) {
const gl = this.gl;
const shader = gl.createShader(type);
if (!shader) {
throw new Error("Failed to create shader");
}
gl.shaderSource(shader, source);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
const error = gl.getShaderInfoLog(shader);
const shaderType = type === gl.VERTEX_SHADER ? "vertex" : "fragment";
throw new Error(`${shaderType} shader compilation failed:
${error}`);
}
return shader;
}
/**
* Cache uniform locations for fast access
*/
cacheUniformLocations() {
if (!this.program) return;
const gl = this.gl;
const numUniforms = gl.getProgramParameter(this.program, gl.ACTIVE_UNIFORMS);
for (let i = 0; i < numUniforms; i++) {
const info = gl.getActiveUniform(this.program, i);
if (info) {
const location = gl.getUniformLocation(this.program, info.name);
this.uniformLocations.set(info.name, location);
}
}
}
/**
* Reserve texture units for special textures
*/
reserveTextureUnits() {
this.textureUnits.set("u_audioFFT", this.nextTextureUnit++);
this.textureUnits.set("u_video", this.nextTextureUnit++);
this.textureUnits.set("u_segmentationMask", this.nextTextureUnit++);
if (this.backbufferEnabled) {
this.textureUnits.set("backbuffer", this.nextTextureUnit++);
}
}
/**
* Create ping-pong framebuffers for backbuffer support
*/
createBackbufferFramebuffers() {
const gl = this.gl;
const width = gl.canvas.width;
const height = gl.canvas.height;
const createFBOTexture = () => {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
const framebuffer = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
return { framebuffer, texture };
};
const fbo1 = createFBOTexture();
const fbo2 = createFBOTexture();
this.backbufferFramebuffer = fbo1.framebuffer;
this.backbufferTexture = fbo1.texture;
this.currentFramebuffer = fbo2.framebuffer;
this.currentTexture = fbo2.texture;
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.bindTexture(gl.TEXTURE_2D, null);
}
/**
* Main render method
*/
render(viji, parameterObjects) {
const gl = this.gl;
if (!this.program || !this.quadBuffer) {
console.warn("Shader not initialized");
return;
}
gl.useProgram(this.program);
this.updateBuiltInUniforms(viji);
this.updateParameterUniforms(parameterObjects);
if (this.backbufferEnabled && this.backbufferTexture) {
const backbufferUnit = this.textureUnits.get("backbuffer");
if (backbufferUnit !== void 0) {
gl.activeTexture(gl.TEXTURE0 + backbufferUnit);
gl.bindTexture(gl.TEXTURE_2D, this.backbufferTexture);
this.setUniform("backbuffer", "sampler2D", backbufferUnit);
}
gl.bindFramebuffer(gl.FRAMEBUFFER, this.currentFramebuffer);
}
const positionLocation = gl.getAttribLocation(this.program, "a_position");
gl.bindBuffer(gl.ARRAY_BUFFER, this.quadBuffer);
gl.enableVertexAttribArray(positionLocation);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
if (this.backbufferEnabled) {
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, this.currentTexture);
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
const tempFB = this.backbufferFramebuffer;
const tempTex = this.backbufferTexture;
this.backbufferFramebuffer = this.currentFramebuffer;
this.backbufferTexture = this.currentTexture;
this.currentFramebuffer = tempFB;
this.currentTexture = tempTex;
}
}
/**
* Update built-in uniforms from viji object
*/
updateBuiltInUniforms(viji) {
this.setUniform("u_resolution", "vec2", [viji.width, viji.height]);
this.setUniform("u_time", "float", viji.time);
this.setUniform("u_deltaTime", "float", viji.deltaTime);
this.setUniform("u_frame", "int", viji.frameCount);
this.setUniform("u_pixelRatio", "float", viji.pixelRatio);
this.setUniform("u_fps", "float", viji.fps);
this.setUniform("u_mouse", "vec2", [viji.mouse.x, viji.height - viji.mouse.y]);
this.setUniform("u_mouseInCanvas", "bool", viji.mouse.isInCanvas);
this.setUniform("u_mousePressed", "bool", viji.mouse.isPressed);
this.setUniform("u_mouseLeft", "bool", viji.mouse.leftButton);
this.setUniform("u_mouseRight", "bool", viji.mouse.rightButton);
this.setUniform("u_mouseMiddle", "bool", viji.mouse.middleButton);
this.setUniform("u_mouseVelocity", "vec2", [viji.mouse.velocity.x, -viji.mouse.velocity.y]);
this.setUniform("u_keySpace", "bool", viji.keyboard.isPressed(" ") || viji.keyboard.isPressed("space"));
this.setUniform("u_keyShift", "bool", viji.keyboard.shift);
this.setUniform("u_keyCtrl", "bool", viji.keyboard.ctrl);
this.setUniform("u_keyAlt", "bool", viji.keyboard.alt);
this.setUniform("u_keyW", "bool", viji.keyboard.isPressed("w") || viji.keyboard.isPressed("W"));
this.setUniform("u_keyA", "bool", viji.keyboard.isPressed("a") || viji.keyboard.isPressed("A"));
this.setUniform("u_keyS", "bool", viji.keyboard.isPressed("s") || viji.keyboard.isPressed("S"));
this.setUniform("u_keyD", "bool", viji.keyboard.isPressed("d") || viji.keyboard.isPressed("D"));
this.setUniform("u_keyUp", "bool", viji.keyboard.isPressed("ArrowUp"));
this.setUniform("u_keyDown", "bool", viji.keyboard.isPressed("ArrowDown"));
this.setUniform("u_keyLeft", "bool", viji.keyboard.isPressed("ArrowLeft"));
this.setUniform("u_keyRight", "bool", viji.keyboard.isPressed("ArrowRight"));
this.setUniform("u_touchCount", "int", viji.touches.count);
for (let i = 0; i < 5; i++) {
const touch = viji.touches.points[i];
if (touch) {
this.setUniform(`u_touch${i}`, "vec2", [touch.x, viji.height - touch.y]);
} else {
this.setUniform(`u_touch${i}`, "vec2", [0, 0]);
}
}
const audio = viji.audio;
this.setUniform("u_audioVolume", "float", audio.volume?.rms || 0);
this.setUniform("u_audioPeak", "float", audio.volume?.peak || 0);
this.setUniform("u_audioBass", "float", audio.bands?.bass || 0);
this.setUniform("u_audioMid", "float", audio.bands?.mid || 0);
this.setUniform("u_audioTreble", "float", audio.bands?.treble || 0);
this.setUniform("u_audioSubBass", "float", audio.bands?.subBass || 0);
this.setUniform("u_audioLowMid", "float", audio.bands?.lowMid || 0);
this.setUniform("u_audioHighMid", "float", audio.bands?.highMid || 0);
this.setUniform("u_audioPresence", "float", audio.bands?.presence || 0);
this.setUniform("u_audioBrilliance", "float", audio.bands?.brilliance || 0);
if (audio.isConnected) {
this.updateAudioFFTTexture(audio.getFrequencyData());
}
const video = viji.video;
if (video.isConnected && video.currentFrame) {
this.updateVideoTexture(video.currentFrame);
this.setUniform("u_videoResolution", "vec2", [video.frameWidth, video.frameHeight]);
this.setUniform("u_videoFrameRate", "float", video.frameRate);
} else {
this.setUniform("u_videoResolution", "vec2", [0, 0]);
this.setUniform("u_videoFrameRate", "float", 0);
}
const faces = video.faces || [];
this.setUniform("u_faceCount", "int", faces.length);
if (faces.length > 0) {
const face = faces[0];
this.setUniform("u_face0Bounds", "vec4", [face.bounds.x, face.bounds.y, face.bounds.width, face.bounds.height]);
this.setUniform("u_face0HeadPose", "vec3", [face.headPose.pitch, face.headPose.yaw, face.headPose.roll]);
this.setUniform("u_face0Confidence", "float", face.confidence);
this.setUniform("u_face0Happy", "float", face.expressions.happy);
this.setUniform("u_face0Sad", "float", face.expressions.sad);
this.setUniform("u_face0Angry", "float", face.expressions.angry);
this.setUniform("u_face0Surprised", "float", face.expressions.surprised);
} else {
this.setUniform("u_face0Bounds", "vec4", [0, 0, 0, 0]);
this.setUniform("u_face0HeadPose", "vec3", [0, 0, 0]);
this.setUniform("u_face0Confidence", "float", 0);
this.setUniform("u_face0Happy", "float", 0);
this.setUniform("u_face0Sad", "float", 0);
this.setUniform("u_face0Angry", "float", 0);
this.setUniform("u_face0Surprised", "float", 0);
}
const hands = video.hands || [];
this.setUniform("u_handCount", "int", hands.length);
const leftHand = hands.find((h) => h.handedness === "left");
const rightHand = hands.find((h) => h.handedness === "right");
if (leftHand) {
this.setUniform("u_leftHandPalm", "vec3", [leftHand.palm.x, leftHand.palm.y, leftHand.palm.z]);
this.setUniform("u_leftHandFist", "float", leftHand.gestures?.fist || 0);
this.setUniform("u_leftHandOpen", "float", leftHand.gestures?.openPalm || 0);
} else {
this.setUniform("u_leftHandPalm", "vec3", [0, 0, 0]);
this.setUniform("u_leftHandFist", "float", 0);
this.setUniform("u_leftHandOpen", "float", 0);
}
if (rightHand) {
this.setUniform("u_rightHandPalm", "vec3", [rightHand.palm.x, rightHand.palm.y, rightHand.palm.z]);
this.setUniform("u_rightHandFist", "float", rightHand.gestures?.fist || 0);
this.setUniform("u_rightHandOpen", "float", rightHand.gestures?.openPalm || 0);
} else {
this.setUniform("u_rightHandPalm", "vec3", [0, 0, 0]);
this.setUniform("u_rightHandFist", "float", 0);
this.setUniform("u_rightHandOpen", "float", 0);
}
const pose = video.pose;
this.setUniform("u_poseDetected", "bool", pose !== null);
if (pose) {
const nose = pose.landmarks[0];
const leftWrist = pose.landmarks[15];
const rightWrist = pose.landmarks[16];
const leftAnkle = pose.landmarks[27];
const rightAnkle = pose.landmarks[28];
this.setUniform("u_nosePosition", "vec2", [nose?.x || 0, nose?.y || 0]);
this.setUniform("u_leftWristPosition", "vec2", [leftWrist?.x || 0, leftWrist?.y || 0]);
this.setUniform("u_rightWristPosition", "vec2", [rightWrist?.x || 0, rightWrist?.y || 0]);
this.setUniform("u_leftAnklePosition", "vec2", [leftAnkle?.x || 0, leftAnkle?.y || 0]);
this.setUniform("u_rightAnklePosition", "vec2", [rightAnkle?.x || 0, rightAnkle?.y || 0]);
} else {
this.setUniform("u_nosePosition", "vec2", [0, 0]);
this.setUniform("u_leftWristPosition", "vec2", [0, 0]);
this.setUniform("u_rightWristPosition", "vec2", [0, 0]);
this.setUniform("u_leftAnklePosition", "vec2", [0, 0]);
this.setUniform("u_rightAnklePosition", "vec2", [0, 0]);
}
const segmentation = video.segmentation;
if (segmentation) {
this.updateSegmentationTexture(segmentation.mask, segmentation.width, segmentation.height);
this.setUniform("u_segmentationRes", "vec2", [segmentation.width, segmentation.height]);
} else {
this.setUniform("u_segmentationRes", "vec2", [0, 0]);
}
}
/**
* Update parameter uniforms from parameter objects
*/
updateParameterUniforms(parameterObjects) {
for (const param of this.parameters) {
const paramObj = parameterObjects.get(param.uniformName);
if (!paramObj) continue;
const value = paramObj.value;
switch (param.type) {
case "slider":
case "number":
this.setUniform(param.uniformName, "float", value);
break;
case "color":
const rgb = this.hexToRgb(value);
this.setUniform(param.uniformName, "vec3", rgb);
break;
case "toggle":
this.setUniform(param.uniformName, "bool", value);
break;
case "select":
const index = param.config.options?.indexOf(value) || 0;
this.setUniform(param.uniformName, "int", index);
break;
case "image":
if (value) {
this.updateImageTexture(param.uniformName, value);
}
break;
}
}
}
/**
* Set uniform value
*/
setUniform(name, type, value) {
const location = this.uniformLocations.get(name);
if (location === null || location === void 0) {
return;
}
const gl = this.gl;
switch (type) {
case "float":
gl.uniform1f(location, value);
break;
case "int":
gl.uniform1i(location, value);
break;
case "bool":
gl.uniform1i(location, value ? 1 : 0);
break;
case "vec2":
gl.uniform2f(location, value[0], value[1]);
break;
case "vec3":
gl.uniform3f(location, value[0], value[1], value[2]);
break;
case "vec4":
gl.uniform4f(location, value[0], value[1], value[2], value[3]);
break;
}
}
/**
* Convert hex color to RGB [0-1]
*/
hexToRgb(hex) {
const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex);
if (result) {
return [
parseInt(result[1], 16) / 255,
parseInt(result[2], 16) / 255,
parseInt(result[3], 16) / 255
];
}
return [0, 0, 0];
}
/**
* Update audio FFT texture
*/
updateAudioFFTTexture(frequencyData) {
const gl = this.gl;
const unit = this.textureUnits.get("u_audioFFT");
if (!this.audioFFTTexture) {
this.audioFFTTexture = gl.createTexture();
}
gl.activeTexture(gl.TEXTURE0 + unit);
gl.bindTexture(gl.TEXTURE_2D, this.audioFFTTexture);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.LUMINANCE,
frequencyData.length,
1,
0,
gl.LUMINANCE,
gl.UNSIGNED_BYTE,
frequencyData
);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
const location = this.uniformLocations.get("u_audioFFT");
if (location) {
gl.uniform1i(location, unit);
}
}
/**
* Update video texture
*/
updateVideoTexture(videoFrame) {
const gl = this.gl;
const unit = this.textureUnits.get("u_video");
if (!this.videoTexture) {
this.videoTexture = gl.createTexture();
}
gl.activeTexture(gl.TEXTURE0 + unit);
gl.bindTexture(gl.TEXTURE_2D, this.videoTexture);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.RGBA,
gl.RGBA,
gl.UNSIGNED_BYTE,
videoFrame
);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
const location = this.uniformLocations.get("u_video");
if (location) {
gl.uniform1i(location, unit);
}
}
/**
* Update segmentation mask texture
*/
updateSegmentationTexture(mask, width, height) {
const gl = this.gl;
const unit = this.textureUnits.get("u_segmentationMask");
if (!this.segmentationTexture) {
this.segmentationTexture = gl.createTexture();
}
gl.activeTexture(gl.TEXTURE0 + unit);
gl.bindTexture(gl.TEXTURE_2D, this.segmentationTexture);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.LUMINANCE,
width,
height,
0,
gl.LUMINANCE,
gl.UNSIGNED_BYTE,
mask
);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
const location = this.uniformLocations.get("u_segmentationMask");
if (location) {
gl.uniform1i(location, unit);
}
}
/**
* Update image parameter texture
*/
updateImageTexture(name, imageBitmap) {
const gl = this.gl;
if (!this.textureUnits.has(name)) {
this.textureUnits.set(name, this.nextTextureUnit++);
}
const unit = this.textureUnits.get(name);
if (!this.textures.has(name)) {
const texture2 = gl.createTexture();
if (texture2) {
this.textures.set(name, texture2);
}
}
const texture = this.textures.get(name);
if (!texture) return;
gl.activeTexture(gl.TEXTURE0 + unit);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.RGBA,
gl.RGBA,
gl.UNSIGNED_BYTE,
imageBitmap
);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
const location = this.uniformLocations.get(name);
if (location) {
gl.uniform1i(location, unit);
}
}
/**
* Handle canvas resize
*/
resize(width, height) {
const gl = this.gl;
gl.viewport(0, 0, width, height);
if (this.backbufferEnabled) {
this.createBackbufferFramebuffers();
}
}
/**
* Get parameter definitions for host
*/
getParameterDefinitions() {
return this.parameters;
}
}
class VijiWorkerRuntime {
canvas = null;
ctx = null;
gl = null;
isRunning = false;
frameCount = 0;
lastTime = 0;
startTime = 0;
frameRateMode = "full";
skipNextFrame = false;
screenRefreshRate = 60;
// Will be detected
// Debug logging control
debugMode = false;
// P5.js adapter for P5 mode
p5Adapter = null;
// Shader adapter for shader mode
shaderAdapter = null;
rendererType = "native";
// Pending capture requests (queue to handle multiple simultaneous requests)
pendingCaptures = [];
/**
* Enable or disable debug logging
*/
setDebugMode(enabled) {
this.debugMode = enabled;
if (this.videoSystem) this.videoSystem.setDebugMode(enabled);
if (this.parameterSystem && "setDebugMode" in this.parameterSystem) {
this.parameterSystem.setDebugMode(enabled);
}
if (this.interactionSystem && "setDebugMode" in this.interactionSystem) {
this.interactionSystem.setDebugMode(enabled);
}
}
/**
* Debug logging helper
*/
debugLog(message, ...args) {
if (this.debugMode) {
console.log(message, ...args);
}
}
// Effective refresh rate tracking
effectiveFrameTimes = [];
lastEffectiveRateReport = 0;
effectiveRateReportInterval = 1e3;
// Report every 1 second
// Parameter system
parameterSystem;
// Interaction system (Phase 7)
interactionSystem;
// Video system (Phase 10) - worker-side video processing
videoSystem;
// Audio state (Phase 5) - receives analysis results from host
audioState = {
isConnected: false,
volume: { rms: 0, peak: 0 },
bands: {
bass: 0,
mid: 0,
treble: 0,
subBass: 0,
lowMid: 0,
highMid: 0,
presence: 0,
brilliance: 0
},
frequencyData: new Uint8Array(0)
};
// Video state is now managed by the worker-side VideoSystem
// Artist API object
viji = {
// Canvas (will be set during init)
canvas: null,
ctx: null,
gl: null,
width: 0,
height: 0,
pixelRatio: 1,
// Timing
time: 0,
deltaTime: 0,
frameCount: 0,
fps: 60,
// Audio API (Phase 5) - will be set in constructor
audio: {},
video: {
isConnected: false,
currentFrame: null,
frameWidth: 0,
frameHeight: 0,
frameRate: 0,
getFrameData: () => null,
faces: [],
hands: []
},
// Interaction APIs will be added during construction
mouse: {},
keyboard: {},
touches: {},
// Parameter helper functions (return parameter objects) - delegate to parameter system
slider: (defaultValue, config) => {
return this.parameterSystem.createSliderParameter(defaultValue, config);
},
color: (defaultValue, config) => {
return this.parameterSystem.createColorParameter(defaultValue, config);
},
toggle: (defaultValue, config) => {
return this.parameterSystem.createToggleParameter(defaultValue, config);
},
select: (defaultValue, config) => {
return this.parameterSystem.createSelectParameter(defaultValue, config);
},
text: (defaultValue, config) => {
return this.parameterSystem.createTextParameter(defaultValue, config);
},
number: (defaultValue, config) => {
return this.parameterSystem.createNumberParameter(defaultValue, config);
},
image: (defaultValue, config) => {
return this.parameterSystem.createImageParameter(defaultValue, config);
},
// Context selection
useContext: (type) => {
if (type === "2d") {
if (!this.ctx && this.canvas) {
this.ctx = this.canvas.getContext("2d");
this.viji.ctx = this.ctx;
}
return this.ctx;
} else if (type === "webgl") {
if (!this.gl && this.canvas) {
this.gl = this.canvas.getContext("webgl2") || this.canvas.getContext("webgl");
this.viji.gl = this.gl;
if (this.gl) {
this.gl.viewport(0, 0, this.viji.width, this.viji.height);
}
}
return this.gl;
}
return null;
}
};
constructor() {
this.parameterSystem = new ParameterSystem((type, data) => {
this.postMessage(type, data);
});
this.interactionSystem = new InteractionSystem();
this.videoSystem = new VideoSystem();
Object.assign(this.viji, this.interactionSystem.getInteractionAPIs());
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
this.viji.audio = {
...this.audioState,
getFrequencyData: () => this.audioState.frequencyData
};
this.setupMessageHandling();
}
/**
* Initialize P5.js mode
* Sets up P5 rendering with P5WorkerAdapter
*/
async initP5Mode(setup, render) {
try {
this.rendererType = "p5";
this.debugLog("🎨 Initializing P5.js mode...");
this.p5Adapter = new P5WorkerAdapter(
this.canvas,
this.viji,
{
setup,
render
}
);
await this.p5Adapter.init();
this.debugLog("✅ P5.js mode initialized successfully");
} catch (error) {
console.error("❌ Failed to initialize P5.js mode:", error);
this.postMessage("error", {
message: `P5.js initialization failed: ${error.message}`,
code: "P5_INIT_ERROR"
});
this.rendererType = "native";
this.p5Adapter = null;
}
}
/**
* Initialize shader rendering mode
* Used when artist code includes // @renderer shader
*/
async initShaderMode(shaderCode) {
try {
this.rendererType = "shader";
this.debugLog("🎨 Initializing Shader mode...");
this.shaderAdapter = new ShaderWorkerAdapter(
this.canvas,
this.viji,
shaderCode
);
await this.shaderAdapter.init();
const parameterDefinitions = this.shaderAdapter.getParameterDefinitions();
for (const param of parameterDefinitions) {
this.registerShaderParameter(param);
}
this.debugLog("✅ Shader mode initialized successfully");
} catch (error) {
console.error("❌ Failed to initialize Shader mode:", error);
this.postMessage("error", {
message: `Shader initialization failed: ${error.message}`,
code: "SHADER_INIT_ERROR"
});
this.rendererType = "native";
this.shaderAdapter = null;
}
}
/**
* Register a shader parameter with the parameter system
* In the parameter system, the 'label' serves as the parameter name/key
*/
registerShaderParameter(param) {
const config = param.config;
const paramConfig = {
...config,
label: param.uniformName,
// uniformName becomes the parameter key
description: config.label ? `${config.label}${config.description ? ": " + config.description : ""}` : config.description
};
switch (param.type) {
case "slider":
this.viji.slider(config.default, paramConfig);
break;
case "number":
this.viji.number(config.default, paramConfig);
break;
case "color":
this.viji.color(config.default, paramConfig);
break;
case "toggle":
this.viji.toggle(config.default, paramConfig);
break;
case "select":
this.viji.select(config.default, {
...paramConfig,
options: config.options
});
break;
case "image":
this.viji.image(null, paramConfig);
break;
}
}
// Reset parameter state (called when loading new scene)
resetParameterState() {
this.parameterSystem.resetParameterState();
this.interactionSystem.resetInteractionState();
this.audioState = {
isConnected: false,
volume: { rms: 0, peak: 0 },
bands: {
bass: 0,
mid: 0,
treble: 0,
subBass: 0,
lowMid: 0,
highMid: 0,
presence: 0,
brilliance: 0
},
frequencyData: new Uint8Array(0)
};
this.viji.audio = {
...this.audioState,
getFrequencyData: () => this.audioState.frequencyData
};
this.videoSystem.resetVideoState();
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
// Send all parameters (from helper functions) to host
sendAllParametersToHost() {
this.parameterSystem.sendAllParametersToHost();
}
setupMessageHandling() {
self.onmessage = (event) => {
const message = event.data;
switch (message.type) {
case "init":
this.handleInit(message);
break;
case "frame-rate-update":
this.handleFrameRateUpdate(message);
break;
case "refresh-rate-update":
this.handleRefreshRateUpdate(message);
break;
case "cv-frame-rate-update":
this.handleCVFrameRateUpdate(message);
break;
case "resolution-update":
this.handleResolutionUpdate(message);
break;
case "set-scene-code":
this.handleSetSceneCode(message);
break;
case "debug-mode":
this.setDebugMode(message.data.enabled);
break;
case "parameter-update":
this.handleParameterUpdate(message);
break;
case "parameter-batch-update":
this.handleParameterBatchUpdate(message);
break;
case "stream-update":
this.handleStreamUpdate(message);
break;
case "audio-analysis-update":
this.handleAudioAnalysisUpdate(message);
break;
case "video-canvas-setup":
this.handleVideoCanvasSetup(message);
break;
case "video-frame-update":
this.handleVideoFrameUpdate(message);
break;
case "video-config-update":
this.handleVideoConfigUpdate(message);
break;
case "mouse-update":
this.handleMouseUpdate(message);
break;
case "keyboard-update":
this.handleKeyboardUpdate(message);
break;
case "touch-update":
this.handleTouchUpdate(message);
break;
case "interaction-enabled":
this.handleInteractionEnabled(message);
break;
case "performance-update":
this.handlePerformanceUpdate(message);
break;
case "capture-frame":
this.handleCaptureFrame(message);
break;
}
};
}
handleInit(message) {
try {
this.canvas = message.data.canvas;
this.viji.canvas = this.canvas;
this.viji.width = this.canvas.width;
this.viji.height = this.canvas.height;
this.startRenderLoop();
this.postMessage("init-response", {
id: message.id
});
this.postMessage("ready", {
rendererType: this.rendererType,
canvasSize: { width: this.canvas.width, height: this.canvas.height }
});
} catch (error) {
this.postMessage("error", {
id: message.id,
message: error.message,
code: "INIT_ERROR"
});
}
}
handleFrameRateUpdate(message) {
if (message.data && message.data.mode) {
this.frameRateMode = message.data.mode;
this.debugLog("Frame rate mode updated to:", message.data.mode);
}
}
handleRefreshRateUpdate(message) {
if (message.data && message.data.screenRefreshRate) {
this.screenRefreshRate = message.data.screenRefreshRate;
this.debugLog("Screen refresh rate updated to:", message.data.screenRefreshRate + "Hz");
}
}
handleCVFrameRateUpdate(message) {
if (message.data && message.data.mode) {
const sceneProcessingFPS = this.frameRateMode === "full" ? this.screenRefreshRate : this.screenRefreshRate / 2;
if (this.videoSystem) {
this.videoSystem.handleVideoConfigUpdate({
cvFrameRate: {
mode: message.data.mode,
sceneTargetFPS: sceneProcessingFPS
},
timestamp: performance.now()
});
}
this.debugLog(`CV frame rate updated to: ${message.data.mode} of ${sceneProcessingFPS} FPS scene processing`);
}
}
trackEffectiveFrameTime(currentTime) {
this.effectiveFrameTimes.push(currentTime);
if (this.effectiveFrameTimes.length > 60) {
this.effectiveFrameTimes.shift();
}
}
reportPerformanceStats(currentTime) {
if (currentTime - this.lastEffectiveRateReport >= this.effectiveRateReportInterval) {
if (this.effectiveFrameTimes.length >= 2) {
const totalTime = this.effectiveFrameTimes[this.effectiveFrameTimes.length - 1] - this.effectiveFrameTimes[0];
const frameCount = this.effectiveFrameTimes.length - 1;
const effectiveRefreshRate = Math.round(frameCount / totalTime * 1e3);
const cvStats = this.videoSystem.getCVStats();
this.postMessage("performance-update", {
effectiveRefreshRate,
frameRateMode: this.frameRateMode,
screenRefreshRate: this.screenRefreshRate,
rendererType: this.rendererType,
parameterCount: this.parameterSystem.getParameterCount(),
// Include CV stats if available
cv: cvStats ? {
activeFeatures: cvStats.activeFeatures,
processingTime: cvStats.processingTime,
targetFPS: cvStats.effectiveFPS,
actualFPS: cvStats.actualFPS,
isProcessing: cvStats.isProcessing
} : void 0
});
}
this.lastEffectiveRateReport = currentTime;
}
}
handleResolutionUpdate(message) {
if (message.data) {
if (this.canvas) {
this.canvas.width = Math.round(message.data.effectiveWidth);
this.canvas.height = Math.round(message.data.effectiveHeight);
}
this.viji.width = Math.round(message.data.effectiveWidth);
this.viji.height = Math.round(message.data.effectiveHeight);
if (this.gl) {
this.gl.viewport(0, 0, this.viji.width, this.viji.height);
}
if (this.p5Adapter && this.rendererType === "p5") {
this.p5Adapter.resize(this.viji.width, this.viji.height);
}
if (this.shaderAdapter && this.rendererType === "shader") {
this.shaderAdapter.resize(this.viji.width, this.viji.height);
}
this.debugLog("Canvas resolution updated to:", this.viji.width + "x" + this.viji.height);
}
}
handleParameterUpdate(message) {
if (message.data && message.data.name !== void 0 && message.data.value !== void 0) {
this.parameterSystem.updateParameterValue(message.data.name, message.data.value);
}
}
handleParameterBatchUpdate(message) {
if (message.data && message.data.updates) {
for (const update of message.data.updates) {
this.parameterSystem.updateParameterValue(update.name, update.value);
}
this.parameterSystem.markInitialValuesSynced();
this.debugLog("Parameter system initialized successfully");
}
}
handleStreamUpdate(message) {
this.debugLog("Stream update:", message.data);
}
handleAudioAnalysisUpdate(message) {
this.audioState = {
isConnected: message.data.isConnected,
volume: message.data.volume,
bands: message.data.bands,
frequencyData: new Uint8Array(message.data.frequencyData)
};
this.viji.audio = {
...this.audioState,
getFrequencyData: () => this.audioState.frequencyData
};
}
handleVideoCanvasSetup(message) {
this.videoSystem.handleCanvasSetup({
offscreenCanvas: message.data.offscreenCanvas,
width: message.data.width,
height: message.data.height,
timestamp: message.data.timestamp
});
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
handleVideoFrameUpdate(message) {
this.videoSystem.handleFrameUpdate({
imageBitmap: message.data.imageBitmap,
timestamp: message.data.timestamp
});
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
handleVideoConfigUpdate(message) {
this.videoSystem.handleVideoConfigUpdate({
...message.data.targetFrameRate && { targetFrameRate: message.data.targetFrameRate },
...message.data.cvConfig && { cvConfig: message.data.cvConfig },
...message.data.width && { width: message.data.width },
...message.data.height && { height: message.data.height },
...message.data.disconnect && { disconnect: message.data.disconnect },
timestamp: message.data.timestamp
});
Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
}
handlePerformanceUpdate(message) {
this.debugLog("Performance update:", message.data);
}
/**
* Handle capture-frame request from host.
* Defers capture to immediately after the next render completes to avoid race conditions.
*/
async handleCaptureFrame(message) {
this.pendingCaptures.push(message);
this.debugLog(`Capture request queued (${this.pendingCaptures.length} pending)`);
}
/**
* Execute a capture frame request immediately after render completes.
* This ensures we capture a fully rendered frame, avoiding race conditions.
*/
async executeCaptureFrame(message) {
try {
if (!this.canvas) {
throw new Error("Canvas not initialized");
}
const mimeType = message.data.type || "image/jpeg";
const srcWidth = this.canvas.width;
const srcHeight = this.canvas.height;
let targetWidth = srcWidth;
let targetHeight = srcHeight;
if (typeof message.data.resolution === "number") {
const scale = message.data.resolution > 0 ? message.data.resolution : 1;
targetWidth = Math.max(1, Math.floor(srcWidth * scale));
targetHeight = Math.max(1, Math.floor(srcHeight * scale));
} else if (message.data.resolution && typeof message.data.resolution === "object") {
targetWidth = Math.max(1, Math.floor(message.data.resolution.width));
targetHeight = Math.max(1, Math.floor(message.data.resolution.height));
}
const srcAspect = srcWidth / srcHeight;
const dstAspect = targetWidth / targetHeight;
let sx = 0;
let sy = 0;
let sWidth = srcWidth;
let sHeight = srcHeight;
if (Math.abs(srcAspect - dstAspect) > 1e-6) {
if (dstAspect > srcAspect) {
sHeight = Math.floor(srcWidth / dstAspect);
sy = Math.floor((srcHeight - sHeight) / 2);
} else {
sWidth = Math.floor(srcHeight * dstAspect);
sx = Math.floor((srcWidth - sWidth) / 2);
}
}
let sourceCanvas;
const gl2 = this.canvas.getContext("webgl2");
const gl = gl2 || this.canvas.getContext("webgl");
if (gl) {
gl.finish();
const pixels = new Uint8Array(srcWidth * srcHeight * 4);
gl.readPixels(0, 0, srcWidth, srcHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
const flippedPixels = new Uint8ClampedArray(srcWidth * srcHeight * 4);
for (let y = 0; y < srcHeight; y++) {
const srcRow = (srcHeight - 1 - y) * srcWidth * 4;
const dstRow = y * srcWidth * 4;
flippedPixels.set(pixels.subarray(srcRow, srcRow + srcWidth * 4), dstRow);
}
sourceCanvas = new OffscreenCanvas(srcWidth, srcHeight);
const sourceCtx = sourceCanvas.getContext("2d");
if (!sourceCtx) throw new Error("Failed to create source context");
const imageData = new ImageData(flippedPixels, srcWidth, srcHeight);
sourceCtx.putImageData(imageData, 0, 0);
this.debugLog("Captured frame from WebGL using readPixels (post-render)");
} else {
sourceCanvas = this.canvas;
this.debugLog("Captured frame from 2D canvas (post-render)");
}
const temp = new OffscreenCanvas(targetWidth, targetHeight);
const tctx = temp.getContext("2d");
if (!tctx) throw new Error("Failed to get 2D context");
tctx.drawImage(sourceCanvas, sx, sy, sWidth, sHeight, 0, 0, targetWidth, targetHeight);
const blob = await temp.convertToBlob({ type: mimeType });
const arrayBuffer = await blob.arrayBuffer();
self.postMessage({
type: "capture-frame-result",
id: message.id,
timestamp: Date.now(),
data: {
mimeType,
buffer: arrayBuffer,
width: targetWidth,
height: targetHeight
}
}, [arrayBuffer]);
} catch (error) {
this.postMessage("error", {
id: message.id,
message: error.message,
code: "CAPTURE_FRAME_ERROR"
});
}
}
handleSetSceneCode(message) {
if (message.data && message.data.sceneCode) {
self.setSceneCode(message.data.sceneCode);
}
}
startRenderLoop() {
this.isRunning = true;
this.startTime = performance.now();
this.lastTime = this.startTime;
this.renderFrame();
}
renderFrame() {
if (!this.isRunning) return;
const currentTime = performance.now();
this.interactionSystem.frameStart();
this.viji.fps = this.frameRateMode === "full" ? this.screenRefreshRate : this.screenRefreshRate / 2;
let shouldRender = true;
if (this.frameRateMode === "half") {
shouldRender = !this.skipNextFrame;
this.skipNextFrame = !this.skipNextFrame;
}
if (shouldRender) {
this.viji.deltaTime = (currentTime - this.lastTime) / 1e3;
this.viji.time = (currentTime - this.startTime) / 1e3;
this.viji.frameCount = ++this.frameCount;
this.trackEffectiveFrameTime(currentTime);
this.lastTime = currentTime;
try {
if (this.shaderAdapter && this.rendererType === "shader") {
const parameterObjects = this.parameterSystem.getAllParameterObjects();
this.shaderAdapter.render(this.viji, parameterObjects);
} else if (this.p5Adapter && this.rendererType === "p5") {
const parameterObjects = this.parameterSystem.getAllParameterObjects();
this.p5Adapter.tick(this.viji, parameterObjects);
} else {
const renderFunction2 = self.renderFunction;
if (renderFunction2 && typeof renderFunction2 === "function") {
renderFunction2(this.viji);
}
}
} catch (error) {
console.error("Render error:", error);
this.postMessage("error", {
message: error.message,
code: "RENDER_ERROR",
stack: error.stack
});
}
if (this.pendingCaptures.length > 0) {
const captures = [...this.pendingCaptures];
this.pendingCaptures = [];
for (const captureMsg of captures) {
this.executeCaptureFrame(captureMsg).catch((error) => {
console.error("Capture execution error:", error);
});
}
}
}
this.reportPerformanceStats(currentTime);
requestAnimationFrame(() => this.renderFrame());
}
postMessage(type, data) {
self.postMessage({
type,
id: data?.id || `${type}_${Date.now()}`,
timestamp: Date.now(),
data
});
}
// Phase 7: Interaction Message Handlers (delegated to InteractionSystem)
handleMouseUpdate(message) {
this.interactionSystem.handleMouseUpdate(message.data);
}
handleKeyboardUpdate(message) {
this.interactionSystem.handleKeyboardUpdate(message.data);
}
handleTouchUpdate(message) {
this.interactionSystem.handleTouchUpdate(message.data);
}
handleInteractionEnabled(message) {
this.interactionSystem.setInteractionEnabled(message.data.enabled);
}
}
class SceneAnalyzer {
/**
* Detects the renderer type from scene code comments
*
* Looks for:
* - // @renderer shader
* - /* @renderer shader *\/
* - // @renderer p5
* - /* @renderer p5 *\/
*
* @param sceneCode - The artist's scene code to analyze
* @returns The detected renderer type ('shader', 'p5', or 'native')
*/
static detectRendererType(sceneCode) {
if (/\/\/\s*@renderer\s+shader|\/\*\s*@renderer\s+shader\s*\*\//.test(sceneCode)) {
return "shader";
}
if (/\/\/\s*@renderer\s+p5|\/\*\s*@renderer\s+p5\s*\*\//.test(sceneCode)) {
return "p5";
}
return "native";
}
}
const runtime = new VijiWorkerRuntime();
let renderFunction = null;
async function setSceneCode(sceneCode) {
try {
runtime.resetParameterState();
const rendererType = SceneAnalyzer.detectRendererType(sceneCode);
if (rendererType === "shader") {
await runtime.initShaderMode(sceneCode);
runtime.sendAllParametersToHost();
} else if (rendererType === "p5") {
const functionBody = sceneCode + '\nreturn { setup: typeof setup !== "undefined" ? setup : null, render: typeof render !== "undefined" ? render : null };';
const sceneFunction = new Function("viji", "p5", functionBody);
const { setup, render } = sceneFunction(runtime.viji, null);
if (!render) {
throw new Error("P5 mode requires a render(viji, p5) function");
}
await runtime.initP5Mode(setup, render);
runtime.sendAllParametersToHost();
} else {
const functionBody = sceneCode + '\nif (typeof render === "function") {\n return render;\n}\nthrow new Error("Scene code must define a render function");';
const sceneFunction = new Function("viji", functionBody);
renderFunction = sceneFunction(runtime.viji);
self.renderFunction = renderFunction;
runtime.sendAllParametersToHost();
}
} catch (error) {
console.error("Failed to load scene code:", error);
self.postMessage({
type: "error",
id: `scene_error_${Date.now()}`,
timestamp: Date.now(),
data: {
message: `Scene code error: ${error.message}`,
code: "SCENE_CODE_ERROR"
}
});
}
}
self.setSceneCode = setSceneCode;
//# sourceMappingURL=viji.worker-BANvuSYW.js.map