@viji-dev/core/dist/assets/cv-tasks.worker.js

Universal execution engine for Viji Creative scenes

/**
 * MediaPipe Tasks Vision Classic Worker
 * 
 * Classic worker for MediaPipe Tasks Vision processing.
 * Uses importScripts() to load MediaPipe Tasks Vision UMD bundle.
 */

// Define CommonJS environment for MediaPipe bundle
self.exports = {};
self.module = { exports: {} };

// Import MediaPipe Tasks Vision UMD bundle
console.log('🔧 [CV Tasks Worker] Starting to load vision_bundle.js...');
try {
  importScripts('/dist/assets/vision_bundle.js');
  console.log('✅ [CV Tasks Worker] vision_bundle.js loaded successfully');
} catch (error) {
  console.error('❌ [CV Tasks Worker] Failed to load vision_bundle.js:', error);
}

// Debug: Check what's available after import (disabled for production)
// console.log('🔧 [CV Tasks Worker] Available globals after import:', Object.keys(self));
// console.log('🔧 [CV Tasks Worker] module.exports:', self.module.exports);
// console.log('🔧 [CV Tasks Worker] exports:', self.exports);

// MediaPipe model instances
let faceDetector = null;
let faceLandmarker = null;
let handLandmarker = null;
let poseLandmarker = null;
let imageSegmenter = null;

// Vision runtime
let vision = null;
let isInitialized = false;

// Active features tracking
const activeFeatures = new Set();

// Configuration queue to prevent race conditions
const configQueue = [];
let processingConfig = false;

// Worker health tracking
let workerHealthy = true;
let memoryPressureDetected = false;

// Note: No longer need reusable canvas - passing ImageBitmap directly to MediaPipe!

// Debug logging
const DEBUG = true; // Temporarily enabled to debug segmentation
function log(...args) {
  if (DEBUG) {
    console.log('🔧 [CV Tasks Worker]', ...args);
  }
}

/**
 * Initialize MediaPipe Tasks Vision runtime
 */
async function initializeVision() {
  if (isInitialized) {
    console.log('🔧 [CV Tasks Worker] Vision already initialized, skipping');
    return;
  }
  
  try {
    console.log('🔧 [CV Tasks Worker] Starting MediaPipe Tasks Vision initialization...');
    
    // Initialize the vision runtime with WASM files
    // MediaPipe Tasks Vision expects the base path without trailing slash
    const wasmBasePath = '/dist/assets/wasm';
    log('WASM base path:', wasmBasePath);
    
    // Try different ways to access FilesetResolver
    const FilesetResolver = self.FilesetResolver || self.module.exports.FilesetResolver || self.exports.FilesetResolver;
    console.log('🔧 [CV Tasks Worker] FilesetResolver found:', !!FilesetResolver);
    
    if (!FilesetResolver) {
      throw new Error('FilesetResolver not found in any expected location');
    }
    
    vision = await FilesetResolver.forVisionTasks(wasmBasePath);
    
    isInitialized = true;
    log('✅ MediaPipe Tasks Vision initialized successfully');
  } catch (error) {
    log('❌ Failed to initialize MediaPipe Tasks Vision:', error);
    throw error;
  }
}

/**
 * Load and initialize Face Detection
 */
async function initializeFaceDetection() {
  if (faceDetector) return;
  
  // Ensure vision runtime is initialized first
  await initializeVision();
  
  try {
    log('Loading Face Detector...');
    
    const options = {
      baseOptions: {
        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.tflite',
        delegate: 'GPU'
      },
      runningMode: 'VIDEO',
      minDetectionConfidence: 0.5,
      minSuppressionThreshold: 0.3
    };
    
    const FaceDetector = self.FaceDetector || self.module.exports.FaceDetector || self.exports.FaceDetector;
    faceDetector = await FaceDetector.createFromOptions(vision, options);
    log('✅ Face Detector loaded');
  } catch (error) {
    log('❌ Failed to load Face Detector:', error);
    throw error;
  }
}

/**
 * Load and initialize Face Landmarks
 */
async function initializeFaceLandmarks() {
  if (faceLandmarker) return;
  
  // Ensure vision runtime is initialized first
  await initializeVision();
  
  try {
    log('Loading Face Landmarker...');
    
    const options = {
      baseOptions: {
        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
        delegate: 'GPU'
      },
      runningMode: 'VIDEO',
      numFaces: 1
    };
    
    const FaceLandmarker = self.FaceLandmarker || self.module.exports.FaceLandmarker || self.exports.FaceLandmarker;
    faceLandmarker = await FaceLandmarker.createFromOptions(vision, options);
    log('✅ Face Landmarker loaded');
  } catch (error) {
    log('❌ Failed to load Face Landmarker:', error);
    throw error;
  }
}

/**
 * Load and initialize Hand Tracking
 */
async function initializeHandTracking() {
  if (handLandmarker) return;
  
  // Ensure vision runtime is initialized first
  await initializeVision();
  
  try {
    log('Loading Hand Landmarker...');
    
    const options = {
      baseOptions: {
        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
        delegate: 'GPU'
      },
      runningMode: 'VIDEO',
      numHands: 2
    };
    
    const HandLandmarker = self.HandLandmarker || self.module.exports.HandLandmarker || self.exports.HandLandmarker;
    handLandmarker = await HandLandmarker.createFromOptions(vision, options);
    log('✅ Hand Landmarker loaded');
  } catch (error) {
    log('❌ Failed to load Hand Landmarker:', error);
    throw error;
  }
}

/**
 * Load and initialize Pose Detection
 */
async function initializePoseDetection() {
  if (poseLandmarker) return;
  
  // Ensure vision runtime is initialized first
  await initializeVision();
  
  try {
    log('Loading Pose Landmarker...');
    
    const options = {
      baseOptions: {
        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task',
        delegate: 'GPU'
      },
      runningMode: 'VIDEO',
      numPoses: 1
    };
    
    const PoseLandmarker = self.PoseLandmarker || self.module.exports.PoseLandmarker || self.exports.PoseLandmarker;
    poseLandmarker = await PoseLandmarker.createFromOptions(vision, options);
    log('✅ Pose Landmarker loaded');
  } catch (error) {
    log('❌ Failed to load Pose Landmarker:', error);
    throw error;
  }
}

/**
 * Load and initialize Body Segmentation
 */
async function initializeBodySegmentation() {
  if (imageSegmenter) return;
  
  // Ensure vision runtime is initialized first
  await initializeVision();
  
  try {
    log('Loading Image Segmenter...');
    
      const options = {
        baseOptions: {
          modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/1/selfie_segmenter.tflite',
          delegate: 'GPU'
        },
        runningMode: 'IMAGE',
        outputCategoryMask: true,
        outputConfidenceMasks: false
      };
    
    const ImageSegmenter = self.ImageSegmenter || self.module.exports.ImageSegmenter || self.exports.ImageSegmenter;
    imageSegmenter = await ImageSegmenter.createFromOptions(vision, options);
    log('✅ Image Segmenter loaded');
  } catch (error) {
    log('❌ Failed to load Image Segmenter:', error);
    throw error;
  }
}

/**
 * Process video frame with active CV features
 * @param {ImageData|ImageBitmap} imageInput - Image input (ImageData or ImageBitmap)
 * @param {number} timestamp - Frame timestamp  
 * @param {string[]} features - Active CV features
 */
async function processFrame(imageInput, timestamp, features) {
  const results = {};
  
  try {
    // Process face detection
    if (features.includes('faceDetection') && faceDetector) {
      const detectionResult = faceDetector.detectForVideo(imageInput, timestamp);
      results.faces = detectionResult.detections.map((detection) => ({
        boundingBox: {
          // Normalize coordinates to 0-1 range to match other CV features
          x: detection.boundingBox.originX / imageInput.width,
          y: detection.boundingBox.originY / imageInput.height,
          width: detection.boundingBox.width / imageInput.width,
          height: detection.boundingBox.height / imageInput.height
        },
        landmarks: [],
        expressions: {},
        confidence: detection.categories[0]?.score || 0
      }));
    }
    
    // Process face landmarks
    if (features.includes('faceMesh') && faceLandmarker) {
      const landmarkResult = faceLandmarker.detectForVideo(imageInput, timestamp);
      if (landmarkResult.faceLandmarks.length > 0) {
        const landmarks = landmarkResult.faceLandmarks[0];
        
        // If no face detection results exist, create a basic face structure
        if (!results.faces) {
          results.faces = [{
            boundingBox: null, // No bounding box when only mesh is enabled
            landmarks: [],
            expressions: {},
            confidence: 0.8 // Default confidence for mesh-only detection
          }];
        }
        
        // Add landmarks to the first face (mesh only processes one face)
        if (results.faces[0]) {
          results.faces[0].landmarks = landmarks.map((landmark) => ({
            x: landmark.x,
            y: landmark.y,
            z: landmark.z || 0
          }));
        }
      }
    }
    
    // Process hand tracking
    if (features.includes('handTracking') && handLandmarker) {
      const handResult = handLandmarker.detectForVideo(imageInput, timestamp);
      results.hands = handResult.landmarks.map((landmarks, index) => ({
        landmarks: landmarks.map((landmark) => ({
          x: landmark.x,
          y: landmark.y,
          z: landmark.z || 0
        })),
        handedness: handResult.handednesses[index]?.[0]?.categoryName || 'Unknown',
        confidence: handResult.handednesses[index]?.[0]?.score || 0
      }));
    }
    
    // Process pose detection
    if (features.includes('poseDetection') && poseLandmarker) {
      const poseResult = poseLandmarker.detectForVideo(imageInput, timestamp);
      if (poseResult.landmarks.length > 0) {
        results.pose = {
          landmarks: poseResult.landmarks[0].map((landmark) => ({
            x: landmark.x,
            y: landmark.y,
            z: landmark.z || 0,
            visibility: landmark.visibility || 1
          })),
          worldLandmarks: poseResult.worldLandmarks?.[0]?.map((landmark) => ({
            x: landmark.x,
            y: landmark.y,
            z: landmark.z || 0,
            visibility: landmark.visibility || 1
          })) || []
        };
      }
    }
    
    // Process body segmentation
    if (features.includes('bodySegmentation') && imageSegmenter) {
      const segmentResult = imageSegmenter.segment(imageInput);
      if (segmentResult.categoryMask) {
        try {
          // Extract data before closing the mask
          results.segmentation = {
            mask: segmentResult.categoryMask.getAsUint8Array(),
            width: segmentResult.categoryMask.width,
            height: segmentResult.categoryMask.height
          };
          
          // Debug logging (temporary)
          if (DEBUG) {
            console.log('🔧 [CV Tasks Worker] Segmentation mask:', {
              width: results.segmentation.width, 
              height: results.segmentation.height,
              maskSize: results.segmentation.mask.length
            });
          }
        } finally {
          // CRITICAL: Close MPMask instance to prevent resource leaks
          segmentResult.categoryMask.close();
        }
      }
    }
    
    return results;
  } catch (error) {
    log('❌ Error processing frame:', error);
    return {};
  }
}

// Note: Removed reusable canvas functions - no longer needed with direct ImageBitmap processing!

/**
 * Clean up WASM instance with proper memory management
 */
function cleanupWasmInstance(instance, featureName) {
  if (instance) {
    try {
      log(`🧹 Cleaning up ${featureName} WASM instance...`);
      instance.close();
      
      // Force garbage collection if available (Chrome DevTools)
      if (typeof gc === 'function') {
        gc();
      }
      
      // Give time for WASM cleanup
      return new Promise(resolve => {
        setTimeout(resolve, 100);
      });
    } catch (error) {
      log(`⚠️ Error cleaning up ${featureName}:`, error);
    }
  }
  return Promise.resolve();
}

/**
 * Process configuration queue sequentially
 */
async function processConfigQueue() {
  if (processingConfig || configQueue.length === 0) return;
  
  processingConfig = true;
  
  try {
    while (configQueue.length > 0) {
      const { features, resolve, reject } = configQueue.shift();
      
      try {
        await handleConfigUpdateInternal(features);
        resolve({ configured: true, activeFeatures: Array.from(activeFeatures) });
      } catch (error) {
        reject(error);
      }
    }
  } finally {
    processingConfig = false;
  }
}

/**
 * Queue configuration update to prevent race conditions
 */
function queueConfigUpdate(features) {
  return new Promise((resolve, reject) => {
    configQueue.push({ features, resolve, reject });
    processConfigQueue();
  });
}

/**
 * Handle feature configuration updates (internal)
 */
async function handleConfigUpdateInternal(features) {
  if (!workerHealthy) {
    throw new Error('Worker is in unhealthy state, restart required');
  }
  
  const newFeatures = new Set(features);
  const toEnable = features.filter(f => !activeFeatures.has(f));
  const toDisable = Array.from(activeFeatures).filter(f => !newFeatures.has(f));
  
  log(`🔄 Config update: enable [${toEnable.join(', ')}], disable [${toDisable.join(', ')}]`);
  
  // Disable unused features first (cleanup instances)
  const cleanupPromises = [];
  for (const feature of toDisable) {
    switch (feature) {
      case 'faceDetection':
        cleanupPromises.push(cleanupWasmInstance(faceDetector, 'FaceDetector'));
        faceDetector = null;
        break;
      case 'faceMesh':
        cleanupPromises.push(cleanupWasmInstance(faceLandmarker, 'FaceLandmarker'));
        faceLandmarker = null;
        break;
      case 'handTracking':
        cleanupPromises.push(cleanupWasmInstance(handLandmarker, 'HandLandmarker'));
        handLandmarker = null;
        break;
      case 'poseDetection':
        cleanupPromises.push(cleanupWasmInstance(poseLandmarker, 'PoseLandmarker'));
        poseLandmarker = null;
        break;
      case 'bodySegmentation':
        cleanupPromises.push(cleanupWasmInstance(imageSegmenter, 'ImageSegmenter'));
        imageSegmenter = null;
        break;
    }
    activeFeatures.delete(feature);
    log(`🗑️ Disabled feature: ${feature}`);
  }
  
  // Wait for all cleanup to complete
  if (cleanupPromises.length > 0) {
    await Promise.all(cleanupPromises);
    log('✅ All cleanup completed');
  }
  
  // Note: No canvas cleanup needed - using direct ImageBitmap processing!
  
  // Enable new features
  for (const feature of toEnable) {
    try {
      switch (feature) {
        case 'faceDetection':
          await initializeFaceDetection();
          break;
        case 'faceMesh':
          await initializeFaceLandmarks();
          break;
        case 'handTracking':
          await initializeHandTracking();
          break;
        case 'poseDetection':
          await initializePoseDetection();
          break;
        case 'bodySegmentation':
          await initializeBodySegmentation();
          break;
      }
      activeFeatures.add(feature);
      log(`✅ Enabled feature: ${feature}`);
    } catch (error) {
      log(`❌ Failed to enable feature ${feature}:`, error);
      
      // Check if this is a memory error
      if (error.message && error.message.includes('Out of memory')) {
        memoryPressureDetected = true;
        workerHealthy = false;
        throw new Error(`Memory exhausted while enabling ${feature}. Worker restart required.`);
      }
      
      throw error;
    }
  }
}

/**
 * Legacy function for backward compatibility
 */
async function handleConfigUpdate(features) {
  return await queueConfigUpdate(features);
}

// Message handler
self.onmessage = async (event) => {
  const message = event.data;
  
  console.log('🔧 [CV Tasks Worker] Received message:', message.type, message);
  
  try {
    switch (message.type) {
      case 'init': {
        log('Received init message');
        
        try {
          await initializeVision();
          log('Vision runtime ready for feature loading');
        } catch (error) {
          log('❌ Vision runtime initialization failed:', error);
          throw error;
        }
        
        const response = {
          type: 'result',
          success: true,
          data: { initialized: true }
        };
        self.postMessage(response);
        break;
      }
      
      case 'config': {
        log('Received config message:', message.features);
        
        try {
          const result = await handleConfigUpdate(message.features);
          
          const response = {
            type: 'result',
            success: true,
            data: result
          };
          self.postMessage(response);
        } catch (error) {
          log('❌ Config update failed:', error);
          
          // Check if worker needs restart
          if (!workerHealthy || memoryPressureDetected) {
            const errorResponse = {
              type: 'result',
              success: false,
              error: error.message,
              restartRequired: true
            };
            self.postMessage(errorResponse);
          } else {
            throw error; // Re-throw for normal error handling
          }
        }
        break;
      }
      
      case 'process': {
        try {
          // 🚀 OPTIMIZED: Pass ImageBitmap directly to MediaPipe (no conversion!)
          const results = await processFrame(message.bitmap, message.timestamp, message.features);
          
          const response = {
            type: 'result',
            success: true,
            data: results
          };
          self.postMessage(response);
        } finally {
          // Clean up ImageBitmap after processing
          if (message.bitmap && typeof message.bitmap.close === 'function') {
            message.bitmap.close();
          }
        }
        break;
      }
      
      default:
        log('❌ Unknown message type:', message.type);
        const errorResponse = {
          type: 'result',
          success: false,
          error: `Unknown message type: ${message.type}`
        };
        self.postMessage(errorResponse);
    }
  } catch (error) {
    log('❌ Error handling message:', error);
    const errorResponse = {
      type: 'result',
      success: false,
      error: error instanceof Error ? error.message : String(error)
    };
    self.postMessage(errorResponse);
  }
};

log('CV Tasks Worker initialized and ready');