id-scanner-lib/src/modules/face/face-detector.ts

Browser-based ID card, QR code, and face recognition scanner with liveness detection

/* eslint-disable */
/**
 * @file 人脸检测模块
 * @description 提供人脸检测、跟踪和分析功能
 * @module modules/face/face-detector
 */

import * as tf from '@tensorflow/tfjs';
import * as faceapi from '@vladmandic/face-api';

import { BaseScannerModule, ModuleCapabilities, ModuleEvent, ModuleInitOptions, ModuleStatus, ModuleType } from '../../interfaces/scanner-module';
import { FaceDetectionOptions, FaceDetectionResult, LivenessDetectionType, Rect } from '../../interfaces/face-detection';
import { ConfigManager } from '../../core/config';
import { Logger } from '../../core/logger';
import { ResourceManager } from '../../core/resource-manager';
import { CameraManager, CameraEvent } from '../../core/camera-manager';
import { Result } from '../../core/result';
import { FaceDetectionError, FaceComparisonError, InitializationError, LivenessDetectionError, ResourceLoadError } from '../../core/errors';
import { generateUUID } from '../../utils';
import { FaceModelLoader } from './face-model-loader';
import { FaceTracker } from './face-tracker';
import { FaceComparator } from './face-comparator';
import { FaceResultConverter } from './face-result-converter';
import { FaceDetectorOptionsFactory } from './face-detector-options';

/**
 * 人脸检测模型类型
 */
export enum FaceModelType {
  /** SSD MobileNet V1 模型 */
  SSD_MOBILENET = 'ssd_mobilenetv1',
  /** Tiny Face 模型 */
  TINY_FACE = 'tiny_face',
  /** MTCNN 模型 */
  MTCNN = 'mtcnn',
  /** BlazeFace 模型 */
  BLAZE_FACE = 'blazeface'
}

/**
 * 68点人脸关键点索引 (face-api 68-point model)
 * 参考: https://github.com/justadudewhohacks/face-api.js/issues/175
 */
const FaceLandmarkIndex = {
  LEFT_EYE: 36,
  RIGHT_EYE: 45,
  NOSE: 30,
  MOUTH: 48,  // 上唇中心
  LEFT_EYE_CORNER: 36,
  RIGHT_EYE_CORNER: 45,
  NOSE_TIP: 30,
  MOUTH_CENTER: 57,
} as const;

/**
 * 人脸检测模块配置
 */
export interface FaceDetectorConfig {
  /** 是否启用 */
  enabled: boolean;
  /** 检测模型类型 */
  detectionModel: FaceModelType;
  /** 置信度阈值 */
  minConfidence: number;
  /** 最大检测人脸数 */
  maxFaces: number;
  /** 是否检测关键点 */
  detectLandmarks: boolean;
  /** 关键点模型类型 */
  landmarksModel: 'tiny' | '68_points';
  /** 是否检测表情 */
  detectExpressions: boolean;
  /** 是否检测年龄和性别 */
  detectAgeGender: boolean;
  /** 是否提取人脸特征向量 */
  extractEmbeddings: boolean;
  /** 人脸匹配阈值(0-1) */
  matchThreshold: number;
  /** 是否启用跟踪 */
  enableTracking: boolean;
  /** 活体检测类型 */
  livenessDetection: LivenessDetectionType | 'none';
  /** 模型路径 */
  modelPath: string;
}

/**
 * 人脸检测模块
 */
export class FaceDetector extends BaseScannerModule {
  /** 模块类型 */
  readonly type: ModuleType = ModuleType.FACE;
  
  /** 模块配置 */
  protected config: FaceDetectorConfig;
  
  /** 默认配置 */
  private static readonly DEFAULT_CONFIG: FaceDetectorConfig = {
    enabled: true,
    detectionModel: FaceModelType.SSD_MOBILENET,
    minConfidence: 0.5,
    maxFaces: 10,
    detectLandmarks: true,
    landmarksModel: 'tiny',
    detectExpressions: false,
    detectAgeGender: false,
    extractEmbeddings: false,
    matchThreshold: 0.6,
    enableTracking: false,
    livenessDetection: 'none',
    modelPath: '/models/face-api'
  };
  
  /** 模型加载器 */
  private modelLoader: FaceModelLoader;

  /** 人脸跟踪器 */
  private faceTracker: FaceTracker;

  /** 人脸比对器 */
  private faceComparator: FaceComparator;

  /** 结果转换器 */
  private resultConverter: FaceResultConverter;

  /** 处理计时器ID */
  private processingTimerId: number | null = null;

  /** 处理间隔(ms) */
  private processingInterval: number = 100;

  /** 摄像头管理器 */
  private cameraManager: CameraManager;

  /** 配置管理器 */
  private configManager: ConfigManager;

  /** 资源管理器 */
  private resourceManager: ResourceManager;

  /** 日志记录器 */
  private logger: Logger;

  /** 画布元素，用于处理帧 */
  private canvas: HTMLCanvasElement;

  /** 画布渲染上下文 */
  private canvasCtx: CanvasRenderingContext2D | null = null;

  /** 最后一次检测结果 */
  private lastDetectionResult: FaceDetectionResult[] = [];

  /**
   * 构造函数
   * @param config 初始配置
   */
  constructor(config: Partial<FaceDetectorConfig> = {}) {
    super({
      enabled: true,
      ...config
    });

    this.configManager = ConfigManager.getInstance();
    this.cameraManager = CameraManager.getInstance();
    this.resourceManager = ResourceManager.getInstance();
    this.logger = Logger.getInstance();

    // 合并配置
    this.config = {
      ...FaceDetector.DEFAULT_CONFIG,
      ...config
    };

    // 初始化组件
    this.modelLoader = new FaceModelLoader({
      modelPath: this.config.modelPath,
      detectionModel: this.config.detectionModel,
      landmarksModel: this.config.landmarksModel,
      detectLandmarks: this.config.detectLandmarks,
      detectExpressions: this.config.detectExpressions,
      detectAgeGender: this.config.detectAgeGender,
      extractEmbeddings: this.config.extractEmbeddings
    });

    this.faceTracker = new FaceTracker({ trackTimeout: 1000 });
    this.faceComparator = new FaceComparator({ matchThreshold: this.config.matchThreshold });
    this.resultConverter = new FaceResultConverter({
      detectLandmarks: this.config.detectLandmarks,
      landmarksModel: this.config.landmarksModel,
      detectExpressions: this.config.detectExpressions,
      detectAgeGender: this.config.detectAgeGender,
      extractEmbeddings: this.config.extractEmbeddings
    });

    // 创建画布
    this.canvas = document.createElement('canvas');
    this.canvasCtx = this.canvas.getContext('2d');
  }
  
  /**
   * 获取模块能力
   */
  get capabilities(): ModuleCapabilities {
    return {
      supportsVideo: true,
      supportsImage: true,
      supportsBatch: false,
      supportsRealtime: true,
      supportsWebWorker: false,
      supportedMediaTypes: ['image/jpeg', 'image/png', 'image/webp']
    };
  }
  
  /**
   * 初始化模块
   * @param options 初始化选项
   */
  async initialize(options?: ModuleInitOptions): Promise<void> {
    if (this._status === ModuleStatus.INITIALIZING) {
      throw new Error('人脸检测模块正在初始化中');
    }
    
    if (this._status === ModuleStatus.READY) {
      this.logger.debug('FaceDetector', '人脸检测模块已初始化');
      return;
    }
    
    this.setStatus(ModuleStatus.INITIALIZING);
    this.emit(ModuleEvent.INIT_START);
    
    try {
      // 应用配置选项
      if (options?.config) {
        this.updateConfig(options.config);
      }
      
      // 设置调试模式
      if (options?.debug !== undefined) {
        this.debug = options.debug;
      }
      
      const modelPath = options?.modelPath || this.config.modelPath;
      
      // 加载模型
      this.logger.info('FaceDetector', `正在加载人脸检测模型，路径：${modelPath}`);
      
      // 设置模型路径
      faceapi.env.monkeyPatch({
        Canvas: HTMLCanvasElement,
        Image: HTMLImageElement,
        ImageData: ImageData,
        Video: HTMLVideoElement,
        createCanvasElement: () => document.createElement('canvas'),
        createImageElement: () => document.createElement('img')
      });
      
      // 确保TensorFlow.js已初始化
      await tf.ready();
      
      // 设置模型路径并加载模型
      await this.loadModels(modelPath);
      
      // 绑定摄像头事件
      if (options?.bindCamera) {
        this.cameraManager.on(CameraEvent.FRAME, this.handleCameraFrame.bind(this));
      }
      
      this.setStatus(ModuleStatus.READY);
      this.emit(ModuleEvent.INIT_COMPLETE);
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : String(error);
      this.logger.error('FaceDetector', `初始化失败: ${errorMessage}`, error as Error);
      
      this.setStatus(ModuleStatus.ERROR);
      this.emit(ModuleEvent.INIT_ERROR, { error });
      
      throw new Error(`人脸检测模块初始化失败: ${errorMessage}`);
    }
  }
  
  // ==================== 模型加载（委托给 FaceModelLoader） ====================

  /**
   * 懒加载模型 - 委托给 FaceModelLoader
   * @deprecated 使用 modelLoader.lazyLoadModel 代替
   */
  private async lazyLoadModel(modelType: string, modelPath: string): Promise<void> {
    await this.modelLoader.lazyLoadModel(modelType, modelPath);
  }

  /**
   * 根据需求加载模型 - 委托给 FaceModelLoader
   * @deprecated 使用 modelLoader.loadModelsOnDemand 代替
   */
  private async loadModelsOnDemand(options: FaceDetectionOptions, modelPath: string): Promise<void> {
    await this.modelLoader.loadModelsOnDemand({
      withLandmarks: options.withLandmarks,
      withAttributes: options.withAttributes,
      withEmbedding: options.withEmbedding
    });
  }

  /**
   * 加载人脸检测模型 - 委托给 FaceModelLoader
   * @deprecated 使用 modelLoader.ensureModelsLoaded 代替
   */
  private async loadModels(modelPath: string): Promise<void> {
    await this.modelLoader.ensureModelsLoaded();
  }
  
  /**
   * 处理图片
   * @param image 图片源
   * @param options 处理选项
   */
  async processImage(
    image: string | HTMLImageElement | HTMLCanvasElement | ImageData,
    options: FaceDetectionOptions = {}
  ): Promise<Result<FaceDetectionResult[]>> {
    this.checkInitialized();
    
    if (this._status === ModuleStatus.PROCESSING) {
      return Result.failure(new FaceDetectionError('另一个处理操作正在进行中'));
    }
    
    this.setStatus(ModuleStatus.PROCESSING);
    this.emit(ModuleEvent.PROCESS_START);

    try {
      // 懒加载所需的模型
      const modelPath = this.config.modelPath || '/models';
      await this.loadModelsOnDemand(options, modelPath);
      
      // 合并选项和配置
      const processOptions: FaceDetectionOptions = {
        minConfidence: this.config.minConfidence,
        maxFaces: this.config.maxFaces,
        withLandmarks: this.config.detectLandmarks,
        withAttributes: this.config.detectExpressions || this.config.detectAgeGender,
        withEmbedding: this.config.extractEmbeddings,
        ...options
      };
      
      // 加载图片
      let imgElement: HTMLImageElement | HTMLCanvasElement;
      
      if (typeof image === 'string') {
        imgElement = await this.loadImage(image);
      } else if (image instanceof HTMLImageElement || image instanceof HTMLCanvasElement) {
        imgElement = image;
      } else if (image instanceof ImageData) {
        // 将ImageData转换为Canvas
        const canvas = document.createElement('canvas');
        canvas.width = image.width;
        canvas.height = image.height;
        const ctx = canvas.getContext('2d');
        ctx?.putImageData(image, 0, 0);
        imgElement = canvas;
      } else {
        throw new FaceDetectionError('不支持的图像格式');
      }
      
      // 开始计时
      const startTime = Date.now();
      
      // 执行人脸检测
      const results = await this.detectFaces(imgElement, processOptions);
      
      // 计算处理时间
      const processingTime = Date.now() - startTime;
      
      this.setStatus(ModuleStatus.READY);
      this.emit(ModuleEvent.PROCESS_COMPLETE, { results, processingTime });
      
      return Result.success(results);
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : String(error);
      this.logger.error('FaceDetector', `图片处理失败: ${errorMessage}`, error as Error);
      
      this.setStatus(ModuleStatus.ERROR);
      this.emit(ModuleEvent.PROCESS_ERROR, { error });
      
      return Result.failure(new FaceDetectionError(`图片处理失败: ${errorMessage}`));
    }
  }
  
  /**
   * 开始实时处理
   * @param videoElement 视频元素
   * @param options 处理选项
   */
  async startRealtime(
    videoElement?: HTMLVideoElement,
    options: FaceDetectionOptions = {}
  ): Promise<Result<boolean>> {
    this.checkInitialized();
    
    if (this._status === ModuleStatus.PROCESSING) {
      return Result.failure(new FaceDetectionError('实时处理已在进行中'));
    }
    
    try {
      // 停止现有处理
      this.stopRealtime();
      
      // 获取视频元素
      const video = videoElement || this.cameraManager.getVideoElement();
      
      if (!video) {
        throw new FaceDetectionError('未提供视频元素且摄像头未初始化');
      }
      
      // 如果视频未播放，尝试启动摄像头
      if (!this.cameraManager.isActive() && !videoElement) {
        const cameraResult = await this.cameraManager.init({ autoStart: true });
        if (!cameraResult.isSuccess()) {
          throw new Error('无法启动摄像头');
        }
      }
      
      // 设置处理间隔
      this.processingInterval = options.processingInterval || 100;
      
      // 设置状态
      this.setStatus(ModuleStatus.PROCESSING);
      
      // 启动处理循环
      this.processingTimerId = window.setInterval(() => {
        this.processVideoFrame(video, options);
      }, this.processingInterval);
      
      return Result.success(true);
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : String(error);
      this.logger.error('FaceDetector', `启动实时处理失败: ${errorMessage}`, error as Error);
      
      this.setStatus(ModuleStatus.ERROR);
      
      return Result.failure(new FaceDetectionError(`启动实时处理失败: ${errorMessage}`));
    }
  }
  
  /**
   * 停止实时处理
   */
  stopRealtime(): void {
    if (this.processingTimerId !== null) {
      window.clearInterval(this.processingTimerId);
      this.processingTimerId = null;
    }
    
    if (this._status === ModuleStatus.PROCESSING) {
      this.setStatus(ModuleStatus.READY);
    }
    
    // 清除人脸跟踪状态
    this.faceTracker.reset();
    this.lastDetectionResult = [];
  }
  
  /**
   * 释放资源
   */
  async dispose(): Promise<void> {
    // 停止实时处理
    this.stopRealtime();

    // 释放模型（通过 FaceModelLoader）
    await this.modelLoader.dispose();

    // 重置跟踪器
    this.faceTracker.reset();

    // 移除事件监听
    this.cameraManager.off(CameraEvent.FRAME, this.handleCameraFrame.bind(this));

    this._status = ModuleStatus.NOT_INITIALIZED;
  }
  
  /**
   * 加载图片
   * @param src 图片URL
   */
  private async loadImage(src: string): Promise<HTMLImageElement> {
    return new Promise((resolve, reject) => {
      const img = new Image();
      img.crossOrigin = 'anonymous';
      img.onload = () => resolve(img);
      img.onerror = () => reject(new Error(`无法加载图片: ${src}`));
      img.src = src;
    });
  }
  
  /**
   * 处理视频帧
   * @param video 视频元素
   * @param options 处理选项
   */
  private async processVideoFrame(
    video: HTMLVideoElement,
    options: FaceDetectionOptions = {}
  ): Promise<void> {
    if (this._status !== ModuleStatus.PROCESSING || !video || video.paused || video.ended) {
      return;
    }
    
    try {
      // 检查视频是否准备好
      if (video.readyState < 2) { // HAVE_CURRENT_DATA
        return;
      }
      
      // 检查视频尺寸
      if (video.videoWidth === 0 || video.videoHeight === 0) {
        return;
      }
      
      // 调整画布大小
      if (this.canvas.width !== video.videoWidth || this.canvas.height !== video.videoHeight) {
        this.canvas.width = video.videoWidth;
        this.canvas.height = video.videoHeight;
      }
      
      // 将视频帧绘制到画布
      if (this.canvasCtx) {
        this.canvasCtx.drawImage(video, 0, 0);
      }
      
      // 执行人脸检测
      const startTime = Date.now();
      const results = await this.detectFaces(video, options);
      const processingTime = Date.now() - startTime;
      
      // 更新最后的检测结果
      this.lastDetectionResult = results;
      
      // 发出实时结果事件
      this.emit(ModuleEvent.REALTIME_RESULT, { 
        results, 
        processingTime,
        timestamp: Date.now() 
      });
    } catch (error) {
      this.logger.error('FaceDetector', `处理视频帧失败: ${error}`);
    }
  }
  
  /**
   * 处理摄像头帧
   */
  private handleCameraFrame(event: any): void {
    if (this._status !== ModuleStatus.PROCESSING || !event.frameData) {
      return;
    }
    
    const { frameData } = event;
    
    // 调整画布大小
    if (this.canvas.width !== frameData.width || this.canvas.height !== frameData.height) {
      this.canvas.width = frameData.width;
      this.canvas.height = frameData.height;
    }
    
    // 将帧数据绘制到画布
    if (this.canvasCtx) {
      this.canvasCtx.putImageData(frameData, 0, 0);
      
      // 执行人脸检测
      this.detectFaces(this.canvas).then(results => {
        // 更新最后的检测结果
        this.lastDetectionResult = results;
        
        // 发出实时结果事件
        this.emit(ModuleEvent.REALTIME_RESULT, { 
          results, 
          timestamp: Date.now() 
        });
      }).catch(error => {
        this.logger.error('FaceDetector', `处理摄像头帧失败: ${error}`);
      });
    }
  }
  
  /**
   * 执行人脸检测
   * @param input 输入图像
   * @param options 检测选项
   */
  private async detectFaces(
    input: HTMLImageElement | HTMLCanvasElement | HTMLVideoElement,
    options: FaceDetectionOptions = {}
  ): Promise<FaceDetectionResult[]> {
    try {
      // 检查模型是否已加载
      if (!this.modelLoader.isModelsLoaded()) {
        throw new FaceDetectionError('人脸检测模型尚未加载');
      }
      
      // 合并选项和配置
      const detectOptions = FaceDetectorOptionsFactory.mergeOptions(this.config, options);

      // 创建 face-api 检测选项
      const faceapiOptions = FaceDetectorOptionsFactory.createFaceAPIOptions(
        this.config.detectionModel,
        detectOptions.minConfidence ?? 0.5
      );

      // 进行检测
      const startTime = Date.now();
      const detections = await FaceDetectorOptionsFactory.detect(
        input,
        faceapiOptions,
        detectOptions,
        this.config.landmarksModel
      );
      
      // 限制检测数量
      const maxFaces = detectOptions.maxFaces || this.config.maxFaces;
      const detectionsArray: any[] = Array.isArray(detections) ? detections : [detections];
      if (detectionsArray.length > maxFaces) {
        detectionsArray.length = maxFaces;
      }

      // 将结果转换为标准格式
      const processingTime = Date.now() - startTime;

      // 临时更新转换器配置
      this.resultConverter.updateConfig({
        detectLandmarks: !!detectOptions.withLandmarks,
        detectExpressions: !!detectOptions.withAttributes,
        detectAgeGender: !!detectOptions.withAttributes,
        extractEmbeddings: !!detectOptions.withEmbedding
      });

      const results = this.resultConverter.convertBatch(detectionsArray, { maxFaces }, processingTime);

      // 处理人脸跟踪（使用 FaceTracker）
      if (detectOptions.enableTracking) {
        const trackedResults = this.faceTracker.update(results);
        for (let i = 0; i < trackedResults.length; i++) {
          if (trackedResults[i].trackId) {
            results[i].trackId = trackedResults[i].trackId;
          }
        }
      }

      return results;
    } catch (error) {
      this.logger.error('FaceDetector', `人脸检测失败: ${error}`);
      throw new FaceDetectionError(`人脸检测失败: ${error}`);
    }
  }
  
  /**
   * 比对两个人脸
   * @param source 源人脸
   * @param target 目标人脸
   */
  async compareFaces(
    source: string | HTMLImageElement | FaceDetectionResult,
    target: string | HTMLImageElement | FaceDetectionResult
  ): Promise<Result<{ similarity: number; isMatch: boolean; threshold: number }>> {
    this.checkInitialized();
    
    try {
      // 获取源人脸的特征向量
      let sourceEmbedding: number[];
      
      if (typeof source === 'string' || source instanceof HTMLImageElement) {
        // 处理图片源
        const result = await this.processImage(source, { withEmbedding: true });
        const resultData = result.getData();
        if (!result.isSuccess() || !resultData || resultData.length === 0) {
          throw new FaceComparisonError('无法从源图像检测人脸');
        }
        if (!resultData[0].embedding) {
          throw new FaceComparisonError('源图像未提取特征向量');
        }
        sourceEmbedding = resultData[0].embedding.vector;
      } else {
        // 使用现有检测结果
        if (!source.embedding || !source.embedding.vector) {
          throw new FaceComparisonError('源人脸未提取特征向量');
        }
        sourceEmbedding = source.embedding.vector;
      }
      
      // 获取目标人脸的特征向量
      let targetEmbedding: number[];
      
      if (typeof target === 'string' || target instanceof HTMLImageElement) {
        // 处理图片源
        const result = await this.processImage(target, { withEmbedding: true });
        const resultData = result.getData();
        if (!result.isSuccess() || !resultData || resultData.length === 0) {
          throw new FaceComparisonError('无法从目标图像检测人脸');
        }
        if (!resultData[0].embedding) {
          throw new FaceComparisonError('目标图像未提取特征向量');
        }
        targetEmbedding = resultData[0].embedding.vector;
      } else {
        // 使用现有检测结果
        if (!target.embedding || !target.embedding.vector) {
          throw new FaceComparisonError('目标人脸未提取特征向量');
        }
        targetEmbedding = target.embedding.vector;
      }
      
      // 计算相似度（使用 FaceComparator）
      const comparisonResult = this.faceComparator.compare(sourceEmbedding, targetEmbedding);
      if (comparisonResult.isFailure()) {
        throw comparisonResult.getError() || new FaceComparisonError('人脸比对失败');
      }
      const { similarity, isMatch, threshold } = comparisonResult.getData()!;

      return Result.success({
        similarity,
        isMatch,
        threshold
      });
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : String(error);
      this.logger.error('FaceDetector', `人脸比对失败: ${errorMessage}`, error as Error);

      return Result.failure(new FaceComparisonError(`人脸比对失败: ${errorMessage}`));
    }
  }

  /**
   * 获取最近的检测结果
   */
  getLatestResults(): FaceDetectionResult[] {
    return [...this.lastDetectionResult];
  }
}