@polygonjs/polygonjs
Version:
node-based WebGL 3D engine https://polygonjs.com
177 lines (176 loc) • 7.08 kB
JavaScript
;
import { Vector4 } from "three";
import { FaceMesh } from "@mediapipe/face_mesh";
import { CoreComputerVisionFaceAttribute } from "./Common";
import { ParamConfig } from "../../../engine/nodes/utils/params/ParamsConfig";
import { isBoolean, isNumber, isString } from "../../Type";
import { DEFAULT_POSITION } from "./Data";
import { coreObjectClassFactory } from "../../geometry/CoreObjectFactory";
const DEFAULT = {
selfieMode: false,
maxNumFaces: 1,
refineLandmarks: false,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5
};
export function CoreComputerVisionFaceParamConfig(Base) {
return class Mixin extends Base {
constructor() {
super(...arguments);
/** @param selfieMode */
this.selfieMode = ParamConfig.BOOLEAN(DEFAULT.selfieMode);
/** @param Maximum number of faces to detect */
this.maxNumFaces = ParamConfig.INTEGER(DEFAULT.maxNumFaces, {
range: [0, 2],
rangeLocked: [true, false]
});
/** @param Whether to further refine the landmark coordinates around the eyes and lips, and output additional landmarks around the irises by applying the Attention Mesh Model */
this.refineLandmarks = ParamConfig.BOOLEAN(DEFAULT.refineLandmarks);
/** @param Minimum confidence value ([0.0, 1.0]) from the face detection model for the detection to be considered successful. Default to 0.5. */
this.minDetectionConfidence = ParamConfig.FLOAT(DEFAULT.minDetectionConfidence, {
range: [0, 1],
rangeLocked: [true, true]
});
/** @param Minimum confidence value ([0.0, 1.0]) from the landmark-tracking model for the face landmarks to be considered tracked successfully, or otherwise face detection will be invoked automatically on the next input image. Setting it to a higher value can increase robustness of the solution, at the expense of a higher latency. Ignored if static_image_mode is true, where face detection simply runs on every image. Default to 0.5. */
this.minTrackingConfidence = ParamConfig.FLOAT(DEFAULT.minTrackingConfidence, {
range: [0, 1],
rangeLocked: [true, true]
});
}
};
}
function locateFile(file) {
return `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`;
}
function attributes(object) {
const coreObjectClass = coreObjectClassFactory(object);
const selfieMode = coreObjectClass.attribValue(object, CoreComputerVisionFaceAttribute.SELFIE_MODE);
const maxNumFaces = coreObjectClass.attribValue(object, CoreComputerVisionFaceAttribute.MAX_NUM_FACES);
const refineLandmarks = coreObjectClass.attribValue(object, CoreComputerVisionFaceAttribute.REFINE_LANDMARKS);
const minDetectionConfidence = coreObjectClass.attribValue(
object,
CoreComputerVisionFaceAttribute.MIN_DETECTION_CONFIDENCE
);
const minTrackingConfidence = coreObjectClass.attribValue(
object,
CoreComputerVisionFaceAttribute.MAX_TRACKING_CONFIDENCE
);
const data = {
selfieMode: isBoolean(selfieMode) ? selfieMode : DEFAULT.selfieMode,
maxNumFaces: isNumber(maxNumFaces) ? maxNumFaces : DEFAULT.maxNumFaces,
refineLandmarks: isBoolean(refineLandmarks) ? refineLandmarks : DEFAULT.refineLandmarks,
minDetectionConfidence: isNumber(minDetectionConfidence) ? minDetectionConfidence : DEFAULT.minDetectionConfidence,
minTrackingConfidence: isNumber(minTrackingConfidence) ? minTrackingConfidence : DEFAULT.minTrackingConfidence
};
return data;
}
function trackerOptions(attributes2) {
return {
selfieMode: attributes2.selfieMode,
maxNumFaces: attributes2.maxNumFaces,
refineLandmarks: attributes2.refineLandmarks,
minDetectionConfidence: attributes2.minDetectionConfidence,
minTrackingConfidence: attributes2.minTrackingConfidence
};
}
function createKey(object) {
return JSON.stringify(attributes(object));
}
function createConvertedResult() {
const pointsCount = DEFAULT_POSITION.length / 3;
const multiFaceLandmarks = new Array(pointsCount);
for (let i = 0; i < pointsCount; i++) {
multiFaceLandmarks[i] = new Vector4();
}
return {
multiFaceLandmarks
};
}
function updateConvertedResult(convertedResult, results, index) {
results.multiFaceLandmarks;
const multiFaceLandmarks = results.multiFaceLandmarks[index];
for (let i = 0; i < multiFaceLandmarks.length; i++) {
const landmark = multiFaceLandmarks[i];
convertedResult.multiFaceLandmarks[i].set(
1 - landmark.x,
1 - landmark.y,
landmark.z,
landmark.visibility != null ? landmark.visibility : 0
);
}
}
class TrackerContainer {
constructor(tracker) {
this.tracker = tracker;
this._inProgress = false;
this.results = [];
for (let i = 0; i < 3; i++) {
this.results.push(createConvertedResult());
}
tracker.onResults((results) => {
this._inProgress = false;
const subResultsCount = results.multiFaceLandmarks.length;
for (let i = 0; i < subResultsCount; i++) {
updateConvertedResult(this.results[i], results, i);
}
});
}
track(source) {
if (this._inProgress) {
return;
}
this._inProgress = true;
this.tracker.send({ image: source });
}
}
export class CoreComputerVisionFace {
static trackerForObject(object) {
const key = this.trackerKey(object);
let tracker = this.trackerByKey.get(key);
if (!tracker) {
tracker = this._createTracker(trackerOptions(attributes(object)));
this.trackerByKey.set(key, tracker);
}
return tracker;
}
static trackMedia(object, source) {
const tracker = this.trackerForObject(object);
tracker.track(source);
}
static trackerResults(object) {
return this.trackerForObject(object).results;
}
static _createTracker(options) {
const faceMesh = new FaceMesh({
locateFile
});
faceMesh.setOptions(options);
return new TrackerContainer(faceMesh);
}
static trackerKey(object) {
const coreObjectClass = coreObjectClassFactory(object);
let key = coreObjectClass.attribValue(object, CoreComputerVisionFaceAttribute.KEY);
if (!key || !isString(key)) {
key = createKey(object);
coreObjectClass.addAttribute(object, CoreComputerVisionFaceAttribute.KEY, key);
}
return key;
}
static setAttributes(object, options) {
const coreObjectClass = coreObjectClassFactory(object);
coreObjectClass.addAttribute(object, CoreComputerVisionFaceAttribute.SELFIE_MODE, options.selfieMode);
coreObjectClass.addAttribute(object, CoreComputerVisionFaceAttribute.MAX_NUM_FACES, options.maxNumFaces);
coreObjectClass.addAttribute(object, CoreComputerVisionFaceAttribute.REFINE_LANDMARKS, options.refineLandmarks);
coreObjectClass.addAttribute(
object,
CoreComputerVisionFaceAttribute.MIN_DETECTION_CONFIDENCE,
options.minDetectionConfidence
);
coreObjectClass.addAttribute(
object,
CoreComputerVisionFaceAttribute.MAX_TRACKING_CONFIDENCE,
options.minTrackingConfidence
);
}
}
CoreComputerVisionFace.trackerByKey = /* @__PURE__ */ new Map();