@quadible/web-sdk/dist/src/common/VideoWorker.js

The web sdk for Quadible's behavioral authentication service.

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const eventemitter2_1 = require("eventemitter2");
const sleep_1 = __importDefault(require("./sleep"));
class VideoWorker extends eventemitter2_1.EventEmitter2 {
    verbose;
    serviceUrl;
    video = document.createElement('video');
    canvas = document.createElement('canvas');
    get workerCode() {
        return `
            let canvas;
            let context;
            let model;

            let offscreenReadyResolver;
            let offscreenReady = new Promise(resolve => offscreenReadyResolver = resolve);

            main();

            async function main() {
                log('Worker started.');
                addEventListener('message', offscreenListener);

                await Promise.all([
                    offscreenReady,
                    loadBlazeface()
                ]);

                postMessage('ready');
            }

            function offscreenListener(event) {
                canvas = event.data.offscreen;
                context = canvas.getContext('2d');
                removeEventListener('message', offscreenListener);
                addEventListener('message', imageListener);
                offscreenReadyResolver();
                log('Worker got context.');
            }

            async function imageListener(event) {
                context.drawImage(event.data.imageBitmap, 0, 0, 320, 240);

                const [blob, predictions] = await Promise.all([
                    canvas.convertToBlob(),
                    model.estimateFaces(canvas, false)
                ]);

                postMessage({
                    predictions,
                    imageBase64: new FileReaderSync().readAsDataURL(blob)
                });
            }

            function log(...args) {
                if(${this.verbose}) {
                    console.debug('[quadible][worker] ' + args.shift(), ...args);
                }
            }

            async function loadBlazeface() {
                log('Loading tfjs and blazeface...');
                
                const urls = [
                    'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core@3.6.0/dist/tf-core.min.js',
                    'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl@3.6.0/dist/tf-backend-webgl.min.js',
                    'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-converter@3.6.0/dist/tf-converter.min.js',
                    'https://cdn.jsdelivr.net/npm/@tensorflow-models/blazeface@0.0.7/dist/blazeface.min.js'
                ];

                importScripts(...urls.map(url => '${this.serviceUrl}/v1/websdk/dependency?url=' + encodeURIComponent(url)));

                log('Loading blazeface model...');
                model = await blazeface.load({ maxFaces: 1, scoreThreshold: 0.96 });
                log('Done.');
            }
        `;
    }
    isInitialized = false;
    constructor(verbose = false, serviceUrl) {
        super();
        this.verbose = verbose;
        this.serviceUrl = serviceUrl;
    }
    async init() {
        if (this.isInitialized) {
            return;
        }
        this.isInitialized = true;
        try {
            await this.bootstrapVideo();
            this.assignCanPlayHandler();
        }
        catch (e) {
            this.emit('error', e);
        }
    }
    assignCanPlayHandler() {
        const handler = () => {
            this.video.removeEventListener('canplay', handler);
            this.bootstrapWorker();
        };
        this.video.addEventListener('canplay', handler);
    }
    async bootstrapVideo() {
        const stream = await navigator?.mediaDevices?.getUserMedia({
            video: true
        });
        this.video.width = 320;
        this.video.height = 240;
        this.video.autoplay = false;
        this.video.loop = true;
        this.video.srcObject = stream;
        // this.video.src = 'vid.mp4';
    }
    async bootstrapWorker() {
        this.emit('video-can-play');
        this.canvas.width = this.video.width;
        this.canvas.height = this.video.height;
        const blob = new Blob([this.workerCode], {});
        const url = URL.createObjectURL(blob);
        const worker = new Worker(url);
        const offscreen = this.canvas.transferControlToOffscreen();
        worker.postMessage({ offscreen }, [offscreen]);
        worker.addEventListener('message', (ev) => {
            if (ev.data === 'ready') {
                this.video.play().catch(() => {
                    const events = ['click', 'keydown', 'touchdown'];
                    const play = () => {
                        unregisterListeners();
                        this.video.play();
                    };
                    const unregisterListeners = () => {
                        for (const event of events) {
                            removeEventListener(event, play);
                        }
                    };
                    for (const event of events) {
                        addEventListener(event, play);
                    }
                });
            }
            else {
                this.emit('predictions', ev.data);
            }
        });
        this.video.addEventListener('play', () => {
            this.emit('video-play-start');
            const video = this.video;
            const videoStream = video.captureStream();
            const [track] = videoStream.getVideoTracks();
            const imageCapture = new window.ImageCapture(track);
            let lastFrameProcessedAt;
            sendFrameToWorker();
            async function sendFrameToWorker() {
                lastFrameProcessedAt = performance.now();
                const imageBitmap = await grabFrame();
                worker.addEventListener('message', responseListener);
                worker.postMessage({ imageBitmap }, [imageBitmap]);
                async function responseListener() {
                    worker.removeEventListener('message', responseListener);
                    const now = performance.now();
                    const THRESHOLD_MS = 100;
                    const deltaTime = now - lastFrameProcessedAt;
                    if (deltaTime < THRESHOLD_MS) {
                        await (0, sleep_1.default)(THRESHOLD_MS - deltaTime);
                    }
                    sendFrameToWorker();
                }
            }
            async function grabFrame() {
                while (true) {
                    try {
                        return await imageCapture.grabFrame();
                    }
                    catch (e) {
                        console.warn({
                            message: `[quadible][VideoWorker] ${e.message}`,
                            stack: e.stack
                        });
                        await (0, sleep_1.default)(100);
                    }
                }
            }
        });
    }
}
exports.default = VideoWorker;
//# sourceMappingURL=VideoWorker.js.map