scrawl-canvas/demo/tensorflow-002.js

Version 8.9.4 - 19 Nov 2022

// # Demo Tensorflow 002
// Tensorflow tfjs-models / body-pix experiment - model image output

// [Run code](../../demo/tensorflow-002.html)
import * as scrawl from '../source/scrawl.js';

import { reportSpeed } from './utilities.js';


// #### Scene setup
const canvas = scrawl.library.artefact.mycanvas;


scrawl.makeFilter({
    name: 'grayscale',
    method: 'grayscale',
}).clone({
    name: 'sepia',
    method: 'sepia',
}).clone({
    name: 'invert',
    method: 'invert',
}).clone({
    name: 'red',
    method: 'red',
});

scrawl.makeFilter({
    name: 'pixelate',
    method: 'pixelate',
    tileWidth: 20,
    tileHeight: 20,
    offsetX: 8,
    offsetY: 8,
});

scrawl.makeFilter({
    name: 'background-blur',
    method: 'gaussianBlur',
    radius: 20,
});

scrawl.makeFilter({
    name: 'body-blur',
    method: 'gaussianBlur',
    radius: 10,
});

// #### TensorFlow functionality
// We'll handle everything in a raw asset object, which a Picture entity can then use as its source
let myAsset = scrawl.makeRawAsset({

    name: 'tensorflow-model-interpreter',

    userAttributes: [{

        // We're only interested in the pixel allocations generated by the tensorflow model in this demo
        key: 'data', 
        defaultValue: [],
        setter: function (item) {

            if (item && item.width && item.height && item.data) {

                this.canvasWidth = item.width;
                this.canvasHeight = item.height;
                this.data = item.data;
                this.dirtyData = true;
            }
        },
    },{
        key: 'canvasWidth', 
        defaultValue: 0,
        setter: () => {},
    },{
        key: 'canvasHeight', 
        defaultValue: 0,
        setter: () => {},
    }],

    updateSource: function (assetWrapper) {

        const { element, engine, canvasWidth, canvasHeight, data } = assetWrapper;

        if (canvasWidth && canvasHeight && data) {

            const segLength = canvasWidth * canvasHeight,
                imageDataLen = segLength * 4,
                imageArray = new Uint8ClampedArray(imageDataLen);

            for (let i = 0, o = 0; i < segLength; i++) {

                o = (i * 4) + 3;
                if (data[i]) imageArray[o] = 255;
            }

            const iData = new ImageData(imageArray, canvasWidth, canvasHeight);

            // Clear the canvas, resizing it if required
            element.width = canvasWidth;
            element.height = canvasHeight;

            engine.putImageData(iData, 0, 0);
        }
    },
});

// The forever loop function, which captures the tensorflow model's output and passes it on to our raw asset for processing
const perform = function (net) {

    net.segmentPerson(video.source)
    .then(data => {

        myAsset.set({data});
        perform(net);
    })
    .catch(e => console.log(e));
};


// ##### Import and use livestream
// convenience handle for the media stream asset 
let video, myBackground, myOutline;

// Capture the media stream
scrawl.importMediaStream({
    name: 'device-camera',
    audio: false,
})
.then(mycamera => {

    video = mycamera;

    // This fixes the issue in Firefox where the media stream will crash Tensorflow if the stream's video element's dimensions have not been set
// @ts-expect-error
    video.source.width = "1280";
// @ts-expect-error
    video.source.height = "720";

    // Take the media stream and display it in our canvas element
    myBackground = scrawl.makePicture({

        name: 'background',
        asset: mycamera.name,
        order: 2,

        width: '100%',
        height: '100%',

        copyWidth: '80%',
        copyHeight: '80%',
        copyStartX: '10%',
        copyStartY: '10%',

        globalCompositeOperation: 'destination-over',
    });

    myBackground.clone({

        name: 'body',
        order: 1,
        globalCompositeOperation: 'source-in',
    });

    // Start the TensorFlow model
// @ts-expect-error
    bodyPix.load()
    .then (net => {

        // Display the visual generated by our raw asset
        myOutline = scrawl.makePicture({

            name: 'outline',
            asset: 'tensorflow-model-interpreter',
            order: 0,

            width: '100%',
            height: '100%',

            copyWidth: '80%',
            copyHeight: '80%',
            copyStartX: '10%',
            copyStartY: '10%',

            // We blur here to make the outline merge into the background
            // + this does slow the demo down, but needs must.
            filters: ['body-blur'],
        });

        // Invoke the forever loop
        perform(net);
    })
    .catch(e => console.log('ERROR: ', e));
})
.catch(err => console.log(err.message));


// #### Scene animation
// Function to display frames-per-second data, and other information relevant to the demo
const report = reportSpeed('#reportmessage');


// Create the Display cycle animation
scrawl.makeRender({

    name: 'demo-animation',
    target: canvas,
    afterShow: report,
});


// #### User interaction
// Event listeners
scrawl.addNativeListener(['input', 'change'], (e) => {

    e.preventDefault();
    e.returnValue = false;

    if (e && e.target) {

        const id = e.target.id,
            val = e.target.value;

        if ('backgroundFilter' === id) {

            myBackground.clearFilters();
            if (val) myBackground.addFilters(val);
        }
        else {

            if ('1' === val) myOutline.addFilters('body-blur');
            else myOutline.clearFilters();
        }
    }
}, '.controlItem');

// Set DOM form initial input values
// @ts-expect-error
document.querySelector('#backgroundFilter').value = '';
// @ts-expect-error
document.querySelector('#outlineFilter').value = '1';


// #### Development and testing
console.log(scrawl.library);