synopsis-video/extractFramesFromVideosAndRemoveBackgrounds.js

Create Video Synopsis from provided videos. Frames are extracted from each video provided and backgrounds are subtracted around moving elements and merged into a single video for quick review.

const {
    rawDirectory,
    createBaseFolder,
    makeVideoImagesFolder,
    grabFrames,
    grabFirstFrame,
} = require('./utils.js');
const fs = require('fs');
const moment = require('moment');
const cv = require('opencv4nodejs');
const spawn = require('child_process').spawn;
const gm = require('gm').subClass({imageMagick: true})

const bgSubtractor = new cv.BackgroundSubtractorMOG2();
const delay = 50;

const buildMask = (frame) => {
    const foreGroundMask = bgSubtractor.apply(frame);
    // const iterations = 2;
    // const dilated = foreGroundMask.dilate(
    //   cv.getStructuringElement(cv.MORPH_ELLIPSE, new cv.Size(4, 4)),
    //   new cv.Point(-1, -1),
    //   iterations
    // );
    const blurred = foreGroundMask.blur(new cv.Size(10, 10));
    const finalMask = blurred.threshold(200, 255, cv.THRESH_BINARY);
    return finalMask
}

const applyMaskToFrame = (frame,mask) => {
    const channels = frame.splitChannels();
    const maskedChannels = channels.map(c => c.bitwiseAnd(mask));
    return new cv.Mat(maskedChannels)
}

const getReasonableContours = (mask) => {
    return mask.copy().findContours(cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE).filter((c) => c.area > 4000)
}

const burnLabelToFrame = (frame,options) => {
    options.color = options.color ? options.color : [255,255,254]
    const text = options.text || moment().format('YYYY-MM-DD HH:mm:ss');
    const org = new cv.Point(options.x, options.y + 15);
    const fontFace = options.font || cv.FONT_HERSHEY_SIMPLEX;
    const fontScale = options.scale || 0.5;
    const textColor = new cv.Vec3(...options.color);
    const thickness = options.weight || 2;
    const backDropSize = cv.getTextSize(text, fontFace, fontScale, thickness).size
    // create backdrop
    frame.drawRectangle(
        new cv.Point(options.x - 1, options.y + 6),
        new cv.Point(options.x + backDropSize.width, options.y + backDropSize.height + 1),
        new cv.Vec3(3,3,3),
        20,
        cv.LINE_8
    );
    // write text
    frame.putText(text, org, fontFace, fontScale, textColor, thickness);
}

const addLabelsToFrame = (contours,frame) => {
    contours.forEach((c) => {
        const box = c.boundingRect()
        burnLabelToFrame(frame,{
            x: box.x,
            y: box.y,
        })
    })
}

const saveFrameAndMakeTransparent = (videoName,number,newFrame) => {
    cv.imwrite(`${rawDirectory}${videoName}/img${number}.png`, newFrame);
    gm(`${rawDirectory}${videoName}/img${number}.png`)
        .transparent('#000')
        .transparent('#FFF')
        .write(`${rawDirectory}${videoName}/img${number}.png`, function (err) {
            if (err) console.log(err);
        })
}

const doOneVideo = (videoName) => {
    var number = 0;
    return new Promise((resolve,reject) => {
        makeVideoImagesFolder(videoName)
        const videoFilename = videoName + '.mp4';
        grabFirstFrame(videoFilename);
        grabFrames(videoFilename, delay, (frame) => {
          const mask = buildMask(frame);
          const newFrame = applyMaskToFrame(frame,mask);
          // do object detection instead of contour?
          var contours = getReasonableContours(mask);
          if(contours.length > 0){
              addLabelsToFrame(contours,newFrame)
              saveFrameAndMakeTransparent(videoName,number,newFrame)
              ++number
          }
        },() => {
            //when no more frames are found
            setTimeout(resolve,2000)
        });
    });
}


exports.extractFramesFromVideo = doOneVideo