node-red-contrib-human-recognition
Version:
Provides a node-red node for Human Detection & Human Recognition.
494 lines (446 loc) • 27.5 kB
JavaScript
//2021 David L Burrows
//Contact me @ https://github.com/meeki007
//or meeki007@gmail.com
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//http://www.apache.org/licenses/LICENSE-2.0
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//Node.js LTS ✔ Node v12.19.0 Npm 6.14.8
//Node-RED core ✔ 1.2.2
module.exports = function(RED)
{
//////////////////////////////////////
// Import Required(s) by this node //
////////////////////////////////////
// provides the path segment separator (\ on Windows, and / on Linux / macOS
const path = require('path'); //Node.js path module see https://nodejs.dev/learn/the-nodejs-path-module
// for Node, using accessing file system
const fs = require('fs'); //Node.js fs module see https://nodejs.dev/learn/the-nodejs-fs-module
// for Node, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
var tf; //value becomes user selection of: require('@tensorflow/tfjs-node'); or require('@tensorflow/tfjs-node-gpu');
// load specific version of Human library that matches TensorFlow mode
var humanLibrary; //value becomes user selection besd on require('@tensorflow/tfjs-node'); or require('@tensorflow/tfjs-node-gpu'); https://www.npmjs.com/package/@vladmandic/human
//////////////////////////////////////////////////////////////////////////////
//Global stuff used by entire node, stuff todo before construcing the node //
////////////////////////////////////////////////////////////////////////////
// check if a module is installed and working, if not return error code;
function module_check (module_name)
{
try
{
require(module_name);
return true;
}
catch (error)
{
return error.code;
//return error;
}
}
// check if module process is installed and working;
let humanLibraryGPU_module_check = module_check('@vladmandic/human/dist/human.node-gpu.js');
// check if library @vladmandic/human/dist/human.node.js is installed and working;
let humanLibraryCPU_module_check = module_check('@vladmandic/human/dist/human.node.js');
// check if module @tensorflow/tfjs-node [GPU] is installed and working;
let tfjs_node_gpu_module_check = module_check('@tensorflow/tfjs-node-gpu');
// check if module @tensorflow/tfjs-node [CPU] is installed and working;
let tfjs_node_cpu_module_check = module_check('@tensorflow/tfjs-node');
/////////////////////////
// construct the node //
///////////////////////
function humanrecognitionNode(config)
{
RED.nodes.createNode(this,config);
//clear status icon if one is hanging about wehn you deploy the node
this.status({});
//Clear user notices Function, used for timmer to clear after set ammont of time
var status_clear = function()
{
//clear status icon
node.status({});
};
//turn stings that should be a boolean into a boolean, used to fix node's html behavior
function stringToBoolean (string)
{
switch(string.toLowerCase().trim())
{
case "true": case "yes": case "1": return true;
case "false": case "no": case "0": case null: return false;
default: return Boolean(string);
}
}
// config from this nodes html file, get user settings
this.image = config.image||'payload';
this.OverRideMyConfigMSG = config.OverRideMyConfigMSG||'config';
this.UserEmbeddings = config.UserEmbeddings||'embeddings';
this.bindings = config.bindings || 'CPU';
this.scoped = stringToBoolean( config.scoped || 'false' );
this.warmup = config.warmup || 'full';
this.FaceDetector = config.FaceDetector || 'faceboxes';
this.FaceDetectorEnabled = config.FaceDetector === 'disabled' ? false : true;
this.FaceDetector_Options_inputSize = Number(config.FaceDetector_Options_inputSize || 256);
this.FaceDetector_Options_rotation = stringToBoolean( config.FaceDetector_Options_rotation || 'false' );
this.FaceDetector_Options_maxFaces = Number(config.FaceDetector_Options_maxFaces || 3);
this.FaceDetector_Options_iouThreshold = Number(config.FaceDetector_Options_iouThreshold || 0.2);
this.FaceDetector_Options_scoreThreshold = Number(config.FaceDetector_Options_scoreThreshold || 0.5);
this.FaceDetector_Options_minConfidence = Number(config.FaceDetector_Options_minConfidence || 0.5);
this.FaceMesh = stringToBoolean( config.FaceMesh || 'false' );
this.FaceMesh_Options_returnRawData = stringToBoolean( config.FaceMesh_Options_returnRawData || 'false' );
this.Iris = stringToBoolean( config.Iris || 'false' );
this.Age = config.Age || 'disabled';
this.AgeEnabled = config.Age === 'disabled' ? false : true;
this.Gender = config.Gender || 'disabled';
this.GenderEnabled = config.Gender === 'disabled' ? false : true;
this.Gender_Options_minConfidence = Number(config.Gender_Options_minConfidence || 0.1);
this.Emotion = config.Emotion || 'disabled';
this.EmotionEnabled = config.Emotion === 'disabled' ? false : true;
this.Emotion_Options_minConfidence = Number(config.Emotion_Options_minConfidence || 0.2);
this.Embedding = stringToBoolean( config.Embedding || 'false' );
this.BodyDetector = stringToBoolean( config.BodyDetector || 'false' );
this.BodyDetector_Options_maxDetections = Number(config.BodyDetector_Options_maxDetections || 3);
this.BodyDetector_Options_scoreThreshold = Number(config.BodyDetector_Options_scoreThreshold || 0.5);
this.BodyDetector_Options_nmsRadius = Number(config.BodyDetector_Options_nmsRadius || 20);
this.BodyDetector_Options_outputStride = Number(config.BodyDetector_Options_outputStride || 16);
this.HandDetector = stringToBoolean( config.HandDetector || 'false' );
this.HandDetector_Options_rotation = stringToBoolean( config.HandDetector_Options_rotation || 'false' );
this.HandDetector_Options_landmarks = stringToBoolean( config.HandDetector_Options_landmarks || 'false' );
this.HandDetector_Options_maxHands = Number(config.HandDetector_Options_maxHands || 3);
this.HandDetector_Options_minConfidence = Number(config.HandDetector_Options_minConfidence || 0.1);
this.HandDetector_Options_iouThreshold = Number(config.HandDetector_Options_iouThreshold || 0.1);
this.HandDetector_Options_scoreThreshold = Number(config.HandDetector_Options_scoreThreshold || 0.5);
//require modules and librarys based on user input to node image bindings
//warn user if bindings selection is not valid for lack of module or library
var humanLibrary_pass = true;
var tf_pass = true;
if ( this.bindings === 'CPU' )
{
if ( tfjs_node_cpu_module_check === true )
{
tf = require('@tensorflow/tfjs-node');
if ( humanLibraryCPU_module_check === true )
{
humanLibrary = require('@vladmandic/human/dist/human.node.js').default;
}
else
{
humanLibrary_pass = "Error: " + humanLibraryCPU_module_check + " - Unable to load library human.node.js (CPU). Check @vladmandic/human module is installed in your Node-RED node_modules directory, typically ~/.node-red/node_modules/@vladmandic/human and that the library file is there, typically ~/.node-red/node_modules/@vladmandic/human/dist";
this.warn(humanLibraryCPU_pass);
this.status(
{
fill: 'red',
shape: 'dot',
text: "detected error"
});
}
}
else
{
tf_pass = "Error: " + tfjs_node_cpu_module_check + " - Unable to use @tensorflow/tfjs-node; CPU binding. Check your loggs when installing. See this node's documentation and Make sure that module @tensorflow/tfjs-node is properly working & installed under your Node-RED user directory, typically ~/.node-red/node_modules/@tensorflow";
this.warn(tfCPU_pass);
this.status(
{
fill: 'red',
shape: 'dot',
text: "detected error"
});
}
}
else if ( this.bindings === 'GPU' )
{
if ( tfjs_node_gpu_module_check === true )
{
tf = require('@tensorflow/tfjs-node-gpu');
if ( humanLibraryGPU_module_check === true )
{
humanLibrary = require('@vladmandic/human/dist/human.node-gpu.js').default;
}
else
{
humanLibrary_pass = "Error: " + humanLibraryGPU_module_check + " - Unable to load library human.node-gpu.js (GPU). Check @vladmandic/human module is installed in your Node-RED node_modules directory, typically ~/.node-red/node_modules/@vladmandic/human and that the library file is there, typically ~/.node-red/node_modules/@vladmandic/human/dist";
this.warn(humanLibraryGPU_pass);
this.status(
{
fill: 'red',
shape: 'dot',
text: "detected error"
});
}
}
else
{
tf_pass = "Error: " + tfjs_node_cpu_module_check + " - Unable to use @tensorflow/tfjs-node-gpu; GPU binding. Check your loggs when installing. See this node's documentation and Make sure that module @tensorflow/tfjs-node is properly working & installed under your Node-RED user directory, typically ~/.node-red/node_modules/@tensorflow";
this.warn(tfGPU_pass);
this.status(
{
fill: 'red',
shape: 'dot',
text: "detected error"
});
}
}
//get path to models from @vladmandic/human
const humanModulePath = path.dirname(require.resolve("@vladmandic/human/package.json"));
//file:// only accepts UNC path as input and will conert it to win if needed. So convert it to posix
const humanModuleUNCPath = humanModulePath.split(path.sep).join(path.posix.sep);
// file:// create the new path in UNC format if on windows or linux
const correctedFileUNCPath = 'file://' + humanModuleUNCPath + '/models/';
//default MODELS formated for fs's file:// (File URL paths)
const faceDetectorModelUNCPath = correctedFileUNCPath + this.FaceDetector + `.json`;
const meshModelUNCPath = correctedFileUNCPath + 'facemesh.json';
const irishModelUNCPath = correctedFileUNCPath + 'iris.json';
const agehModelUNCPath = correctedFileUNCPath + this.Age + `.json`;
const genderhModelUNCPath = correctedFileUNCPath + this.Gender + `.json`;
const emotionhModelUNCPath = correctedFileUNCPath + this.Emotion + `.json`;
const embeddinghModelUNCPath = correctedFileUNCPath + 'mobilefacenet.json';
const bodyhModelUNCPath = correctedFileUNCPath + 'posenet.json';
const handDetectorModelUNCPath = correctedFileUNCPath + 'handdetect.json';
const skeletonModelUNCPath = correctedFileUNCPath + 'handskeleton.json';
//Default Config object for humanLibrary(myConfig);
var myConfig = {
backend: 'tensorflow', // select tfjs backend to use
wasmPath: '../assets/', // path for wasm binaries
async: true, // execute enabled models in parallel. this disables per-model performance data but slightly increases performance. cannot be used if profiling is enabled
profile: false, // enable tfjs profiling. this has significant performance impact. only enable for debugging purposes. currently only implemented for age,gender,emotion models
deallocate: false, // aggresively deallocate gpu memory after each usage. only valid for webgl backend and only during first call. cannot be changed unless library is reloaded. this has significant performance impact. only enable on low-memory devices
scoped: this.scoped, // enable scoped runs some models *may* have memory leaks, this wrapps everything in a local scope at a cost of performance, typically not needed
videoOptimized: false, // perform additional optimizations when input is video, must be disabled for images, basically this skips object box boundary detection for every n frames, while maintaining in-box detection since objects cannot move that fast
warmup: this.warmup, // what to use for human.warmup(), can be 'none', 'face', 'full'. warmup pre-initializes all models for faster inference but can take significant time on deploment of node
filter: {
enabled: false, // True or False, enable image pre-processing filters
width: 0, // resize input width. If both width and height are set to 0, there is no resizing. If just one is set, second one is scaled automatically. If both are set, values are used as-is
height: 0, // resize input height. If both width and height are set to 0, there is no resizing. If just one is set, second one is scaled automatically. If both are set, values are used as-is
return: false, // True or False, return processed canvas imagedata in result
brightness: 0, // range: -1 (darken) to 1 (lighten) 0 to do nothing. Example: to lighten a little set valuse to 0.1
contrast: 0, // range: -1 (reduce contrast) to 1 (increase contrast) 0 to do nothing. Example: to increse contrast a little set valuse to 0.1
sharpness: 0, // range: 0 (no sharpening) to 1 (maximum sharpening) Example: to increse sharpening a little set valuse to 0.1
blur: 0, // range: 0 (no blur) to N (blur radius in pixels) Example: to increse blur a little set valuse to 0.1
saturation: 0, // range: -1 (reduce saturation) to 1 (increase saturation) Example: to increase saturation a little set valuse to 0.1
hue: 0, // range: 0 (no change) to 360 (hue rotation in degrees)
negative: false, // True or False, image negative
sepia: false, // True or False, image sepia colors
vintage: false, // True or False, image vintage colors
kodachrome: false, // True or False, image kodachrome colors
technicolor: false, // True or False, image technicolor colors
polaroid: false, // True or False, image polaroid camera effect
pixelate: 0, // range: 0 (no pixelate) to N (number of pixels to pixelate)
},
gesture: {
enabled: true, // True or False, enable simple gesture recognition
},
face: {
enabled: this.FaceDetectorEnabled, // True or False, controls if Face modul is enabled. face.enabled is required for all face models: detector, mesh, iris, age, gender, emotion. (note: module is not loaded until it is required)
detector: {
modelPath: faceDetectorModelUNCPath, // Path to the model file. Can be 'blazeface-front.json', 'blazeface-back.json' or 'faceboxes.json'. 'blazeface-front' is blazeface model optimized for large faces such as front-facing camera. 'blazeface-back' is blazeface model optimized for smaller and/or distanct faces. 'faceboxes' is alternative model to 'blazeface and is the only one that works with nodeJS at the moment.
inputSize: this.FaceDetector_Options_inputSize, //fixed value: 128 for front(upclose image) and 256 for 'back'(far away image)
rotation: this.FaceDetector_Options_rotation, // True or False, use best-guess rotated face image or just box with rotation as-is. false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
maxFaces: this.FaceDetector_Options_maxFaces, // maximum number of faces detected in the input. should be set to the minimum number for performance
skipFrames: 11, // how many frames to go without re-running the face bounding box detector. only used for video inputs. e.g., if model is running st 25 FPS, we can re-use existing bounding box for updated face analysis as the head probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: this.FaceDetector_Options_minConfidence, // threshold for discarding a prediction. 0 discard none, 1 discard all.
iouThreshold: this.FaceDetector_Options_iouThreshold, // threshold for deciding whether boxes overlap too much in non-maximum suppression (0.1 means drop if overlap 10%)
scoreThreshold: this.FaceDetector_Options_scoreThreshold, // threshold for deciding when to remove boxes based on score in non-maximum suppression, this is applied on detection objects only and before minConfidence
},
mesh: {
enabled: this.FaceMesh, // True or False, controls if Mesh modul is enabled.
modelPath: meshModelUNCPath, // Path to the model file. facemesh.json
inputSize: 192, // fixed value
returnRawData: this.FaceMesh_Options_returnRawData, // in addition to standard mesh and box values, return raw normalized values as well
},
iris: {
enabled: this.Iris, // True or False, controls if Mesh modul is enabled.
modelPath: irishModelUNCPath, // Path to the model file. iris.json
inputSize: 64, // fixed value
},
age: {
enabled: this.AgeEnabled, // True or False, controls if age modul is enabled.
modelPath: agehModelUNCPath, // Path to the model file. can be age-ssrnet-imdb.json or age-ssrnet-wiki..json
inputSize: 64, // fixed value
skipFrames: 31, // how many frames to go without re-running the detector
},
gender: {
enabled: this.GenderEnabled, // True or False, controls if gender modul is enabled.
minConfidence: this.Gender_Options_minConfidence, // threshold for discarding a prediction. 0 discard none, 1 discard all.
modelPath: genderhModelUNCPath, // Path to the model file. can be 'gender.json', 'gender-ssrnet-imdb.json' or 'gender-ssrnet-wiki.json'
inputSize: 64, // fixed value
skipFrames: 41, // how many frames to go without re-running the detector
},
emotion: {
enabled: this.EmotionEnabled, // True or False, controls if emotion modul is enabled.
inputSize: 64, // fixed value
minConfidence: this.Emotion_Options_minConfidence, // threshold for discarding a prediction. 0 discard none, 1 discard all.
skipFrames: 21, // how many frames to go without re-running the detector
modelPath: emotionhModelUNCPath, // Path to the model file. can be emotion-large.json or emotion-mini.json
},
embedding: {
enabled: this.Embedding, // True or False, controls if embedding modul is enabled.
inputSize: 112, // fixed value
modelPath: embeddinghModelUNCPath, // Path to the model file. mobilefacenet.json
},
},
body: {
enabled: this.BodyDetector, // True or False, controls if body modul is enabled.
modelPath: bodyhModelUNCPath, // Path to the model file. posenet.json
inputSize: 257, // fixed value
maxDetections: this.BodyDetector_Options_maxDetections, // maximum number of people detected in the input
scoreThreshold:this.BodyDetector_Options_scoreThreshold, // threshold for deciding when to remove boxes based on score in non-maximum suppression
nmsRadius: this.BodyDetector_Options_nmsRadius, // radius for deciding points are too close in non-maximum suppression
outputStride: this.BodyDetector_Options_outputStride, // size of block in which to run point detectopn, smaller value means higher resolution defined by model itself, can be 8, 16, or 32
modelType: 'MobileNet', // Human includes MobileNet version, but you can switch to ResNet
},
hand: {
enabled: this.HandDetector, // True or False, controls if hand modul is enabled.
rotation: this.HandDetector_Options_rotation, // True or False, use best-guess rotated hand image or just box with rotation as-is. false means higher performance, but incorrect mapping
inputSize: 256, // fixed value
skipFrames: 12, // how many frames to go without re-running the detector
minConfidence: this.HandDetector_Options_minConfidence, // threshold for discarding a prediction. 0 discard none, 1 discard all.
iouThreshold: this.HandDetector_Options_iouThreshold, // threshold for deciding whether boxes overlap too much in non-maximum suppression. (0.1 means drop if overlap 10%)
scoreThreshold: this.HandDetector_Options_scoreThreshold, // threshold for deciding when to remove boxes based on score in non-maximum suppression
maxHands: this.HandDetector_Options_maxHands, // maximum number of hands detected in the input. should be set to the minimum number for performance
landmarks: this.HandDetector_Options_landmarks, // True or False, detect hand landmarks(true) or just hand boundary box(false)
detector: {
modelPath: handDetectorModelUNCPath, // Path to the model file. handdetect.json
},
skeleton: {
modelPath: skeletonModelUNCPath, // Path to the model file. handskeleton.json
},
},
};
//is tfjs_ready
var tfjs_ready; //error check of tfjs_ready
Promise.resolve
(
tf.ready()
)
.then( tfjs_ready = true )
.catch(error =>
{
tfjs_ready = ("tfjs is not ready " + error),
this.warn(tfjs_ready),
this.status(
{
fill: 'red',
shape: 'dot',
text: "detected error"
});
});
//clear human before accepting new config when deploying node
let human = null;
//load human
human = new humanLibrary(myConfig);
//for testing
this.warn('Human:' + human.version);
this.warn('Active Configuration:' + JSON.stringify(human.config));
this.warn('TFJS Version:' + JSON.stringify(tf.version_core));
this.warn('TFJS Backend:' + JSON.stringify(tf.getBackend()));
this.warn('TFJS Flags:' + JSON.stringify(tf.env().features));
this.warn('Loading models:');
//is human_ready - pre-load models
var human_ready; //error check of human_ready
Promise.resolve
(
human.load()
)
.then( human_ready = true )
.catch(error =>
{
human_ready = ("human is not ready " + error),
this.warn(human_ready),
this.status(
{
fill: 'red',
shape: 'dot',
text: "detected error"
});
});
//for testing
for (const model of Object.keys(human.models)) this.warn(' Loaded:' + model);
this.warn('Memory state:' + JSON.stringify(human.tf.engine().memory()));
this.warn('Test Complete');
//////////////////////////////////////////////
// Do Stuff when a msg is sent to this node //
//////////////////////////////////////////////
var node = this;
this.on('input', async function(msg, send, done)
{
// For maximum backwards compatibility, check that send exists.
// If this node is installed in Node-RED 0.x, it will need to
// fallback to using `node.send`
send = send || function() { node.send.apply(node,arguments); };
//user error function
function notify_user_errors(err)
{
if (done)
{
// Node-RED 1.0 compatible
done(err);
}
else
{
// Node-RED 0.x compatible
node.error(err, msg);
}
}
//function to work with buffered image in from msg payload. Turns image into a tensor
async function image(buffer)
{
try {
const decoded = human.tf.node.decodeImage(buffer);
const casted = decoded.toFloat();
const result = casted.expandDims(0);
decoded.dispose();
casted.dispose();
return result;
} catch (error) {
notify_user_errors(error);
}
}
async function listDirectories(rootPath)
{
const fileNames = await fs.promises.readdir(rootPath);
if ( fileNames.indexOf(".DS_Store") == 0 ) //for MacOS - if fileNames contains .DS_Store
{
fileNames.shift(); // get rid of it
}
const filePaths = fileNames.map(fileName => path.join(rootPath, fileName));
const filePathsAndIsDirectoryFlagsPromises = filePaths.map(async filePath => ({path: filePath, isDirectory: (await fs.promises.stat(filePath)).isDirectory()}));
const filePathsAndIsDirectoryFlags = await Promise.all(filePathsAndIsDirectoryFlagsPromises);
return filePathsAndIsDirectoryFlags.filter(filePathAndIsDirectoryFlag => filePathAndIsDirectoryFlag.isDirectory)
.map(filePathAndIsDirectoryFlag => filePathAndIsDirectoryFlag.path);
}
async function listFiles(rootPath)
{
const fileNames = await fs.promises.readdir(rootPath);
if ( fileNames.indexOf(".DS_Store") == 0 ) //for MacOS - if fileNames contains .DS_Store
{
fileNames.shift(); // get rid of it
}
const filePaths = fileNames.map(fileName => path.join(rootPath, fileName));
const filePathsAndIsFileFlagsPromises = filePaths.map(async filePath => ({path: filePath, isFile: (await fs.promises.stat(filePath)).isFile()}));
const filePathsAndIsFileFlags = await Promise.all(filePathsAndIsFileFlagsPromises);
return filePathsAndIsFileFlags.filter(filePathAndIsFileFlag => filePathAndIsFileFlag.isFile)
.map(filePathAndIsFileFlag => filePathAndIsFileFlag.path);
}
//set user img to node to var
var img = msg[node.image.valueOf()];
//get the msg.name used for for img into node
var img_name = node.image.valueOf();
// load image from payload
const imageTensor = await image(img);
const result = await human.detect(imageTensor, myConfig);
// dispose image tensor as we no longer need it
imageTensor.dispose();
msg.result = result;
msg.myConfig = myConfig;
//msg.loadedHumanModels = Object.keys(human.models).filter((a) => human.models[a]);
send(msg);
if (done) {
done();
}
});
}
RED.nodes.registerType("human-recognition",humanrecognitionNode);
};