imgrecog
Version:
Node.js tool to parse and act on images, using the Google Vision and Sightengine APIs.
265 lines (264 loc) • 11.3 kB
JavaScript
;
// IMGRECOG INDEX
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.IMGRecog = void 0;
const utils_1 = require("./utils");
const actions_1 = require("./actions");
const clarifai_1 = __importDefault(require("./clarifai"));
const sightengine_1 = __importDefault(require("./sightengine"));
const vision_1 = __importDefault(require("./vision"));
const asyncLib = require("async");
const logger = require("anyhow");
const fs = require("fs");
const path = require("path");
/**
* IMGRecog.js main module.
*/
class IMGRecog {
constructor(options) {
/**
* Run the thing!
*/
this.run = async () => {
if (!this.options.folders || this.options.folders.length < 1) {
throw new Error("No folders were passed");
}
// Reset state.
this.results = [];
this.startTime = new Date();
const logOptions = Object.entries(this.options).map((opt) => `${opt[0]}: ${opt[1]}`);
utils_1.logDebug(this.options, `Options: ${logOptions.join(" | ")}`);
// Implied options.
if (this.options.deleteBloat) {
if (!this.options.labels) {
utils_1.logDebug(this.options, "Action deleteBloat implies 'labels' detection");
}
this.options.labels = true;
}
if (this.options.deleteUnsafe) {
if (!this.options.unsafe) {
utils_1.logDebug(this.options, "Action deleteUnsafe implies 'unsafe' detection");
}
this.options.unsafe = true;
}
// Prepare the detection clients.
if (this.options.googleKeyfile)
await vision_1.default.prepare(this.options);
if (this.options.clarifaiKey)
await clarifai_1.default.prepare(this.options);
if (this.options.sightengineUser && this.options.sightengineSecret)
await sightengine_1.default.prepare(this.options);
// Create the images scanning queue.
this.queue = asyncLib.queue(this.scanFile, this.options.parallel);
// Scan image files on the specified folders.
for (let folder of this.options.folders) {
this.scanFolder(folder);
}
// Run file scanning tasks in parallel.
try {
await this.queue.drain();
this.end();
}
catch (ex) {
utils_1.logError(this.options, `Failure processing images`, ex);
}
};
/**
* End the scanning tasks.
* @param kill Force kill the scanning queue.
*/
this.end = async (kill) => {
try {
if (kill) {
this.queue.kill();
}
const duration = (Date.now() - this.startTime.valueOf()) / 1000;
utils_1.logInfo(this.options, `Scanned ${this.results.length} images in ${duration} seconds`);
await this.executeActions();
await this.saveOutput();
if (this.options.console) {
console.log("");
}
}
catch (ex) {
utils_1.logError(this.options, `Failure ending the processing queue`, ex);
}
};
/**
* Get image files for the specified folder.
* @param folder Image file path.
*/
this.scanFolder = (folder) => {
const scanner = (scanpath) => {
const filepath = path.join(folder, scanpath);
try {
const stats = fs.statSync(filepath);
if (stats.isDirectory()) {
if (this.options.deep) {
this.scanFolder(filepath);
}
}
else {
const ext = path.extname(filepath).toLowerCase().replace(".", "");
if (this.options.extensions.indexOf(ext) >= 0) {
utils_1.logDebug(this.options, `${filepath} added to queue`);
this.queue.push(filepath);
}
}
}
catch (ex) {
utils_1.logError(this.options, `Error reading ${filepath}`, ex);
}
};
// Make sure we have the correct folder path.
if (!path.isAbsolute(folder)) {
folder = path.join(process.cwd(), folder);
}
utils_1.logInfo(this.options, `Scanning ${folder}`);
try {
const contents = fs.readdirSync(folder);
utils_1.logDebug(this.options, `Found ${contents.length} files`);
for (let filepath of contents) {
scanner(filepath);
}
}
catch (ex) {
utils_1.logError(this.options, `Error reading ${folder}`, ex);
}
};
/**
* Scan the specified image file.
* @param filepath Image file path.
* @param callback Callback method.
*/
this.scanFile = async (filepath, callback) => {
const result = {
file: filepath,
details: {},
tags: {}
};
// Do not proceed if file was already scanned before.
if (this.results[filepath]) {
utils_1.logWarn(this.options, `File ${filepath} was already scanned`);
}
// Stop here once we have reached the API calls limit.
if (vision_1.default.apiCalls >= this.options.limit) {
if (vision_1.default.apiCalls === this.options.limit) {
utils_1.logInfo(this.options, `Limit of ${this.options.limit} API calls reached! Will NOT process more files...`);
}
this.end(true);
return callback();
}
const extension = path.extname(filepath).toLowerCase();
// Get file stats.
try {
const stat = fs.statSync(filepath);
result.details.size = stat.size;
result.details.date = stat.mtime.toISOString();
}
catch (ex) {
utils_1.logError(this.options, `Can't get filestat for ${filepath}`, ex);
}
utils_1.logDebug(this.options, `${filepath}: size ${result.details.size}, date ${result.details.date}`);
// Extract EXIF tags.
if (extension == ".jpg" || extension == ".jpeg") {
const exif = await utils_1.getEXIF(this.options, filepath);
result.details = Object.assign(result.details, exif);
}
// Google Vision detection.
if (this.options.googleKeyfile) {
if (this.options.objects) {
const dResult = await vision_1.default.detectObjects(this.options, filepath);
if (dResult)
result.tags = Object.assign(result.tags, dResult.tags);
}
if (this.options.labels) {
const dResult = await vision_1.default.detectLabels(this.options, filepath);
if (dResult)
result.tags = Object.assign(result.tags, dResult.tags);
}
if (this.options.landmarks) {
const dResult = await vision_1.default.detectLandmarks(this.options, filepath);
if (dResult)
result.tags = Object.assign(result.tags, dResult.tags);
}
if (this.options.logos) {
const dResult = await vision_1.default.detectLogos(this.options, filepath);
if (dResult)
result.tags = Object.assign(result.tags, dResult.tags);
}
if (this.options.unsafe) {
const dResult = await vision_1.default.detectUnsafe(this.options, filepath);
if (dResult)
result.tags = Object.assign(result.tags, dResult.tags);
}
}
// Clarifai detection.
if (this.options.clarifaiKey) {
const dResult = await clarifai_1.default.detect(this.options, filepath);
if (dResult)
result.tags = Object.assign(result.tags, dResult.tags);
}
// Sightengine detection.
if (this.options.sightengineUser && this.options.sightengineSecret) {
const dResult = await sightengine_1.default.detect(this.options, filepath);
if (dResult)
result.tags = Object.assign(result.tags, dResult.tags);
}
this.results.push(result);
return callback();
};
/**
* Execute actions after all passed images have been scanned.
*/
this.executeActions = async () => {
let executedActions = [];
const startTime = Date.now();
// Delete bloat images?
if (this.options.deleteBloat) {
executedActions.push("deleteBloat");
await actions_1.deleteBloat(this.options, this.results);
}
// Delete unsafe images?
if (this.options.deleteUnsafe) {
await actions_1.deleteUnsafe(this.options, this.results);
}
// Move scanned files to specific directory?
if (this.options.move) {
await actions_1.moveImages(this.options, this.results);
}
const duration = (Date.now() - startTime) / 1000;
// Log duration.
if (executedActions.length > 0) {
utils_1.logDebug(this.options, `Executed actions ${executedActions.join(", ")} in ${duration} seconds`);
}
else {
utils_1.logDebug(this.options, `No extra actions were executed`);
}
};
/**
* Save the execution output results to a file.
*/
this.saveOutput = async () => {
const executableFolder = path.dirname(require.main.filename) + "/";
const target = path.isAbsolute(this.options.output) ? this.options.output : path.join(executableFolder, this.options.output);
try {
fs.writeFileSync(target, JSON.stringify(this.results, null, 2));
utils_1.logInfo(this.options, `Saved results to ${target}`);
}
catch (ex) {
utils_1.logError(this.options, `Could not save output to ${target}`, ex);
}
};
this.options = options;
// Make sure the logger is set.
if (options.verbose) {
logger.levels.push("debug");
}
}
}
exports.IMGRecog = IMGRecog;
exports.default = IMGRecog;