@salama/image-finder
Version:
Advanced template matching tool with OpenCV.js featuring color sensitivity, batch processing, and performance optimization
1,161 lines (979 loc) • 54.6 kB
JavaScript
const TemplateMatcher = require('./TemplateMatcher');
const path = require('path');
class MatcherWrapper {
// OpenCV initialization caching
static _isInitialized = false;
static _initializationPromise = null;
/**
* Debug logger utility
* @param {boolean} debug - Whether debug is enabled
* @param {string} message - Message to log
* @param {Object} data - Optional data to log
*/
static debugLog(debug, message, data = null) {
if (debug) {
const timestamp = new Date().toISOString();
if (data) {
console.log(`[DEBUG ${timestamp}] ${message}`, data);
} else {
console.log(`[DEBUG ${timestamp}] ${message}`);
}
}
}
/**
* Simple info logger for non-debug output
* @param {string} message - Message to log
*/
static infoLog(message) {
console.log(message);
}
/**
* Performance timer utility
* @param {boolean} debug - Whether debug is enabled
* @param {string} operation - Operation name
* @returns {Object} Timer object with end() method
*/
static debugTimer(debug, operation) {
const startTime = debug ? performance.now() : null;
return {
end: () => {
if (debug && startTime) {
const duration = performance.now() - startTime;
console.log(`[TIMING] ${operation}: ${duration.toFixed(2)}ms`);
return duration;
}
return 0;
}
};
}
/**
* Scale images for performance optimization
* @param {string} imagePath - Path to image file
* @param {number} scaleFactor - Scale factor (e.g., 0.33 for 33% size)
* @param {boolean} debug - Enable debug logging
* @returns {Object} { scaledPath: string, originalSize: {width, height}, scaledSize: {width, height} }
*/
static async scaleImageForProcessing(imagePath, scaleFactor, debug = false) {
if (scaleFactor === 1.0) {
// No scaling needed
const sharp = require('sharp');
const metadata = await sharp(imagePath).metadata();
return {
scaledPath: imagePath,
originalSize: { width: metadata.width, height: metadata.height },
scaledSize: { width: metadata.width, height: metadata.height }
};
}
const sharp = require('sharp');
const path = require('path');
const fs = require('fs').promises;
this.debugLog(debug, `Scaling image by factor ${scaleFactor}`, { imagePath });
// Get original dimensions
const metadata = await sharp(imagePath).metadata();
const originalSize = { width: metadata.width, height: metadata.height };
// Calculate new dimensions
const newWidth = Math.round(metadata.width * scaleFactor);
const newHeight = Math.round(metadata.height * scaleFactor);
const scaledSize = { width: newWidth, height: newHeight };
// Create scaled image path
const ext = path.extname(imagePath);
const base = path.basename(imagePath, ext);
const dir = path.dirname(imagePath);
const scaledPath = path.join(dir, `${base}_scaled_${scaleFactor}${ext}`);
// Scale and save image with minimal artifacts
await sharp(imagePath)
.resize(newWidth, newHeight, {
kernel: sharp.kernel.mitchell, // Better for downscaling, less ringing
fit: 'inside', // Preserve aspect ratio
withoutEnlargement: true // Never upscale
})
.toFile(scaledPath);
this.debugLog(debug, `Scaled image saved`, {
originalSize,
scaledSize,
scaledPath,
sizeReduction: `${(100 * (1 - scaleFactor * scaleFactor)).toFixed(1)}%`
});
return { scaledPath, originalSize, scaledSize };
}
/**
* Scale coordinates back to original image size
* @param {Array} matches - Array of match objects with x, y, width, height
* @param {number} scaleFactor - Original scale factor used
* @param {boolean} debug - Enable debug logging
* @returns {Array} Matches with coordinates scaled back to original size
*/
static scaleMatchesBackToOriginal(matches, scaleFactor, debug = false) {
if (scaleFactor === 1.0) {
return matches; // No scaling needed
}
this.debugLog(debug, `Scaling ${matches.length} matches back by factor ${1/scaleFactor}`);
return matches.map(match => ({
...match,
x: Math.round(match.x / scaleFactor),
y: Math.round(match.y / scaleFactor),
width: Math.round(match.width / scaleFactor),
height: Math.round(match.height / scaleFactor),
centerX: Math.round(match.centerX / scaleFactor),
centerY: Math.round(match.centerY / scaleFactor)
}));
}
/**
* Generate unique annotation filename with timestamp and random component
* @param {string} baseDir - Base directory for annotations
* @param {string} prefix - Filename prefix (e.g., 'single', 'batch')
* @param {string} screenshotPath - Original screenshot path for context
* @returns {Promise<string>} Full path to unique annotation file
*/
static async generateAnnotationPath(baseDir, prefix, screenshotPath) {
const fs = require('fs').promises;
const path = require('path');
const crypto = require('crypto');
// Create annotation directory if it doesn't exist
await fs.mkdir(baseDir, { recursive: true });
// Generate timestamp (YYYYMMDD_HHMMSS)
const now = new Date();
const timestamp = now.getFullYear().toString() +
(now.getMonth() + 1).toString().padStart(2, '0') +
now.getDate().toString().padStart(2, '0') + '_' +
now.getHours().toString().padStart(2, '0') +
now.getMinutes().toString().padStart(2, '0') +
now.getSeconds().toString().padStart(2, '0');
// Generate random component (6 chars)
const randomId = crypto.randomBytes(3).toString('hex');
// Get screenshot basename without extension for context
const screenshotBase = path.basename(screenshotPath, path.extname(screenshotPath));
// Combine components: prefix_screenshotname_timestamp_random.png
const filename = `${prefix}_${screenshotBase}_${timestamp}_${randomId}.png`;
return path.join(baseDir, filename);
}
/**
* Find template variants automatically
* @param {string} templatePath - Base template path (e.g., "image.png")
* @param {boolean} debug - Enable debug logging
* @returns {Promise<Array<string>>} Array of template paths including original and variants
*/
static async findTemplateVariants(templatePath, debug = false) {
const path = require('path');
const fs = require('fs').promises;
const dir = path.dirname(templatePath);
const ext = path.extname(templatePath);
const baseName = path.basename(templatePath, ext);
const variants = [];
// Always include the original template if it exists
try {
await fs.access(templatePath);
variants.push(templatePath);
this.debugLog(debug, `Found base template: ${templatePath}`);
} catch (error) {
this.debugLog(debug, `Base template not found: ${templatePath}`);
}
// Look for numbered variants: image1.png, image2.png, etc.
let variantNumber = 1;
let consecutiveMisses = 0;
const maxConsecutiveMisses = 3; // Stop after 3 consecutive missing files
while (consecutiveMisses < maxConsecutiveMisses && variantNumber <= 100) { // Safety limit
const variantPath = path.join(dir, `${baseName}${variantNumber}${ext}`);
try {
await fs.access(variantPath);
variants.push(variantPath);
consecutiveMisses = 0; // Reset counter on found file
this.debugLog(debug, `Found variant: ${variantPath}`);
} catch (error) {
consecutiveMisses++;
this.debugLog(debug, `Variant not found: ${variantPath} (miss ${consecutiveMisses}/${maxConsecutiveMisses})`);
}
variantNumber++;
}
this.debugLog(debug, `Template variant detection complete`, {
baseTemplate: templatePath,
totalVariants: variants.length,
variants: variants.map(v => path.basename(v))
});
return variants;
}
/**
* Apply HSV color space enhancement for better color sensitivity
* @param {Object} cv - OpenCV instance (global cv object)
* @param {Object} img - OpenCV Mat image
* @param {Array<number>} hsvWeights - [Hue, Saturation, Value] channel weights
* @returns {Object} Enhanced OpenCV Mat
*/
static applyHSVEnhancement(cv, img, hsvWeights = [2.0, 1.5, 0.8]) {
// Convert BGR to HSV
const hsv = new cv.Mat();
cv.cvtColor(img, hsv, cv.COLOR_BGR2HSV);
// Split HSV channels
const hsvChannels = new cv.MatVector();
cv.split(hsv, hsvChannels);
// Apply weights to each channel
const weightedChannels = new cv.MatVector();
for (let i = 0; i < 3; i++) {
const channel = hsvChannels.get(i);
const weighted = new cv.Mat();
// Apply weight by scaling the channel
channel.convertTo(weighted, cv.CV_32F);
weighted.convertTo(weighted, cv.CV_8U, hsvWeights[i], 0);
weightedChannels.push_back(weighted);
channel.delete();
}
// Merge weighted channels back
const enhancedHSV = new cv.Mat();
cv.merge(weightedChannels, enhancedHSV);
// Convert back to BGR for template matching
const enhancedBGR = new cv.Mat();
cv.cvtColor(enhancedHSV, enhancedBGR, cv.COLOR_HSV2BGR);
// Clean up
hsv.delete();
hsvChannels.delete();
for (let i = 0; i < weightedChannels.size(); i++) {
weightedChannels.get(i).delete();
}
weightedChannels.delete();
enhancedHSV.delete();
return enhancedBGR;
}
/**
* Create HSV-enhanced TemplateMatcher for color-sensitive matching
* @param {Object} options - Original options
* @returns {Object} Enhanced options with HSV preprocessing
*/
static createHSVEnhancedOptions(options) {
return {
...options,
// Disable grayscale to preserve color information
grayscale: false,
// Store HSV settings for preprocessing
_hsvEnhanced: true,
_hsvWeights: options.hsvWeights || [2.0, 1.5, 0.8]
};
}
/**
* Universal scoring method that handles RGB, HSV, and dual scoring
* This method ensures consistent scoring logic across CLI and API
* @param {TemplateMatcher} matcher - Matcher instance
* @param {string} screenshotPath - Screenshot path
* @param {string} templatePath - Template path
* @param {Object} options - Options with scoring settings
* @returns {Object} Object with matches and annotated image
*/
static async performUniversalScoring(matcher, screenshotPath, templatePath, options) {
const debug = options.debug || false;
this.debugLog(debug, 'Universal scoring decision', {
useDualScoring: options.useDualScoring,
useHSV: options.useHSV,
rgbWeight: options.rgbWeight,
hsvWeight: options.hsvWeight
});
// Determine scoring strategy based on options
if (options.useDualScoring) {
this.debugLog(debug, 'Choosing dual scoring path');
return await this.performDualScoring(matcher, screenshotPath, templatePath, options);
} else if (options.useHSV) {
this.debugLog(debug, 'Choosing HSV-only scoring path');
return await this.performHSVScoring(matcher, screenshotPath, templatePath, options);
} else {
this.debugLog(debug, 'Choosing standard RGB scoring path');
return await this.performStandardScoring(matcher, screenshotPath, templatePath, options);
}
}
/**
* Perform dual scoring with both RGB and HSV matching
* @param {TemplateMatcher} matcher - Matcher instance
* @param {string} screenshotPath - Screenshot path
* @param {string} templatePath - Template path
* @param {Object} options - Options with dual scoring settings
* @returns {Object} Object with combined matches and annotated image
*/
static async performDualScoring(matcher, screenshotPath, templatePath, options) {
const debug = options.debug || false;
const rgbWeight = options.rgbWeight ?? 1.0;
const hsvWeight = options.hsvWeight ?? 0.5;
this.debugLog(debug, 'Starting dual scoring', {
rgbWeight: rgbWeight,
hsvWeight: hsvWeight,
totalWeight: rgbWeight + hsvWeight
});
// 1. Standard RGB matching
const rgbTimer = this.debugTimer(debug, 'RGB matching');
const rgbResult = await matcher.findMatches(screenshotPath, templatePath);
rgbTimer.end();
this.debugLog(debug, 'RGB matching complete', {
rgbMatches: rgbResult.matches.length,
rgbScores: rgbResult.matches.map(m => m.confidence)
});
// 2. HSV matching
const hsvTimer = this.debugTimer(debug, 'HSV matching');
const { screenshot, template } = await this.applyHSVPreprocessing(matcher, screenshotPath, templatePath, { ...options, useHSV: true });
// Perform template matching on HSV-enhanced images
const processedScreenshot = matcher.preprocessImage(screenshot);
const processedTemplate = matcher.preprocessImage(template);
const cv = matcher.cv;
const matchResult = new cv.Mat();
const mask = new cv.Mat();
const methods = options.method ? [options.method] : [cv.TM_CCOEFF_NORMED, cv.TM_SQDIFF_NORMED];
let hsvMatches = [];
for (const method of methods) {
cv.matchTemplate(processedScreenshot, processedTemplate, matchResult, method, mask);
const matches = matcher.extractBestMatch(matchResult, template.cols, template.rows, method);
hsvMatches = hsvMatches.concat(matches);
}
hsvMatches.sort((a, b) => b.confidence - a.confidence);
const hsvResult = { matches: matcher.applyNonMaxSuppression(hsvMatches) };
// Clean up HSV processing
screenshot.delete();
template.delete();
processedScreenshot.delete();
processedTemplate.delete();
matchResult.delete();
mask.delete();
hsvTimer.end();
this.debugLog(debug, 'HSV matching complete', {
hsvMatches: hsvResult.matches.length,
hsvScores: hsvResult.matches.map(m => m.confidence)
});
// 3. Combine scores using spatial matching
const combinedTimer = this.debugTimer(debug, 'Score combination');
const combinedMatches = this.combineRGBandHSVScores(rgbResult.matches, hsvResult.matches, rgbWeight, hsvWeight, options);
combinedTimer.end();
this.debugLog(debug, 'Score combination complete', {
finalMatches: combinedMatches.length,
finalScores: combinedMatches.map(m => ({
position: [m.x, m.y],
rgbScore: m.rgbScore,
hsvScore: m.hsvScore,
combinedScore: m.confidence
}))
});
// Create annotated image
const annotatedImage = matcher.annotateMatches(rgbResult.annotatedImage || await matcher.loadImageFromPath(screenshotPath), combinedMatches);
// Clean up
if (rgbResult.annotatedImage) {
rgbResult.annotatedImage.delete();
}
return {
matches: combinedMatches,
annotatedImage: annotatedImage
};
}
/**
* Perform HSV-only scoring
* @param {TemplateMatcher} matcher - Matcher instance
* @param {string} screenshotPath - Screenshot path
* @param {string} templatePath - Template path
* @param {Object} options - Options with HSV settings
* @returns {Object} Object with matches and annotated image
*/
static async performHSVScoring(matcher, screenshotPath, templatePath, options) {
const debug = options.debug || false;
this.debugLog(debug, 'Starting HSV-only scoring', {
hsvWeights: options.hsvWeights || [2.0, 1.5, 0.8]
});
// Apply HSV enhancement and perform custom matching
const { screenshot, template } = await this.applyHSVPreprocessing(matcher, screenshotPath, templatePath, options);
// Perform template matching on HSV-enhanced images
const processedScreenshot = matcher.preprocessImage(screenshot);
const processedTemplate = matcher.preprocessImage(template);
const cv = matcher.cv;
const matchResult = new cv.Mat();
const mask = new cv.Mat();
const methods = options.method ? [options.method] : [cv.TM_CCOEFF_NORMED, cv.TM_SQDIFF_NORMED];
let allMatches = [];
for (const method of methods) {
cv.matchTemplate(processedScreenshot, processedTemplate, matchResult, method, mask);
const matches = matcher.extractBestMatch(matchResult, template.cols, template.rows, method);
allMatches = allMatches.concat(matches);
}
allMatches.sort((a, b) => b.confidence - a.confidence);
const nmsMatches = matcher.applyNonMaxSuppression(allMatches);
// Create annotated image for HSV result
const annotatedImage = matcher.annotateMatches(screenshot, nmsMatches);
const result = {
matches: nmsMatches.slice(0, options.maxMatches || 10),
annotatedImage: annotatedImage
};
// Clean up HSV processing
screenshot.delete();
template.delete();
processedScreenshot.delete();
processedTemplate.delete();
matchResult.delete();
mask.delete();
this.debugLog(debug, 'HSV-only scoring complete', {
matchCount: result.matches.length,
scores: result.matches.map(m => m.confidence)
});
return result;
}
/**
* Perform standard RGB scoring
* @param {TemplateMatcher} matcher - Matcher instance
* @param {string} screenshotPath - Screenshot path
* @param {string} templatePath - Template path
* @param {Object} options - Options with standard settings
* @returns {Object} Object with matches and annotated image
*/
static async performStandardScoring(matcher, screenshotPath, templatePath, options) {
const debug = options.debug || false;
this.debugLog(debug, 'Starting standard RGB scoring');
// Standard matching without HSV
const result = await matcher.findMatches(screenshotPath, templatePath);
this.debugLog(debug, 'Standard RGB scoring complete', {
matchCount: result.matches.length,
scores: result.matches.map(m => m.confidence)
});
return result;
}
/**
* Combine RGB and HSV scores by spatial matching
* @param {Array<Object>} rgbMatches - RGB matching results
* @param {Array<Object>} hsvMatches - HSV matching results
* @param {number} rgbWeight - Weight for RGB scores
* @param {number} hsvWeight - Weight for HSV scores
* @param {Object} options - Options for spatial tolerance
* @returns {Array<Object>} Combined matches with weighted scores
*/
static combineRGBandHSVScores(rgbMatches, hsvMatches, rgbWeight, hsvWeight, options) {
const spatialTolerance = options.spatialTolerance || 10; // pixels
const combinedMatches = [];
// Start with RGB matches as base
for (const rgbMatch of rgbMatches) {
// Find corresponding HSV match within spatial tolerance
const correspondingHSV = hsvMatches.find(hsvMatch => {
const dx = Math.abs(rgbMatch.x - hsvMatch.x);
const dy = Math.abs(rgbMatch.y - hsvMatch.y);
return dx <= spatialTolerance && dy <= spatialTolerance;
});
let combinedScore, hsvScore;
if (correspondingHSV) {
// Both RGB and HSV found same location
hsvScore = correspondingHSV.confidence;
combinedScore = (rgbMatch.confidence * rgbWeight) + (correspondingHSV.confidence * hsvWeight);
} else {
// Only RGB match found
hsvScore = 0;
combinedScore = rgbMatch.confidence * rgbWeight;
}
combinedMatches.push({
...rgbMatch,
confidence: combinedScore,
rgbScore: rgbMatch.confidence,
hsvScore: hsvScore,
scoringMethod: correspondingHSV ? 'RGB+HSV' : 'RGB-only'
});
}
// Add HSV-only matches (not found in RGB)
for (const hsvMatch of hsvMatches) {
const hasCorrespondingRGB = rgbMatches.some(rgbMatch => {
const dx = Math.abs(rgbMatch.x - hsvMatch.x);
const dy = Math.abs(rgbMatch.y - hsvMatch.y);
return dx <= spatialTolerance && dy <= spatialTolerance;
});
if (!hasCorrespondingRGB) {
combinedMatches.push({
...hsvMatch,
confidence: hsvMatch.confidence * hsvWeight,
rgbScore: 0,
hsvScore: hsvMatch.confidence,
scoringMethod: 'HSV-only'
});
}
}
// Sort by combined confidence
return combinedMatches.sort((a, b) => b.confidence - a.confidence);
}
/**
* Apply HSV preprocessing to both screenshot and template
* @param {TemplateMatcher} matcher - Matcher instance
* @param {string} screenshotPath - Screenshot path
* @param {string} templatePath - Template path
* @param {Object} options - Options with HSV settings
* @returns {Object} Object with enhanced screenshot and template Mats
*/
static async applyHSVPreprocessing(matcher, screenshotPath, templatePath, options) {
// Ensure OpenCV is ready and get the cv instance
await matcher.ensureOpenCVReady();
const cv = matcher.cv;
if (!cv || !cv.Mat) {
throw new Error('OpenCV not properly initialized for HSV processing');
}
// Load original images
const screenshot = await matcher.loadImageFromPath(screenshotPath);
const template = await matcher.loadImageFromPath(templatePath);
let enhancedScreenshot = screenshot;
let enhancedTemplate = template;
if (options.useHSV) {
const hsvWeights = options.hsvWeights || [2.0, 1.5, 0.8];
// Apply HSV enhancement to both images
enhancedScreenshot = this.applyHSVEnhancement(cv, screenshot, hsvWeights);
enhancedTemplate = this.applyHSVEnhancement(cv, template, hsvWeights);
// Clean up original images
screenshot.delete();
template.delete();
}
return {
screenshot: enhancedScreenshot,
template: enhancedTemplate
};
}
/**
* Convert normalized threshold (0-1) to raw threshold for specific matching method
* @param {number} normalizedThreshold - Threshold value 0-1 (e.g., 0.8 for 80%)
* @param {string} method - OpenCV matching method name
* @returns {number} Raw threshold value for the specific method
*/
static unnormalizeThreshold(normalizedThreshold, method) {
if (!method) {
// Default methods - return average of both normalized methods
return normalizedThreshold;
}
const isMinMethod = method === 'TM_SQDIFF' || method === 'TM_SQDIFF_NORMED';
const isNormalized = method === 'TM_CCOEFF_NORMED' ||
method === 'TM_SQDIFF_NORMED' ||
method === 'TM_CCORR_NORMED';
if (isNormalized) {
// For normalized methods, raw threshold = normalized threshold
// For min methods (SQDIFF), invert the threshold
return isMinMethod ? (1.0 - normalizedThreshold) : normalizedThreshold;
} else {
// For non-normalized methods, convert back using the 100000 divisor
if (isMinMethod) {
// For SQDIFF: raw = (1.0 - normalized) * 100000
return (1.0 - normalizedThreshold) * 100000;
} else {
// For CCOEFF/CCORR: raw = normalized * 100000
return normalizedThreshold * 100000;
}
}
}
/**
* Ensure OpenCV is initialized (cached)
* @returns {Promise<void>}
*/
static async ensureOpenCVInitialized() {
if (this._isInitialized) {
return;
}
if (this._initializationPromise) {
return this._initializationPromise;
}
this._initializationPromise = this._initializeOpenCV();
await this._initializationPromise;
this._isInitialized = true;
}
/**
* Initialize OpenCV (internal method)
* @returns {Promise<void>}
*/
static async _initializeOpenCV() {
const tempMatcher = new TemplateMatcher();
await tempMatcher.ensureOpenCVReady();
// OpenCV is now initialized globally and cached
}
/**
* Find template matches in a screenshot image (unified method supporting single or multiple templates)
* @param {string} screenshotPath - Path to the large screenshot image
* @param {string|Array<string>} templatePathOrPaths - Single template path (string) or multiple template paths (array)
* @param {Object} options - Options object with all possible matcher options
* @param {number} options.threshold - Matching threshold (default 0.8)
* @param {boolean} options.multiscale - Enable multi-scale matching (default false)
* @param {Array<number>} options.scales - Scale factors for multi-scale matching
* @param {boolean} options.grayscale - Convert to grayscale (default true)
* @param {boolean} options.useEdges - Use edge detection (default true)
* @param {boolean} options.useBlur - Apply Gaussian blur (default false)
* @param {boolean} options.useInvert - Use color inversion (default true)
* @param {string} options.useChannel - Use specific color channel (default null)
* @param {Object} options.annotationColor - Color for annotations (default green)
* @param {number} options.lineThickness - Line thickness for annotations (default 4)
* @param {string} options.method - Force specific matching method (default null)
* @param {number} options.maxMatches - Maximum number of matches to return (default 10)
* @param {string} options.outputPath - Path to save annotated image (optional)
* @param {boolean} options.crossTemplateNMS - Enable cross-template non-maximum suppression (default true)
* @param {number} options.overlapThreshold - IoU threshold for cross-template NMS (default 0.7)
* @param {boolean} options.useHSV - Enable HSV color space for better color sensitivity (default false)
* @param {Array<number>} options.hsvWeights - HSV channel weights [H, S, V] (default [2.0, 1.5, 0.8])
* @param {boolean} options.useDualScoring - Enable dual RGB+HSV scoring system (default false)
* @param {number} options.rgbWeight - Weight for RGB matching score (default 1.0)
* @param {number} options.hsvWeight - Weight for HSV matching score (default 0.5)
* @param {boolean} options.debug - Enable debug logging and timing (default false)
* @returns {Promise<Array<Object>>} Array of match coordinates sorted top-to-bottom, left-to-right
*/
static async findMatches(screenshotPath, templatePathOrPaths, options = {}) {
// Merge with default options (same as CLI)
const mergedOptions = {
...this.getDefaultOptions(),
...options
};
// Normalize input: handle both single template (string) and multiple templates (array)
const templatePaths = Array.isArray(templatePathOrPaths)
? templatePathOrPaths
: [templatePathOrPaths];
const debug = mergedOptions.debug || false;
const overallTimer = this.debugTimer(debug, 'Overall findMatches');
// Separate timer for summary (always active)
const summaryStartTime = performance.now();
this.debugLog(debug, 'Starting template matching', {
screenshot: screenshotPath,
templateCount: templatePaths.length,
templates: templatePaths,
inputType: Array.isArray(templatePathOrPaths) ? 'array' : 'single',
options: { ...mergedOptions, debug: '[ENABLED]' }
});
// Ensure OpenCV is initialized (cached)
const initTimer = this.debugTimer(debug, 'OpenCV initialization');
await this.ensureOpenCVInitialized();
initTimer.end();
this.debugLog(debug, 'OpenCV initialization complete');
// Handle variant detection for single template in batch mode
let finalTemplatePaths = templatePaths;
if (mergedOptions.enableVariantDetection && templatePaths.length === 1) {
this.debugLog(debug, 'Single template in batch mode with variant detection enabled, searching for variants...');
const variants = await this.findTemplateVariants(templatePaths[0], debug);
if (variants.length > 1) {
this.debugLog(debug, `Found ${variants.length} template variants for batch processing`, {
baseTemplate: templatePaths[0],
variants: variants.map(v => require('path').basename(v))
});
finalTemplatePaths = variants;
} else {
this.debugLog(debug, 'No additional variants found, proceeding with original template');
}
}
// Handle input scaling for performance optimization
const inputScaling = mergedOptions.inputScaling || 1.0;
let actualScreenshotPath = screenshotPath;
let scaledTemplatePaths = [];
let scalingInfo = null;
if (inputScaling !== 1.0) {
const scalingTimer = this.debugTimer(debug, 'Input image scaling for batch');
// Scale screenshot and all templates in parallel
const scalingPromises = [
this.scaleImageForProcessing(screenshotPath, inputScaling, debug)
];
for (const templatePath of finalTemplatePaths) {
scalingPromises.push(this.scaleImageForProcessing(templatePath, inputScaling, debug));
}
const scalingResults = await Promise.all(scalingPromises);
const screenshotScaling = scalingResults[0];
actualScreenshotPath = screenshotScaling.scaledPath;
// Collect scaled template paths
for (let i = 1; i < scalingResults.length; i++) {
scaledTemplatePaths.push(scalingResults[i].scaledPath);
}
scalingInfo = {
scaleFactor: inputScaling,
screenshotOriginal: screenshotScaling.originalSize,
templateCount: finalTemplatePaths.length
};
scalingTimer.end();
this.debugLog(debug, 'Batch input scaling complete', scalingInfo);
} else {
scaledTemplatePaths = [...finalTemplatePaths];
}
// Remove duplicate template paths
const dedupeTimer = this.debugTimer(debug, 'Template deduplication');
const uniqueTemplatePaths = [...new Set(scaledTemplatePaths)];
dedupeTimer.end();
this.debugLog(debug, 'Template deduplication complete', {
originalCount: finalTemplatePaths.length,
uniqueCount: uniqueTemplatePaths.length,
removedDuplicates: finalTemplatePaths.length - uniqueTemplatePaths.length
});
// Convert normalized threshold to raw threshold for performance optimization
const normalizedThreshold = mergedOptions.threshold || 0.8;
const rawThreshold = this.unnormalizeThreshold(normalizedThreshold, mergedOptions.method);
this.debugLog(debug, 'Batch threshold conversion', {
normalizedThreshold: normalizedThreshold,
method: mergedOptions.method || 'default',
rawThreshold: rawThreshold
});
// Create a single matcher instance to ensure OpenCV is properly initialized
const initMatcherTimer = this.debugTimer(debug, 'Matcher initialization');
const baseMatcher = new TemplateMatcher({ threshold: rawThreshold });
await baseMatcher.ensureOpenCVReady();
initMatcherTimer.end();
this.debugLog(debug, 'Base matcher initialized, OpenCV ready for parallel processing');
// Process all templates in parallel
const batchTimer = this.debugTimer(debug, 'Parallel template processing');
const batchResults = await Promise.all(
uniqueTemplatePaths.map(async (templatePath, index) => {
const templateTimer = this.debugTimer(debug, `Template ${index + 1}/${uniqueTemplatePaths.length} (${templatePath})`);
// Only log template processing progress in debug mode
if (debug) {
const templateName = require('path').basename(templatePath);
this.debugLog(debug, `Processing template ${index + 1}/${uniqueTemplatePaths.length}: ${templateName}`);
}
// Ensure checkerboard options are disabled while preserving dual scoring config
const safeOptions = {
...mergedOptions,
useCheckerboard: false,
outputPath: undefined, // Don't save individual annotations
debug: false, // Always disable debug for individual templates to avoid spam
threshold: rawThreshold, // Pass raw threshold to TemplateMatcher
// Force color preservation for HSV processing
grayscale: mergedOptions.useHSV ? false : mergedOptions.grayscale
};
// Create matcher instance with provided options (OpenCV already initialized)
const matcher = new TemplateMatcher(safeOptions);
// Set method if specified (OpenCV already initialized)
if (mergedOptions.method) {
await matcher.ensureOpenCVReady();
matcher.setMethodByName(mergedOptions.method);
}
// Use universal scoring method (same as CLI and single API)
const result = await this.performUniversalScoring(matcher, actualScreenshotPath, templatePath, safeOptions);
// Don't create annotation for batch items
if (result.annotatedImage) {
result.annotatedImage.delete();
result.annotatedImage = null;
}
// Clean up the annotated image
if (result.annotatedImage) {
result.annotatedImage.delete();
}
// Add template path to each match
const matchesWithTemplate = result.matches.map(match => ({
...match,
templatePath: templatePath
}));
const templateTime = templateTimer.end();
this.debugLog(debug, `Template ${templatePath} complete`, {
matchCount: matchesWithTemplate.length,
processingTime: `${templateTime.toFixed(2)}ms`,
matches: matchesWithTemplate.map(m => ({
position: [m.x, m.y],
confidence: m.confidence
}))
});
return matchesWithTemplate;
})
);
batchTimer.end();
// Flatten all results
const flattenTimer = this.debugTimer(debug, 'Flattening batch results');
const allMatches = batchResults.flat();
flattenTimer.end();
this.debugLog(debug, 'Batch results flattened', {
totalMatches: allMatches.length,
matchesByTemplate: batchResults.map((matches, index) => ({
template: uniqueTemplatePaths[index],
matchCount: matches.length
}))
});
// Apply cross-template NMS if enabled
const crossTemplateNMS = mergedOptions.crossTemplateNMS !== false;
const overlapThreshold = mergedOptions.overlapThreshold ?? 0.7;
let finalMatches = allMatches;
if (crossTemplateNMS && allMatches.length > 1) {
const nmsTimer = this.debugTimer(debug, 'Cross-template NMS');
finalMatches = this.applyCrossTemplateNMS(allMatches, overlapThreshold);
nmsTimer.end();
this.debugLog(debug, 'Cross-template NMS complete', {
overlapThreshold: overlapThreshold,
beforeNMS: allMatches.length,
afterNMS: finalMatches.length,
removedOverlaps: allMatches.length - finalMatches.length
});
} else {
this.debugLog(debug, 'Cross-template NMS skipped', {
enabled: crossTemplateNMS,
matchCount: allMatches.length
});
}
// Apply post-processing filter to ensure all matches are above normalized threshold
// This is a safety net in case the raw threshold conversion wasn't perfect
const preFilterCount = finalMatches.length;
const postFilteredMatches = finalMatches.filter(match => match.confidence >= normalizedThreshold);
this.debugLog(debug, 'Post-processing threshold safety filter applied', {
normalizedThreshold: normalizedThreshold,
rawThreshold: rawThreshold,
beforeFilter: preFilterCount,
afterFilter: postFilteredMatches.length,
removedLowConfidence: preFilterCount - postFilteredMatches.length,
filteredMatches: finalMatches.filter(m => m.confidence < normalizedThreshold).map(m => ({
template: m.templatePath,
confidence: m.confidence,
position: [m.x, m.y]
}))
});
finalMatches = postFilteredMatches;
// Sort matches top-to-bottom, then left-to-right
const sortTimer = this.debugTimer(debug, 'Sorting matches');
const sortedMatches = this.sortMatches(finalMatches);
sortTimer.end();
this.debugLog(debug, 'Sorting complete', {
sortedOrder: sortedMatches.map(m => ({
template: m.templatePath,
position: [m.x, m.y],
confidence: m.confidence
}))
});
// Convert to simplified format
const simplifiedMatches = sortedMatches.map(match => ({
x: match.x,
y: match.y,
width: match.width,
height: match.height,
confidence: match.confidence,
centerX: match.x + Math.floor(match.width / 2),
centerY: match.y + Math.floor(match.height / 2),
templatePath: match.templatePath
}));
// Scale coordinates back to original size if input scaling was used
let batchFinalMatches = simplifiedMatches;
if (inputScaling !== 1.0 && simplifiedMatches.length > 0) {
const scaleBackTimer = this.debugTimer(debug, 'Scaling batch coordinates back to original size');
batchFinalMatches = this.scaleMatchesBackToOriginal(simplifiedMatches, inputScaling, debug);
scaleBackTimer.end();
}
// Save annotated image if annotation is enabled (use original screenshot)
if (batchFinalMatches.length > 0 && (mergedOptions.outputPath || mergedOptions.annotationDir)) {
const annotateTimer = this.debugTimer(debug, 'Creating batch annotated image');
let actualOutputPath = mergedOptions.outputPath;
if (!actualOutputPath) {
// Generate unique annotation path
const annotationDir = mergedOptions.annotationDir || './tmp/annotations';
actualOutputPath = await this.generateAnnotationPath(annotationDir, 'batch', screenshotPath);
}
await this.saveAnnotatedBatchImage(screenshotPath, batchFinalMatches, actualOutputPath, mergedOptions);
annotateTimer.end();
this.debugLog(debug, `Batch annotated image saved to: ${actualOutputPath}`);
// Store the actual path used for CLI output
mergedOptions._actualOutputPath = actualOutputPath;
}
// Clean up scaled images if they were created
if (inputScaling !== 1.0) {
const cleanupTimer = this.debugTimer(debug, 'Cleaning up batch scaled images');
const fs = require('fs').promises;
try {
if (actualScreenshotPath !== screenshotPath) {
await fs.unlink(actualScreenshotPath);
}
for (let i = 0; i < scaledTemplatePaths.length; i++) {
if (scaledTemplatePaths[i] !== templatePaths[i]) {
await fs.unlink(scaledTemplatePaths[i]);
}
}
this.debugLog(debug, 'Batch scaled images cleaned up');
} catch (error) {
this.debugLog(debug, 'Warning: Could not clean up batch scaled images', { error: error.message });
}
cleanupTimer.end();
}
const totalTime = overallTimer.end();
const summaryTotalTime = performance.now() - summaryStartTime;
// Show summary for regular (non-debug) runs
if (!debug) {
const originalTemplateCount = Array.isArray(templatePathOrPaths) ? templatePathOrPaths.length : 1;
const variantsChecked = finalTemplatePaths.length;
const inputType = Array.isArray(templatePathOrPaths) ? 'array' : 'string';
console.log(`📊 Template Matching Summary:`);
console.log(` Input: ${inputType} (${originalTemplateCount} template${originalTemplateCount > 1 ? 's' : ''})`);
console.log(` Variants checked: ${variantsChecked}`);
console.log(` Matches found: ${batchFinalMatches.length}`);
console.log(` Total time: ${summaryTotalTime.toFixed(2)}ms`);
console.log(` Average time per variant: ${(summaryTotalTime / variantsChecked).toFixed(2)}ms`);
}
this.debugLog(debug, 'Template matching complete', {
finalResultCount: batchFinalMatches.length,
totalTime: `${totalTime.toFixed(2)}ms`,
averageTimePerTemplate: `${(totalTime / uniqueTemplatePaths.length).toFixed(2)}ms`,
inputScaling: inputScaling,
performanceBoost: inputScaling !== 1.0 ? `~${Math.round(100 * (1 - inputScaling * inputScaling))}% faster` : 'none'
});
return batchFinalMatches;
}
/**
* Apply cross-template Non-Maximum Suppression
* @param {Array<Object>} matches - Array of match objects from different templates
* @param {number} overlapThreshold - IoU threshold for suppression (default 0.7)
* @returns {Array<Object>} Filtered array of matches
*/
static applyCrossTemplateNMS(matches, overlapThreshold = 0.7) {
if (matches.length <= 1) return matches;
// Sort by confidence (highest first)
const sortedMatches = matches.sort((a, b) => b.confidence - a.confidence);
const kept = [];
const suppressed = new Set();
for (let i = 0; i < sortedMatches.length; i++) {
if (suppressed.has(i)) continue;
const current = sortedMatches[i];
kept.push(current);
// Check against remaining matches
for (let j = i + 1; j < sortedMatches.length; j++) {
if (suppressed.has(j)) continue;
const other = sortedMatches[j];
const iou = this.calculateIoU(current, other);
// Remove if overlap exceeds threshold
if (iou > overlapThreshold) {
suppressed.add(j);
}
}
}
return kept;
}
/**
* Calculate Intersection over Union (IoU) between two matches
* @param {Object} match1 - First match object
* @param {Object} match2 - Second match object
* @returns {number} IoU value between 0 and 1
*/
static calculateIoU(match1, match2) {
const x1 = Math.max(match1.x, match2.x);
const y1 = Math.max(match1.y, match2.y);
const x2 = Math.min(match1.x + match1.width, match2.x + match2.width);
const y2 = Math.min(match1.y + match1.height, match2.y + match2.height);
if (x2 <= x1 || y2 <= y1) return 0;
const intersectionArea = (x2 - x1) * (y2 - y1);
const area1 = match1.width * match1.height;
const area2 = match2.width * match2.height;
const unionArea = area1 + area2 - intersectionArea;
return intersectionArea / unionArea;
}
/**
* Save annotated image for single template results
* @param {string} screenshotPath - Path to screenshot
* @param {Array<Object>} matches - Array of matches to annotate
* @param {string} outputPath - Path to save annotated image
* @param {Object} options - Options for annotation
*/
static async saveAnnotatedSingleImage(screenshotPath, matches, outputPath, options) {
// Create a temporary matcher for annotation
const matcher = new TemplateMatcher(options);
await matcher.ensureOpenCVReady();
// Load screenshot
const screenshot = await matcher.loadImageFromPath(screenshotPath);
// Create annotated image with consistent color
const annotatedImage = this.createAnnotatedImage(matcher, screenshot, matches, options);
await matcher.saveAnnotatedImage(annotatedImage, outputPath);
// Clean up
screenshot.delete();
annotatedImage.delete();
}
/**
* Save annotated image for batch results
* @param {string} screenshotPath - Path to screenshot
* @param {Array<Object>} matches - Array of matches to annotate
* @param {string} outputPath - Path to save annotated image
* @param {Object} options - Options for annotation
*/
static async saveAnnotatedBatchImage(screenshotPath, matches, outputPath, options) {
// Create a temporary matcher for annotation
const matcher = new TemplateMatcher(options);
await matcher.ensureOpenCVReady();
// Load screenshot
const screenshot = await matcher.loadImageFromPath(screenshotPath);
// Create annotated image with consistent color
const annotatedImage = this.createAnnotatedImage(matcher, screenshot, matches, options);
await matcher.saveAnnotatedImage(annotatedImage, outputPath);
// Clean up
screenshot.delete();
annotatedImage.delete();
}
/**
* Create annotated image with consistent color for all matches
* @param {TemplateMatcher} matcher - TemplateMatcher instance
* @param {Object} screenshot - OpenCV Mat of screenshot
* @param {Array<Object>} matches - Array of matches to annotate
* @param {Object} options - Options for annotation
* @returns {Object} Annotated OpenCV Mat