img-duplicates
Version:
Find duplicate images based on visual similarity
420 lines (410 loc) • 13.5 kB
JavaScript
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
// src/cli.ts
var import_node_util = require("util");
var import_node_path2 = require("path");
var import_promises3 = require("fs/promises");
var import_node_readline = require("readline");
// src/lib.ts
var import_node_path = __toESM(require("path"));
var import_sharp2 = __toESM(require("sharp"));
var import_promises2 = __toESM(require("fs/promises"));
var import_static_kdtree = __toESM(require("static-kdtree"));
// src/utils.ts
var import_sharp = __toESM(require("sharp"));
var import_promises = __toESM(require("fs/promises"));
var import_assert = __toESM(require("assert"));
async function isFolder(path2) {
try {
return (await import_promises.default.stat(path2)).isDirectory();
} catch {
return false;
}
}
async function mapLimit(array, limit, iteratee) {
const results = [];
for (let i = 0; i < array.length; i += limit) {
const batch = array.slice(i, i + limit);
const batchResults = await Promise.all(batch.map(iteratee));
results.push(...batchResults);
}
return results;
}
async function pathExists(filePath) {
try {
await import_promises.default.access(filePath);
return true;
} catch {
return false;
}
}
function isImageFile(path2) {
return path2.endsWith(".png") || path2.endsWith(".jpg") || path2.endsWith(".jpeg") || path2.endsWith(".webp") || path2.endsWith(".gif") || path2.endsWith(".avif") || path2.endsWith(".tiff") || path2.endsWith(".tif") || path2.endsWith(".svg");
}
function px(pixels, width, x, y) {
const pixel = width * y + x;
(0, import_assert.default)(pixel < pixels.length);
return pixels[pixel];
}
function binaryToHex(s) {
let output = "";
for (let i = 0; i < s.length; i += 4) {
const bytes = s.slice(i, i + 4);
const decimal = parseInt(bytes, 2);
const hex = decimal.toString(16);
output += hex;
}
return Buffer.from(output, "hex");
}
async function dhash(path2, hashSize = 8) {
const height = hashSize;
const width = height + 1;
const pixels = await (0, import_sharp.default)(path2).grayscale().resize({ width, height, fit: "fill" }).raw().toBuffer();
let difference = "";
for (let row = 0; row < height; row++) {
for (let col = 0; col < height; col++) {
const left = px(pixels, width, col, row);
const right = px(pixels, width, col + 1, row);
difference += left < right ? 1 : 0;
}
}
return binaryToHex(difference);
}
// src/lib.ts
var defaultOptions = {
hashSize: 8,
maxDuplicates: 100,
maxDistance: 5
};
async function findDuplicateImages(source, {
hashSize = 8,
maxDuplicates = 100,
maxDistance = 5
} = defaultOptions) {
let imageFilePaths;
if (Array.isArray(source)) {
const allImagePaths = [];
for (const item of source) {
if (!await pathExists(item)) {
continue;
}
if (await isFolder(item)) {
try {
const files = await import_promises2.default.readdir(item);
const imageFiles = files.filter(isImageFile).sort();
const imagePaths = imageFiles.map((f) => import_node_path.default.join(item, f));
allImagePaths.push(...imagePaths);
} catch {
continue;
}
} else if (isImageFile(item)) {
allImagePaths.push(item);
}
}
imageFilePaths = allImagePaths.sort();
} else {
const files = await import_promises2.default.readdir(source);
const imageFiles = files.filter(isImageFile).sort();
imageFilePaths = imageFiles.map((f) => import_node_path.default.join(source, f));
}
const hashList = await mapLimit(
imageFilePaths,
1,
async (file) => {
const hash = await dhash(file, hashSize);
return [...hash];
}
);
const tree = (0, import_static_kdtree.default)(hashList);
const duplicateIdxSet = /* @__PURE__ */ new Set();
const duplicates = [];
for (let i = 0; i < hashList.length; i++) {
if (duplicateIdxSet.has(i)) {
continue;
}
const hash = hashList[i];
let duplicatesIdx = tree.knn(hash, maxDuplicates + 1, maxDistance);
if (duplicatesIdx && duplicatesIdx.length > 1) {
duplicatesIdx = duplicatesIdx.filter((idx) => !duplicateIdxSet.has(idx));
if (duplicatesIdx.length > 1) {
const limitedDuplicates = duplicatesIdx.slice(0, maxDuplicates);
const dups = limitedDuplicates.map((i2) => imageFilePaths[i2]);
duplicates.push(dups);
for (const idx of limitedDuplicates) {
duplicateIdxSet.add(idx);
}
}
}
}
const duplicatesWithMetadata = [];
for (const files of duplicates) {
const filesWithMetadata = [];
for (const file of files) {
if (await pathExists(file)) {
try {
const metadata = await (0, import_sharp2.default)(file).metadata();
filesWithMetadata.push({
path: file,
width: metadata.width || 0,
height: metadata.height || 0
});
} catch (error) {
throw new Error(`Could not read metadata for ${file}:`, error);
}
}
}
filesWithMetadata.sort((a, b) => {
const resolutionDiff = b.width * b.height - a.width * a.height;
if (resolutionDiff !== 0) return resolutionDiff;
return import_node_path.default.basename(a.path).localeCompare(import_node_path.default.basename(b.path));
});
if (filesWithMetadata.length > 0) {
duplicatesWithMetadata.push(filesWithMetadata);
}
}
return duplicatesWithMetadata;
}
// src/cli.ts
var HELP_TEXT = `
Duplicate Image Finder
Usage: img-duplicates [options] <source...>
Arguments:
source One or more directories or image files to search for duplicates
Options:
-h, --hash-size <size> Hash size for perceptual hashing (default: 8)
-d, --max-duplicates <num> Maximum number of duplicates to find per image (default: 100)
-m, --max-distance <dist> Maximum distance for similarity matching (default: 5)
--delete Delete duplicate images (keeps highest resolution, asks for confirmation)
--force-delete Delete duplicate images without confirmation
--help Show this help message
--version Show version number
Examples:
img-duplicates /path/to/images
img-duplicates /path/to/dir1 /path/to/dir2
img-duplicates --hash-size 16 --max-distance 3 /path/to/images
img-duplicates /path/to/image1.jpg /path/to/image2.png /path/to/dir
img-duplicates --delete /path/to/images
img-duplicates --force-delete /path/to/images
Note: When using --delete or --force-delete, the image with the highest resolution
in each duplicate group will be kept, and all others will be deleted.
`;
async function parseArguments() {
try {
const { values, positionals } = (0, import_node_util.parseArgs)({
args: process.argv.slice(2),
options: {
"hash-size": {
type: "string",
short: "h"
},
"max-duplicates": {
type: "string",
short: "d"
},
"max-distance": {
type: "string",
short: "m"
},
delete: {
type: "boolean"
},
"force-delete": {
type: "boolean"
},
help: {
type: "boolean"
},
version: {
type: "boolean"
}
},
allowPositionals: true
});
return {
source: positionals,
hashSize: values["hash-size"] ? parseInt(values["hash-size"]) : 8,
maxDuplicates: values["max-duplicates"] ? parseInt(values["max-duplicates"]) : 100,
maxDistance: values["max-distance"] ? parseInt(values["max-distance"]) : 5,
delete: values.delete || false,
forceDelete: values["force-delete"] || false,
help: values.help || false,
version: values.version || false
};
} catch (error) {
console.error("Error parsing arguments:", error.message);
process.exit(1);
}
}
async function validatePaths(paths) {
const resolvedPaths = [];
for (const path2 of paths) {
const resolvedPath = (0, import_node_path2.resolve)(path2);
try {
await (0, import_promises3.access)(resolvedPath);
resolvedPaths.push(resolvedPath);
} catch (error) {
console.error(`Error: Path does not exist: ${path2}`);
process.exit(1);
}
}
return resolvedPaths;
}
function validateOptions(options) {
if (options.hashSize < 1 || options.hashSize > 32) {
console.error("Error: Hash size must be between 1 and 32");
process.exit(1);
}
if (options.maxDuplicates < 1) {
console.error("Error: Max duplicates must be at least 1");
process.exit(1);
}
if (options.maxDistance < 0) {
console.error("Error: Max distance must be non-negative");
process.exit(1);
}
if (options.delete && options.forceDelete) {
console.error(
"Error: Cannot use both --delete and --force-delete at the same time"
);
process.exit(1);
}
}
function formatResults(duplicates, showDelete = false) {
if (duplicates.length === 0) {
console.log("No duplicate images found.");
return;
}
console.log(`Found ${duplicates.length} group(s) of duplicate images:
`);
duplicates.forEach((group, index) => {
console.log(`Group ${index + 1}:`);
group.forEach((image, imageIndex) => {
const resolution = `${image.width}x${image.height}`;
let marker = "";
if (imageIndex === 0) {
marker = showDelete ? "(highest resolution - will be kept)" : "(highest resolution)";
} else if (showDelete) {
marker = "(will be deleted)";
}
console.log(` ${image.path} [${resolution}] ${marker}`);
});
console.log("");
});
}
async function askForConfirmation(message) {
const rl = (0, import_node_readline.createInterface)({
input: process.stdin,
output: process.stdout
});
return new Promise((resolve2) => {
rl.question(`${message} (y/N): `, (answer) => {
rl.close();
resolve2(answer.toLowerCase() === "y" || answer.toLowerCase() === "yes");
});
});
}
async function deleteDuplicates(duplicates, forceDelete = false) {
if (duplicates.length === 0) {
return;
}
const filesToDelete = [];
duplicates.forEach((group) => {
for (let i = 1; i < group.length; i++) {
filesToDelete.push(group[i].path);
}
});
if (filesToDelete.length === 0) {
console.log("No files to delete.");
return;
}
console.log(`
About to delete ${filesToDelete.length} duplicate image(s):`);
filesToDelete.forEach((file) => console.log(` ${file}`));
console.log("");
let shouldDelete = forceDelete;
if (!forceDelete) {
shouldDelete = await askForConfirmation(
"Do you want to proceed with deletion?"
);
}
if (!shouldDelete) {
console.log("Deletion cancelled.");
return;
}
let deletedCount = 0;
let errorCount = 0;
for (const file of filesToDelete) {
try {
await (0, import_promises3.unlink)(file);
console.log(`Deleted: ${file}`);
deletedCount++;
} catch (error) {
console.error(`Failed to delete ${file}: ${error.message}`);
errorCount++;
}
}
console.log(
`
Deletion completed: ${deletedCount} files deleted, ${errorCount} errors.`
);
}
async function main() {
const options = await parseArguments();
if (options.help) {
console.log(HELP_TEXT);
process.exit(0);
}
if (options.version) {
console.log("1.0.2");
process.exit(0);
}
if (options.source.length === 0) {
console.error("Error: No source paths provided");
console.log(HELP_TEXT);
process.exit(1);
}
validateOptions(options);
const validatedPaths = await validatePaths(options.source);
try {
console.log("Searching for duplicate images...");
const duplicates = await findDuplicateImages(validatedPaths, {
hashSize: options.hashSize,
maxDuplicates: options.maxDuplicates,
maxDistance: options.maxDistance
});
const shouldDelete = options.delete || options.forceDelete;
formatResults(duplicates, shouldDelete);
if (shouldDelete) {
await deleteDuplicates(duplicates, options.forceDelete);
}
} catch (error) {
console.error("Error finding duplicates:", error.message);
process.exit(1);
}
}
main().catch((error) => {
console.error("Unexpected error:", error);
process.exit(1);
});
//# sourceMappingURL=cli.js.map