img-duplicates
Version:
Find duplicate images based on visual similarity
202 lines (198 loc) • 6.98 kB
JavaScript
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/lib.ts
var lib_exports = {};
__export(lib_exports, {
default: () => findDuplicateImages
});
module.exports = __toCommonJS(lib_exports);
var import_node_path = __toESM(require("path"));
var import_sharp2 = __toESM(require("sharp"));
var import_promises2 = __toESM(require("fs/promises"));
var import_static_kdtree = __toESM(require("static-kdtree"));
// src/utils.ts
var import_sharp = __toESM(require("sharp"));
var import_promises = __toESM(require("fs/promises"));
var import_assert = __toESM(require("assert"));
async function isFolder(path2) {
try {
return (await import_promises.default.stat(path2)).isDirectory();
} catch {
return false;
}
}
async function mapLimit(array, limit, iteratee) {
const results = [];
for (let i = 0; i < array.length; i += limit) {
const batch = array.slice(i, i + limit);
const batchResults = await Promise.all(batch.map(iteratee));
results.push(...batchResults);
}
return results;
}
async function pathExists(filePath) {
try {
await import_promises.default.access(filePath);
return true;
} catch {
return false;
}
}
function isImageFile(path2) {
return path2.endsWith(".png") || path2.endsWith(".jpg") || path2.endsWith(".jpeg") || path2.endsWith(".webp") || path2.endsWith(".gif") || path2.endsWith(".avif") || path2.endsWith(".tiff") || path2.endsWith(".tif") || path2.endsWith(".svg");
}
function px(pixels, width, x, y) {
const pixel = width * y + x;
(0, import_assert.default)(pixel < pixels.length);
return pixels[pixel];
}
function binaryToHex(s) {
let output = "";
for (let i = 0; i < s.length; i += 4) {
const bytes = s.slice(i, i + 4);
const decimal = parseInt(bytes, 2);
const hex = decimal.toString(16);
output += hex;
}
return Buffer.from(output, "hex");
}
async function dhash(path2, hashSize = 8) {
const height = hashSize;
const width = height + 1;
const pixels = await (0, import_sharp.default)(path2).grayscale().resize({ width, height, fit: "fill" }).raw().toBuffer();
let difference = "";
for (let row = 0; row < height; row++) {
for (let col = 0; col < height; col++) {
const left = px(pixels, width, col, row);
const right = px(pixels, width, col + 1, row);
difference += left < right ? 1 : 0;
}
}
return binaryToHex(difference);
}
// src/lib.ts
var defaultOptions = {
hashSize: 8,
maxDuplicates: 100,
maxDistance: 5
};
async function findDuplicateImages(source, {
hashSize = 8,
maxDuplicates = 100,
maxDistance = 5
} = defaultOptions) {
let imageFilePaths;
if (Array.isArray(source)) {
const allImagePaths = [];
for (const item of source) {
if (!await pathExists(item)) {
continue;
}
if (await isFolder(item)) {
try {
const files = await import_promises2.default.readdir(item);
const imageFiles = files.filter(isImageFile).sort();
const imagePaths = imageFiles.map((f) => import_node_path.default.join(item, f));
allImagePaths.push(...imagePaths);
} catch {
continue;
}
} else if (isImageFile(item)) {
allImagePaths.push(item);
}
}
imageFilePaths = allImagePaths.sort();
} else {
const files = await import_promises2.default.readdir(source);
const imageFiles = files.filter(isImageFile).sort();
imageFilePaths = imageFiles.map((f) => import_node_path.default.join(source, f));
}
const hashList = await mapLimit(
imageFilePaths,
1,
async (file) => {
const hash = await dhash(file, hashSize);
return [...hash];
}
);
const tree = (0, import_static_kdtree.default)(hashList);
const duplicateIdxSet = /* @__PURE__ */ new Set();
const duplicates = [];
for (let i = 0; i < hashList.length; i++) {
if (duplicateIdxSet.has(i)) {
continue;
}
const hash = hashList[i];
let duplicatesIdx = tree.knn(hash, maxDuplicates + 1, maxDistance);
if (duplicatesIdx && duplicatesIdx.length > 1) {
duplicatesIdx = duplicatesIdx.filter((idx) => !duplicateIdxSet.has(idx));
if (duplicatesIdx.length > 1) {
const limitedDuplicates = duplicatesIdx.slice(0, maxDuplicates);
const dups = limitedDuplicates.map((i2) => imageFilePaths[i2]);
duplicates.push(dups);
for (const idx of limitedDuplicates) {
duplicateIdxSet.add(idx);
}
}
}
}
const duplicatesWithMetadata = [];
for (const files of duplicates) {
const filesWithMetadata = [];
for (const file of files) {
if (await pathExists(file)) {
try {
const metadata = await (0, import_sharp2.default)(file).metadata();
filesWithMetadata.push({
path: file,
width: metadata.width || 0,
height: metadata.height || 0
});
} catch (error) {
throw new Error(`Could not read metadata for ${file}:`, error);
}
}
}
filesWithMetadata.sort((a, b) => {
const resolutionDiff = b.width * b.height - a.width * a.height;
if (resolutionDiff !== 0) return resolutionDiff;
return import_node_path.default.basename(a.path).localeCompare(import_node_path.default.basename(b.path));
});
if (filesWithMetadata.length > 0) {
duplicatesWithMetadata.push(filesWithMetadata);
}
}
return duplicatesWithMetadata;
}
//# sourceMappingURL=lib.js.map
// fix-cjs-exports
if (module.exports.default) {
Object.assign(module.exports.default, module.exports);
module.exports = module.exports.default;
delete module.exports.default;
}