img-duplicates/dist/cli.mjs

Find duplicate images based on visual similarity

#!/usr/bin/env node

// src/cli.ts
import { parseArgs } from "util";
import { resolve } from "path";
import { access, unlink } from "fs/promises";
import { createInterface } from "readline";

// src/lib.ts
import path from "path";
import sharp2 from "sharp";
import fs2 from "fs/promises";
import createKDTree from "static-kdtree";

// src/utils.ts
import sharp from "sharp";
import fs from "fs/promises";
import assert from "assert";
async function isFolder(path2) {
  try {
    return (await fs.stat(path2)).isDirectory();
  } catch {
    return false;
  }
}
async function mapLimit(array, limit, iteratee) {
  const results = [];
  for (let i = 0; i < array.length; i += limit) {
    const batch = array.slice(i, i + limit);
    const batchResults = await Promise.all(batch.map(iteratee));
    results.push(...batchResults);
  }
  return results;
}
async function pathExists(filePath) {
  try {
    await fs.access(filePath);
    return true;
  } catch {
    return false;
  }
}
function isImageFile(path2) {
  return path2.endsWith(".png") || path2.endsWith(".jpg") || path2.endsWith(".jpeg") || path2.endsWith(".webp") || path2.endsWith(".gif") || path2.endsWith(".avif") || path2.endsWith(".tiff") || path2.endsWith(".tif") || path2.endsWith(".svg");
}
function px(pixels, width, x, y) {
  const pixel = width * y + x;
  assert(pixel < pixels.length);
  return pixels[pixel];
}
function binaryToHex(s) {
  let output = "";
  for (let i = 0; i < s.length; i += 4) {
    const bytes = s.slice(i, i + 4);
    const decimal = parseInt(bytes, 2);
    const hex = decimal.toString(16);
    output += hex;
  }
  return Buffer.from(output, "hex");
}
async function dhash(path2, hashSize = 8) {
  const height = hashSize;
  const width = height + 1;
  const pixels = await sharp(path2).grayscale().resize({ width, height, fit: "fill" }).raw().toBuffer();
  let difference = "";
  for (let row = 0; row < height; row++) {
    for (let col = 0; col < height; col++) {
      const left = px(pixels, width, col, row);
      const right = px(pixels, width, col + 1, row);
      difference += left < right ? 1 : 0;
    }
  }
  return binaryToHex(difference);
}

// src/lib.ts
var defaultOptions = {
  hashSize: 8,
  maxDuplicates: 100,
  maxDistance: 5
};
async function findDuplicateImages(source, {
  hashSize = 8,
  maxDuplicates = 100,
  maxDistance = 5
} = defaultOptions) {
  let imageFilePaths;
  if (Array.isArray(source)) {
    const allImagePaths = [];
    for (const item of source) {
      if (!await pathExists(item)) {
        continue;
      }
      if (await isFolder(item)) {
        try {
          const files = await fs2.readdir(item);
          const imageFiles = files.filter(isImageFile).sort();
          const imagePaths = imageFiles.map((f) => path.join(item, f));
          allImagePaths.push(...imagePaths);
        } catch {
          continue;
        }
      } else if (isImageFile(item)) {
        allImagePaths.push(item);
      }
    }
    imageFilePaths = allImagePaths.sort();
  } else {
    const files = await fs2.readdir(source);
    const imageFiles = files.filter(isImageFile).sort();
    imageFilePaths = imageFiles.map((f) => path.join(source, f));
  }
  const hashList = await mapLimit(
    imageFilePaths,
    1,
    async (file) => {
      const hash = await dhash(file, hashSize);
      return [...hash];
    }
  );
  const tree = createKDTree(hashList);
  const duplicateIdxSet = /* @__PURE__ */ new Set();
  const duplicates = [];
  for (let i = 0; i < hashList.length; i++) {
    if (duplicateIdxSet.has(i)) {
      continue;
    }
    const hash = hashList[i];
    let duplicatesIdx = tree.knn(hash, maxDuplicates + 1, maxDistance);
    if (duplicatesIdx && duplicatesIdx.length > 1) {
      duplicatesIdx = duplicatesIdx.filter((idx) => !duplicateIdxSet.has(idx));
      if (duplicatesIdx.length > 1) {
        const limitedDuplicates = duplicatesIdx.slice(0, maxDuplicates);
        const dups = limitedDuplicates.map((i2) => imageFilePaths[i2]);
        duplicates.push(dups);
        for (const idx of limitedDuplicates) {
          duplicateIdxSet.add(idx);
        }
      }
    }
  }
  const duplicatesWithMetadata = [];
  for (const files of duplicates) {
    const filesWithMetadata = [];
    for (const file of files) {
      if (await pathExists(file)) {
        try {
          const metadata = await sharp2(file).metadata();
          filesWithMetadata.push({
            path: file,
            width: metadata.width || 0,
            height: metadata.height || 0
          });
        } catch (error) {
          throw new Error(`Could not read metadata for ${file}:`, error);
        }
      }
    }
    filesWithMetadata.sort((a, b) => {
      const resolutionDiff = b.width * b.height - a.width * a.height;
      if (resolutionDiff !== 0) return resolutionDiff;
      return path.basename(a.path).localeCompare(path.basename(b.path));
    });
    if (filesWithMetadata.length > 0) {
      duplicatesWithMetadata.push(filesWithMetadata);
    }
  }
  return duplicatesWithMetadata;
}

// src/cli.ts
var HELP_TEXT = `
Duplicate Image Finder

Usage: img-duplicates [options] <source...>

Arguments:
  source                One or more directories or image files to search for duplicates

Options:
  -h, --hash-size <size>     Hash size for perceptual hashing (default: 8)
  -d, --max-duplicates <num> Maximum number of duplicates to find per image (default: 100)
  -m, --max-distance <dist>  Maximum distance for similarity matching (default: 5)
  --delete                   Delete duplicate images (keeps highest resolution, asks for confirmation)
  --force-delete             Delete duplicate images without confirmation
  --help                     Show this help message
  --version                  Show version number

Examples:
  img-duplicates /path/to/images
  img-duplicates /path/to/dir1 /path/to/dir2
  img-duplicates --hash-size 16 --max-distance 3 /path/to/images
  img-duplicates /path/to/image1.jpg /path/to/image2.png /path/to/dir
  img-duplicates --delete /path/to/images
  img-duplicates --force-delete /path/to/images

Note: When using --delete or --force-delete, the image with the highest resolution
      in each duplicate group will be kept, and all others will be deleted.
`;
async function parseArguments() {
  try {
    const { values, positionals } = parseArgs({
      args: process.argv.slice(2),
      options: {
        "hash-size": {
          type: "string",
          short: "h"
        },
        "max-duplicates": {
          type: "string",
          short: "d"
        },
        "max-distance": {
          type: "string",
          short: "m"
        },
        delete: {
          type: "boolean"
        },
        "force-delete": {
          type: "boolean"
        },
        help: {
          type: "boolean"
        },
        version: {
          type: "boolean"
        }
      },
      allowPositionals: true
    });
    return {
      source: positionals,
      hashSize: values["hash-size"] ? parseInt(values["hash-size"]) : 8,
      maxDuplicates: values["max-duplicates"] ? parseInt(values["max-duplicates"]) : 100,
      maxDistance: values["max-distance"] ? parseInt(values["max-distance"]) : 5,
      delete: values.delete || false,
      forceDelete: values["force-delete"] || false,
      help: values.help || false,
      version: values.version || false
    };
  } catch (error) {
    console.error("Error parsing arguments:", error.message);
    process.exit(1);
  }
}
async function validatePaths(paths) {
  const resolvedPaths = [];
  for (const path2 of paths) {
    const resolvedPath = resolve(path2);
    try {
      await access(resolvedPath);
      resolvedPaths.push(resolvedPath);
    } catch (error) {
      console.error(`Error: Path does not exist: ${path2}`);
      process.exit(1);
    }
  }
  return resolvedPaths;
}
function validateOptions(options) {
  if (options.hashSize < 1 || options.hashSize > 32) {
    console.error("Error: Hash size must be between 1 and 32");
    process.exit(1);
  }
  if (options.maxDuplicates < 1) {
    console.error("Error: Max duplicates must be at least 1");
    process.exit(1);
  }
  if (options.maxDistance < 0) {
    console.error("Error: Max distance must be non-negative");
    process.exit(1);
  }
  if (options.delete && options.forceDelete) {
    console.error(
      "Error: Cannot use both --delete and --force-delete at the same time"
    );
    process.exit(1);
  }
}
function formatResults(duplicates, showDelete = false) {
  if (duplicates.length === 0) {
    console.log("No duplicate images found.");
    return;
  }
  console.log(`Found ${duplicates.length} group(s) of duplicate images:
`);
  duplicates.forEach((group, index) => {
    console.log(`Group ${index + 1}:`);
    group.forEach((image, imageIndex) => {
      const resolution = `${image.width}x${image.height}`;
      let marker = "";
      if (imageIndex === 0) {
        marker = showDelete ? "(highest resolution - will be kept)" : "(highest resolution)";
      } else if (showDelete) {
        marker = "(will be deleted)";
      }
      console.log(`  ${image.path} [${resolution}] ${marker}`);
    });
    console.log("");
  });
}
async function askForConfirmation(message) {
  const rl = createInterface({
    input: process.stdin,
    output: process.stdout
  });
  return new Promise((resolve2) => {
    rl.question(`${message} (y/N): `, (answer) => {
      rl.close();
      resolve2(answer.toLowerCase() === "y" || answer.toLowerCase() === "yes");
    });
  });
}
async function deleteDuplicates(duplicates, forceDelete = false) {
  if (duplicates.length === 0) {
    return;
  }
  const filesToDelete = [];
  duplicates.forEach((group) => {
    for (let i = 1; i < group.length; i++) {
      filesToDelete.push(group[i].path);
    }
  });
  if (filesToDelete.length === 0) {
    console.log("No files to delete.");
    return;
  }
  console.log(`
About to delete ${filesToDelete.length} duplicate image(s):`);
  filesToDelete.forEach((file) => console.log(`  ${file}`));
  console.log("");
  let shouldDelete = forceDelete;
  if (!forceDelete) {
    shouldDelete = await askForConfirmation(
      "Do you want to proceed with deletion?"
    );
  }
  if (!shouldDelete) {
    console.log("Deletion cancelled.");
    return;
  }
  let deletedCount = 0;
  let errorCount = 0;
  for (const file of filesToDelete) {
    try {
      await unlink(file);
      console.log(`Deleted: ${file}`);
      deletedCount++;
    } catch (error) {
      console.error(`Failed to delete ${file}: ${error.message}`);
      errorCount++;
    }
  }
  console.log(
    `
Deletion completed: ${deletedCount} files deleted, ${errorCount} errors.`
  );
}
async function main() {
  const options = await parseArguments();
  if (options.help) {
    console.log(HELP_TEXT);
    process.exit(0);
  }
  if (options.version) {
    console.log("1.0.2");
    process.exit(0);
  }
  if (options.source.length === 0) {
    console.error("Error: No source paths provided");
    console.log(HELP_TEXT);
    process.exit(1);
  }
  validateOptions(options);
  const validatedPaths = await validatePaths(options.source);
  try {
    console.log("Searching for duplicate images...");
    const duplicates = await findDuplicateImages(validatedPaths, {
      hashSize: options.hashSize,
      maxDuplicates: options.maxDuplicates,
      maxDistance: options.maxDistance
    });
    const shouldDelete = options.delete || options.forceDelete;
    formatResults(duplicates, shouldDelete);
    if (shouldDelete) {
      await deleteDuplicates(duplicates, options.forceDelete);
    }
  } catch (error) {
    console.error("Error finding duplicates:", error.message);
    process.exit(1);
  }
}
main().catch((error) => {
  console.error("Unexpected error:", error);
  process.exit(1);
});
//# sourceMappingURL=cli.mjs.map