img-duplicates/dist/lib.mjs.map

Find duplicate images based on visual similarity

{"version":3,"sources":["../src/lib.ts","../src/utils.ts"],"sourcesContent":["import path from \"node:path\";\nimport sharp from \"sharp\";\nimport fs from \"node:fs/promises\";\nimport createKDTree from \"static-kdtree\";\nimport { dhash, isFolder, isImageFile, mapLimit, pathExists } from \"./utils\";\n\n/**\n * Configuration options for the duplicate image detection algorithm.\n */\ninterface Options {\n  /**\n   * Size of the hash grid used for perceptual hashing.\n   * Larger values provide more precision but require more processing time.\n   * @default 8\n   */\n  hashSize?: number;\n\n  /**\n   * Maximum number of duplicate images to find for each source image.\n   * @default 100\n   */\n  maxDuplicates?: number;\n\n  /**\n   * Maximum Hamming distance between image hashes to consider them as duplicates.\n   * Lower values require higher visual similarity.\n   * @default 5\n   */\n  maxDistance?: number;\n}\n\nconst defaultOptions = {\n  hashSize: 8,\n  maxDuplicates: 100,\n  maxDistance: 5,\n};\n\n/**\n * Finds duplicate images based on visual similarity using perceptual hashing.\n *\n * This function analyzes images using difference hashing (dHash) to create perceptual fingerprints\n * and then uses a k-d tree for efficient similarity matching. Images are considered duplicates\n * if their hash distance is within the specified threshold.\n *\n * @param source - Path to a directory containing images, or an array of paths to directories and/or image files\n * @param options - Configuration options for the duplicate detection algorithm\n * @param options.hashSize - Size of the hash grid (default: 8). Larger values provide more precision but slower processing\n * @param options.maxDuplicates - Maximum number of duplicates to find per image (default: 100)\n * @param options.maxDistance - Maximum Hamming distance between hashes to consider images as duplicates (default: 5). Lower values require higher similarity\n *\n * @returns Promise that resolves to an array of duplicate groups. Each group is an array of image objects\n *          containing path, width, and height. Images within each group are sorted by resolution (highest first).\n *\n * @throws {Error} When image metadata cannot be read or when source paths don't exist\n *\n * @example\n * ```typescript\n * // Find duplicates in a single directory\n * const duplicates = await findDuplicateImages('/path/to/images');\n *\n * // Find duplicates across multiple sources with custom settings\n * const duplicates = await findDuplicateImages(\n *   ['/path/to/dir1', '/path/to/dir2', '/path/to/image.jpg'],\n *   {\n *     hashSize: 16,     // Higher precision\n *     maxDistance: 3,   // Stricter similarity\n *     maxDuplicates: 5  // Limit results\n *   }\n * );\n *\n * // Process results\n * duplicates.forEach((group, index) => {\n *   console.log(`Duplicate group ${index + 1}:`);\n *   group.forEach((image, i) => {\n *     const isHighest = i === 0 ? '(highest resolution)' : '';\n *     console.log(`  ${image.path} [${image.width}x${image.height}] ${isHighest}`);\n *   });\n * });\n * ```\n *\n * @since 1.0.0\n */\nexport default async function findDuplicateImages(\n  source: string | string[],\n  {\n    hashSize = 8,\n    maxDuplicates = 100,\n    maxDistance = 5,\n  }: Options = defaultOptions\n): Promise<Array<Array<{ path: string; width: number; height: number }>>> {\n  let imageFilePaths: string[];\n\n  if (Array.isArray(source)) {\n    // If source is an array of file and directory paths, process each item\n    const allImagePaths: string[] = [];\n\n    for (const item of source) {\n      // Check if the path exists first\n      if (!(await pathExists(item))) {\n        continue;\n      }\n\n      if (await isFolder(item)) {\n        // If it's a directory, read files and add image paths\n        try {\n          const files = await fs.readdir(item);\n          const imageFiles = files.filter(isImageFile).sort(); // Sort for consistent ordering\n          const imagePaths = imageFiles.map((f) => path.join(item, f));\n          allImagePaths.push(...imagePaths);\n        } catch {\n          // Skip directories that can't be read\n          continue;\n        }\n      } else if (isImageFile(item)) {\n        // If it's an image file, add it directly\n        allImagePaths.push(item);\n      }\n    }\n\n    imageFilePaths = allImagePaths.sort(); // Sort for consistent ordering\n  } else {\n    // If source is a directory path, read files and create full paths\n    const files = await fs.readdir(source);\n    const imageFiles = files.filter(isImageFile).sort(); // Sort for consistent ordering\n    imageFilePaths = imageFiles.map((f) => path.join(source, f));\n  }\n  const hashList: Array<Array<number>> = await mapLimit(\n    imageFilePaths,\n    1,\n    async (file: string): Promise<Array<number>> => {\n      const hash = await dhash(file, hashSize);\n      return [...hash];\n    }\n  );\n\n  const tree = createKDTree(hashList);\n\n  const duplicateIdxSet = new Set();\n  const duplicates: Array<Array<string>> = [];\n\n  for (let i = 0; i < hashList.length; i++) {\n    if (duplicateIdxSet.has(i)) {\n      continue;\n    }\n\n    const hash = hashList[i];\n    let duplicatesIdx = tree.knn(hash, maxDuplicates + 1, maxDistance); // +1 to account for the original image\n\n    if (duplicatesIdx && duplicatesIdx.length > 1) {\n      // Filter out any indices that have already been processed\n      duplicatesIdx = duplicatesIdx.filter((idx) => !duplicateIdxSet.has(idx));\n\n      // Only add if we still have more than one duplicate after filtering\n      if (duplicatesIdx.length > 1) {\n        // Ensure we don't exceed the maxDuplicates limit\n        const limitedDuplicates = duplicatesIdx.slice(0, maxDuplicates);\n        const dups = limitedDuplicates.map((i) => imageFilePaths[i]);\n        duplicates.push(dups);\n\n        // Mark all found duplicates as processed\n        for (const idx of limitedDuplicates) {\n          duplicateIdxSet.add(idx);\n        }\n      }\n    }\n  }\n\n  // Get metadata for each duplicate group and sort by resolution\n  const duplicatesWithMetadata: Array<\n    Array<{ path: string; width: number; height: number }>\n  > = [];\n\n  for (const files of duplicates) {\n    const filesWithMetadata: Array<{\n      path: string;\n      width: number;\n      height: number;\n    }> = [];\n\n    for (const file of files) {\n      if (await pathExists(file)) {\n        try {\n          const metadata = await sharp(file).metadata();\n          filesWithMetadata.push({\n            path: file,\n            width: metadata.width || 0,\n            height: metadata.height || 0,\n          });\n        } catch (error) {\n          throw new Error(`Could not read metadata for ${file}:`, error);\n        }\n      }\n    }\n\n    // Sort by resolution (width * height) in descending order (highest resolution first)\n    filesWithMetadata.sort((a, b) => {\n      // First sort by resolution (highest first)\n      const resolutionDiff = b.width * b.height - a.width * a.height;\n      if (resolutionDiff !== 0) return resolutionDiff;\n\n      // If resolution is the same, sort by filename\n      return path.basename(a.path).localeCompare(path.basename(b.path));\n    });\n\n    if (filesWithMetadata.length > 0) {\n      duplicatesWithMetadata.push(filesWithMetadata);\n    }\n  }\n\n  return duplicatesWithMetadata;\n}\n","import sharp from \"sharp\";\nimport fs from \"node:fs/promises\";\nimport assert from \"assert\";\n\n/**\n * Checks if a given path points to a directory.\n * @param path - The file system path to check\n * @returns Promise that resolves to true if the path is a directory\n */\nexport async function isFolder(path: string) {\n  try {\n    return (await fs.stat(path)).isDirectory();\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Processes an array with a concurrency limit to avoid overwhelming the system.\n * @param array - Array of items to process\n * @param limit - Maximum number of concurrent operations\n * @param iteratee - Async function to apply to each item\n * @returns Promise that resolves to an array of results\n */\nexport async function mapLimit<T, R>(\n  array: T[],\n  limit: number,\n  iteratee: (item: T) => Promise<R>\n): Promise<R[]> {\n  const results: R[] = [];\n  for (let i = 0; i < array.length; i += limit) {\n    const batch = array.slice(i, i + limit);\n    const batchResults = await Promise.all(batch.map(iteratee));\n    results.push(...batchResults);\n  }\n  return results;\n}\n\n/**\n * Checks if a file exists at the given path.\n * @param filePath - The path to check\n * @returns Promise that resolves to true if the file exists\n */\nexport async function pathExists(filePath: string): Promise<boolean> {\n  try {\n    await fs.access(filePath);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Determines if a file is a supported image format based on its extension.\n * @param path - The file path to check\n * @returns True if the file extension indicates a supported image format\n */\nexport function isImageFile(path: string) {\n  return (\n    path.endsWith(\".png\") ||\n    path.endsWith(\".jpg\") ||\n    path.endsWith(\".jpeg\") ||\n    path.endsWith(\".webp\") ||\n    path.endsWith(\".gif\") ||\n    path.endsWith(\".avif\") ||\n    path.endsWith(\".tiff\") ||\n    path.endsWith(\".tif\") ||\n    path.endsWith(\".svg\")\n  );\n}\n\nexport function px(pixels: Buffer, width: number, x: number, y: number) {\n  const pixel = width * y + x;\n  assert(pixel < pixels.length);\n  return pixels[pixel];\n}\n\nexport function binaryToHex(s: string) {\n  let output = \"\";\n  for (let i = 0; i < s.length; i += 4) {\n    const bytes = s.slice(i, i + 4);\n    const decimal = parseInt(bytes, 2);\n    const hex = decimal.toString(16);\n    output += hex;\n  }\n  return Buffer.from(output, \"hex\");\n}\n\n/**\n * Computes a difference hash (dHash) for an image, creating a perceptual fingerprint.\n *\n * The dHash algorithm works by:\n * 1. Converting the image to grayscale\n * 2. Resizing to a small grid (hashSize x hashSize+1)\n * 3. Comparing adjacent pixels to create a binary hash\n * 4. Converting the binary string to a hexadecimal buffer\n *\n * This creates a hash that is resilient to minor changes like compression,\n * resizing, and slight color adjustments while being sensitive to structural changes.\n *\n * @param path - Path to the image file\n * @param hashSize - Size of the hash grid (default: 8, creates 64-bit hash)\n * @returns Promise that resolves to a Buffer containing the image hash\n * @throws {Error} When the image cannot be processed by Sharp\n */\nexport async function dhash(path: string, hashSize = 8) {\n  const height = hashSize;\n  const width = height + 1;\n\n  // Covert to small gray image\n  const pixels = await sharp(path)\n    .grayscale()\n    .resize({ width, height, fit: \"fill\" })\n    .raw()\n    .toBuffer();\n\n  let difference = \"\";\n  for (let row = 0; row < height; row++) {\n    for (let col = 0; col < height; col++) {\n      // height is not a mistake here...\n      const left = px(pixels, width, col, row);\n      const right = px(pixels, width, col + 1, row);\n      difference += left < right ? 1 : 0;\n    }\n  }\n  return binaryToHex(difference);\n}"],"mappings":";AAAA,OAAO,UAAU;AACjB,OAAOA,YAAW;AAClB,OAAOC,SAAQ;AACf,OAAO,kBAAkB;;;ACHzB,OAAO,WAAW;AAClB,OAAO,QAAQ;AACf,OAAO,YAAY;AAOnB,eAAsB,SAASC,OAAc;AAC3C,MAAI;AACF,YAAQ,MAAM,GAAG,KAAKA,KAAI,GAAG,YAAY;AAAA,EAC3C,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AASA,eAAsB,SACpB,OACA,OACA,UACc;AACd,QAAM,UAAe,CAAC;AACtB,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,OAAO;AAC5C,UAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,KAAK;AACtC,UAAM,eAAe,MAAM,QAAQ,IAAI,MAAM,IAAI,QAAQ,CAAC;AAC1D,YAAQ,KAAK,GAAG,YAAY;AAAA,EAC9B;AACA,SAAO;AACT;AAOA,eAAsB,WAAW,UAAoC;AACnE,MAAI;AACF,UAAM,GAAG,OAAO,QAAQ;AACxB,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAOO,SAAS,YAAYA,OAAc;AACxC,SACEA,MAAK,SAAS,MAAM,KACpBA,MAAK,SAAS,MAAM,KACpBA,MAAK,SAAS,OAAO,KACrBA,MAAK,SAAS,OAAO,KACrBA,MAAK,SAAS,MAAM,KACpBA,MAAK,SAAS,OAAO,KACrBA,MAAK,SAAS,OAAO,KACrBA,MAAK,SAAS,MAAM,KACpBA,MAAK,SAAS,MAAM;AAExB;AAEO,SAAS,GAAG,QAAgB,OAAe,GAAW,GAAW;AACtE,QAAM,QAAQ,QAAQ,IAAI;AAC1B,SAAO,QAAQ,OAAO,MAAM;AAC5B,SAAO,OAAO,KAAK;AACrB;AAEO,SAAS,YAAY,GAAW;AACrC,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK,GAAG;AACpC,UAAM,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;AAC9B,UAAM,UAAU,SAAS,OAAO,CAAC;AACjC,UAAM,MAAM,QAAQ,SAAS,EAAE;AAC/B,cAAU;AAAA,EACZ;AACA,SAAO,OAAO,KAAK,QAAQ,KAAK;AAClC;AAmBA,eAAsB,MAAMA,OAAc,WAAW,GAAG;AACtD,QAAM,SAAS;AACf,QAAM,QAAQ,SAAS;AAGvB,QAAM,SAAS,MAAM,MAAMA,KAAI,EAC5B,UAAU,EACV,OAAO,EAAE,OAAO,QAAQ,KAAK,OAAO,CAAC,EACrC,IAAI,EACJ,SAAS;AAEZ,MAAI,aAAa;AACjB,WAAS,MAAM,GAAG,MAAM,QAAQ,OAAO;AACrC,aAAS,MAAM,GAAG,MAAM,QAAQ,OAAO;AAErC,YAAM,OAAO,GAAG,QAAQ,OAAO,KAAK,GAAG;AACvC,YAAM,QAAQ,GAAG,QAAQ,OAAO,MAAM,GAAG,GAAG;AAC5C,oBAAc,OAAO,QAAQ,IAAI;AAAA,IACnC;AAAA,EACF;AACA,SAAO,YAAY,UAAU;AAC/B;;;AD/FA,IAAM,iBAAiB;AAAA,EACrB,UAAU;AAAA,EACV,eAAe;AAAA,EACf,aAAa;AACf;AA+CA,eAAO,oBACL,QACA;AAAA,EACE,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,cAAc;AAChB,IAAa,gBAC2D;AACxE,MAAI;AAEJ,MAAI,MAAM,QAAQ,MAAM,GAAG;AAEzB,UAAM,gBAA0B,CAAC;AAEjC,eAAW,QAAQ,QAAQ;AAEzB,UAAI,CAAE,MAAM,WAAW,IAAI,GAAI;AAC7B;AAAA,MACF;AAEA,UAAI,MAAM,SAAS,IAAI,GAAG;AAExB,YAAI;AACF,gBAAM,QAAQ,MAAMC,IAAG,QAAQ,IAAI;AACnC,gBAAM,aAAa,MAAM,OAAO,WAAW,EAAE,KAAK;AAClD,gBAAM,aAAa,WAAW,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,CAAC,CAAC;AAC3D,wBAAc,KAAK,GAAG,UAAU;AAAA,QAClC,QAAQ;AAEN;AAAA,QACF;AAAA,MACF,WAAW,YAAY,IAAI,GAAG;AAE5B,sBAAc,KAAK,IAAI;AAAA,MACzB;AAAA,IACF;AAEA,qBAAiB,cAAc,KAAK;AAAA,EACtC,OAAO;AAEL,UAAM,QAAQ,MAAMA,IAAG,QAAQ,MAAM;AACrC,UAAM,aAAa,MAAM,OAAO,WAAW,EAAE,KAAK;AAClD,qBAAiB,WAAW,IAAI,CAAC,MAAM,KAAK,KAAK,QAAQ,CAAC,CAAC;AAAA,EAC7D;AACA,QAAM,WAAiC,MAAM;AAAA,IAC3C;AAAA,IACA;AAAA,IACA,OAAO,SAAyC;AAC9C,YAAM,OAAO,MAAM,MAAM,MAAM,QAAQ;AACvC,aAAO,CAAC,GAAG,IAAI;AAAA,IACjB;AAAA,EACF;AAEA,QAAM,OAAO,aAAa,QAAQ;AAElC,QAAM,kBAAkB,oBAAI,IAAI;AAChC,QAAM,aAAmC,CAAC;AAE1C,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,QAAI,gBAAgB,IAAI,CAAC,GAAG;AAC1B;AAAA,IACF;AAEA,UAAM,OAAO,SAAS,CAAC;AACvB,QAAI,gBAAgB,KAAK,IAAI,MAAM,gBAAgB,GAAG,WAAW;AAEjE,QAAI,iBAAiB,cAAc,SAAS,GAAG;AAE7C,sBAAgB,cAAc,OAAO,CAAC,QAAQ,CAAC,gBAAgB,IAAI,GAAG,CAAC;AAGvE,UAAI,cAAc,SAAS,GAAG;AAE5B,cAAM,oBAAoB,cAAc,MAAM,GAAG,aAAa;AAC9D,cAAM,OAAO,kBAAkB,IAAI,CAACC,OAAM,eAAeA,EAAC,CAAC;AAC3D,mBAAW,KAAK,IAAI;AAGpB,mBAAW,OAAO,mBAAmB;AACnC,0BAAgB,IAAI,GAAG;AAAA,QACzB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,yBAEF,CAAC;AAEL,aAAW,SAAS,YAAY;AAC9B,UAAM,oBAID,CAAC;AAEN,eAAW,QAAQ,OAAO;AACxB,UAAI,MAAM,WAAW,IAAI,GAAG;AAC1B,YAAI;AACF,gBAAM,WAAW,MAAMC,OAAM,IAAI,EAAE,SAAS;AAC5C,4BAAkB,KAAK;AAAA,YACrB,MAAM;AAAA,YACN,OAAO,SAAS,SAAS;AAAA,YACzB,QAAQ,SAAS,UAAU;AAAA,UAC7B,CAAC;AAAA,QACH,SAAS,OAAO;AACd,gBAAM,IAAI,MAAM,+BAA+B,IAAI,KAAK,KAAK;AAAA,QAC/D;AAAA,MACF;AAAA,IACF;AAGA,sBAAkB,KAAK,CAAC,GAAG,MAAM;AAE/B,YAAM,iBAAiB,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE;AACxD,UAAI,mBAAmB,EAAG,QAAO;AAGjC,aAAO,KAAK,SAAS,EAAE,IAAI,EAAE,cAAc,KAAK,SAAS,EAAE,IAAI,CAAC;AAAA,IAClE,CAAC;AAED,QAAI,kBAAkB,SAAS,GAAG;AAChC,6BAAuB,KAAK,iBAAiB;AAAA,IAC/C;AAAA,EACF;AAEA,SAAO;AACT;","names":["sharp","fs","path","fs","i","sharp"]}