image-js/src/align/affineTransfrom/getAffineTransform.ts

Image processing and manipulation in JavaScript

import { getAffineTransform as matrixGetAffineTransform } from 'ml-affine-transform';
import { ransac } from 'ml-ransac';

import type { Image } from '../../Image.js';
import { getBrief } from '../../featureMatching/descriptors/getBrief.js';
import type { Match } from '../../featureMatching/index.js';
import {
  Montage,
  MontageDisposition,
  bruteForceOneMatch,
  getCrosscheckMatches,
} from '../../featureMatching/index.js';
import { filterEuclideanDistance } from '../../featureMatching/matching/filterEuclideanDistance.js';
import type { Point } from '../../geometry/index.js';
import { writeSync } from '../../save/index.js';
import { ImageColorModel } from '../../utils/constants/colorModels.js';
import { getMinMax } from '../../utils/getMinMax.js';

import { affineFitFunction } from './affineFitFunction.js';
import { createAffineTransformModel } from './createAffineTransformModel.js';
import { getEuclideanDistance } from './getEuclideanDistance.js';
import { getMatrixFromPoints } from './getMatrixFromPoints.js';
import { getSourceWithoutMargins } from './utils/getSourceWithoutMargins.js';

export interface GetAffineTransformOptions {
  /**
   * @default `31`
   */
  centroidPatchDiameter?: number;
  /**
   * @default `10`
   */
  bestKeypointRadius?: number;
  /**
   * Should only the crossckeck matches be considered.
   * @default `true`
   */
  crosscheck?: boolean;
  /**
   * Should the contrast of the images be enhanced before feature matching.
   * @default `true`
   */
  enhanceContrast?: boolean;
  /**
   * Origin of the destination image relative to the top-left corner of the source image.
   * Roughly indicates the position of the destination image in the source image. Is used
   * to filter matches by distance as well as to define a subarea of the source image to
   * use for contrast enhancement.
   * @default `{ column: 0, row: 0 }`
   */
  destinationOrigin?: Point;
  /**
   * Max number of iterations of the ransac algorithm.
   */
  maxRansacNbIterations?: number;
  /**
   * Save images with matches for debugging.
   * @default `false`
   */
  debug?: boolean;
  /**
   * Path of the debug image.
   * @default `${import.meta.dirname}/montage.png`
   */
  debugImagePath?: string;
}

export interface AffineTransform {
  /**
   * Translation of source points along x and y axes.
   */
  translation: Point;
  /**
   * Clockwise angle in degrees.
   */
  rotation: number;
  /**
   * Scaling factor from source to destination.
   */
  scale: number;
}

export interface GetAffineTransformResult {
  /**
   * Affine transformation from source to destination.
   */
  transform: AffineTransform;
  stats: {
    /**
     * Number of matches of feature matching between source and destination.
     * The bigger this number is, the better.
     */
    nbMatches: number;
    /**
     * Number of inliers resulting from the ransac algorithm.
     */
    nbInliers: number;
    /**
     * Number of iterations of the RANSAC algorithm.
     */
    nbRansacIterations: number;
    /**
     * Number of source keypoints used for matching.
     */
    nbSourceKeypoints: number;
    /**
     * Number of destination keypoints used for matching.
     */
    nbDestinationKeypoints: number;
  };
}

/**
 * Get the affine transformation from the source to the destination image.
 * @param source - Source image. Should be the image to align on the reference image.
 * It can have an additional margin, specified in the options.
 * @param destination - Destination image. Should be the reference image.
 * @param options - Get destination translation options.
 * @returns The affine transformation from source to destination image.
 */
export function getAffineTransform(
  source: Image,
  destination: Image,
  options: GetAffineTransformOptions = {},
): GetAffineTransformResult {
  const {
    centroidPatchDiameter = 31,
    bestKeypointRadius = 5,
    enhanceContrast = true,
    crosscheck = true,
    destinationOrigin = { column: 0, row: 0 },
    maxRansacNbIterations,
    debug = false,
    debugImagePath = `${import.meta.dirname}/montage.png`,
  } = options;

  if (source.colorModel !== ImageColorModel.GREY) {
    source = source.grey();
  }
  if (destination.colorModel !== ImageColorModel.GREY) {
    destination = destination.grey();
  }

  // enhance images contrast
  if (enhanceContrast) {
    const sourceWithoutMargin = getSourceWithoutMargins(
      source,
      destination,
      destinationOrigin,
    );

    const sourceExtremums = getMinMax(sourceWithoutMargin);
    source.level({
      inputMin: sourceExtremums.min[0],
      inputMax: sourceExtremums.max[0],
      out: source,
    });

    const destinationExtremums = getMinMax(destination);
    destination.level({
      inputMin: destinationExtremums.min[0],
      inputMax: destinationExtremums.max[0],
      out: destination,
    });
  }

  // compute briefs
  const sourceBrief = getBrief(source, {
    centroidPatchDiameter,
    bestKptRadius: bestKeypointRadius,
  });
  const destinationBrief = getBrief(destination, {
    centroidPatchDiameter,
    bestKptRadius: bestKeypointRadius,
  });

  const nbSourceKeypoints = sourceBrief.keypoints.length;
  const nbDestinationKeypoints = destinationBrief.keypoints.length;

  // match reference and destination keypoints
  let matches: Match[] = [];
  if (crosscheck) {
    matches = getCrosscheckMatches(
      sourceBrief.descriptors,
      destinationBrief.descriptors,
    );
  } else {
    matches = bruteForceOneMatch(
      sourceBrief.descriptors,
      destinationBrief.descriptors,
    );

    matches = filterEuclideanDistance(
      matches,
      sourceBrief.keypoints,
      destinationBrief.keypoints,
      { origin: destinationOrigin },
    );
  }

  if (matches.length < 2) {
    throw new Error(
      'Insufficient number of matches found to compute affine transform (less than 2).',
    );
  }

  // extract source and destination points
  let sourcePoints: Point[] = [];
  let destinationPoints: Point[] = [];
  for (const match of matches) {
    sourcePoints.push(sourceBrief.keypoints[match.sourceIndex].origin);
    destinationPoints.push(
      destinationBrief.keypoints[match.destinationIndex].origin,
    );
  }

  // find inliers with ransac
  let nbInliers = sourcePoints.length;
  let nbRansacIterations = 0;
  let inliers: number[] = [0, 1];
  if (sourcePoints.length > 2) {
    const ransacResult = ransac(sourcePoints, destinationPoints, {
      distanceFunction: getEuclideanDistance,
      modelFunction: createAffineTransformModel,
      fitFunction: affineFitFunction,
      maxNbIterations: maxRansacNbIterations,
    });
    nbRansacIterations = ransacResult.nbIterations;

    inliers = ransacResult.inliers;
    nbInliers = inliers.length;

    const newSrcPoints = [];
    const newDstPoints = [];
    for (const inlier of inliers) {
      newSrcPoints.push(sourcePoints[inlier]);
      newDstPoints.push(destinationPoints[inlier]);
    }
    sourcePoints = newSrcPoints;
    destinationPoints = newDstPoints;
  }

  // create debug image
  if (debug) {
    const montage = new Montage(source, destination, {
      disposition: MontageDisposition.VERTICAL,
    });

    montage.drawMatches(
      matches,
      sourceBrief.keypoints,
      destinationBrief.keypoints,
      { showDistance: true },
    );

    const inlierMatches: Match[] = [];

    for (const inlier of inliers) {
      inlierMatches.push(matches[inlier]);
    }

    montage.drawMatches(
      inlierMatches,
      sourceBrief.keypoints,
      destinationBrief.keypoints,
      { strokeColor: [0, 0, 255] },
    );

    const drawKeypointsBaseOptions = {
      fill: true,
      color: [0, 255, 0],
      showScore: true,
      markerSize: 3,
    };

    montage.drawKeypoints(sourceBrief.keypoints, drawKeypointsBaseOptions);
    montage.drawKeypoints(destinationBrief.keypoints, {
      origin: montage.destinationOrigin,
      ...drawKeypointsBaseOptions,
    });

    writeSync(debugImagePath, montage.image);
  }

  // compute affine transform from destination to reference
  const sourceMatrix = getMatrixFromPoints(sourcePoints);
  const destinationMatrix = getMatrixFromPoints(destinationPoints);
  const affineTransform = matrixGetAffineTransform(
    sourceMatrix,
    destinationMatrix,
  );

  return {
    transform: {
      rotation: affineTransform.rotation,
      scale: affineTransform.scale,
      translation: {
        column: Math.round(affineTransform.translation.x),
        row: Math.round(affineTransform.translation.y),
      },
    },
    stats: {
      nbMatches: matches.length,
      nbInliers,
      nbRansacIterations,
      nbSourceKeypoints,
      nbDestinationKeypoints,
    },
  };
}