dicom-microscopy-viewer/src/pyramid.js

Interactive web-based viewer for DICOM Microscopy Images

import * as dwc from 'dicomweb-client'

import { _decodeAndTransformFrame } from './decode.js'
import publish from './eventPublisher'
import EVENT from './events'
import { getFrameMapping, VLWholeSlideMicroscopyImage } from './metadata.js'
import { getPixelSpacing } from './scoord3dUtils'
import {
  _fetchBulkdata,
  are1DArraysAlmostEqual,
  are2DArraysAlmostEqual,
} from './utils.js'

/**
 * Get Image ICC profiles.
 *
 * @param {Array<metadata.VLWholeSlideMicroscopyImage>} pyramid - Metadata of
 * VL Whole Slide Microscopy Image instances
 * @param {object} options - options object
 * @param {object} options.metadata - metadata of VL Whole Slide Microscopy Image instances
 * @param {object} options.client - dicom web client
 * @param {function} options.onError - function to call when an error occurs
 *
 * @returns {Promise<Array<TypedArray>>} image array with ICC profiles (only for images with SamplesPerPixel === 3 and ICCProfile present)
 *
 * @private
 */
async function _getIccProfiles({ metadata, client, onError }) {
  const fetchPromises = metadata.map((image) => {
    if (image.SamplesPerPixel === 3) {
      let iccProfile = false
      const metadataItem = image.OpticalPathSequence[0]
      if (metadataItem.ICCProfile == null) {
        if ('OpticalPathSequence' in image.bulkdataReferences) {
          const bulkdataItem = image.bulkdataReferences.OpticalPathSequence[0]
          if ('ICCProfile' in bulkdataItem) {
            iccProfile = bulkdataItem.ICCProfile
          }
        }
      } else {
        iccProfile = metadataItem.ICCProfile
      }
      if (!iccProfile) {
        console.warn(
          `ICC Profile was not found for image "${image.SOPInstanceUID}"`,
        )
        return null
      } else if ('BulkDataURI' in iccProfile) {
        console.debug(
          `fetching ICC Profile for image "${image.SOPInstanceUID}"`,
          iccProfile,
        )
        return _fetchBulkdata({
          client,
          reference: iccProfile,
        }).catch(onError)
      } else {
        return iccProfile
      }
    }
    return null
  })
  const validPromises = fetchPromises.filter(Boolean)
  const results = await Promise.allSettled(validPromises)
  return results
    .filter((result) => result.status === 'fulfilled' && result.value != null)
    .map((result) => result.value)
}

/**
 * Compute image pyramid.
 *
 * @param {object[]} metadata - Metadata of VL Whole Slide Microscopy Image instances
 * @returns {object} Information about the image pyramid
 *
 * @private
 */
function _computeImagePyramid({ metadata }) {
  if (metadata.length === 0) {
    throw new Error(
      'No image metadata was provided to computate image pyramid structure.',
    )
  }

  // Sort instances and optionally concatenation parts if present.
  metadata.sort((a, b) => {
    const sizeDiff = a.TotalPixelMatrixColumns - b.TotalPixelMatrixColumns
    if (sizeDiff === 0) {
      if (a.ConcatenationFrameOffsetNumber !== undefined) {
        return (
          a.ConcatenationFrameOffsetNumber - b.ConcatenationFrameOffsetNumber
        )
      }
      return sizeDiff
    }
    return sizeDiff
  })

  const pyramidMetadata = []
  const pyramidFrameMappings = []
  let pyramidNumberOfChannels
  for (let i = 0; i < metadata.length; i++) {
    if (metadata[0].FrameOfReferenceUID !== metadata[i].FrameOfReferenceUID) {
      throw new Error(
        'Images of pyramid must all have the same Frame of Reference UID.',
      )
    }
    if (metadata[0].ContainerIdentifier !== metadata[i].ContainerIdentifier) {
      throw new Error(
        'Images of pyramid must all have the same Container Identifier.',
      )
    }

    const numberOfFrames = Number(metadata[i].NumberOfFrames || 1)
    const cols = metadata[i].TotalPixelMatrixColumns || metadata[i].Columns
    const rows = metadata[i].TotalPixelMatrixRows || metadata[i].Rows

    const { frameMapping, numberOfChannels } = getFrameMapping(metadata[i])
    if (i > 0) {
      if (pyramidNumberOfChannels !== numberOfChannels) {
        throw new Error(
          'Images of pyramid must all have the same number of channels ' +
            '(optical paths, segments, mappings, etc.)',
        )
      }
    } else {
      pyramidNumberOfChannels = numberOfChannels
    }

    /*
     * Instances may be broken down into multiple concatentation parts.
     * Therefore, we have to re-assemble instance metadata.
     */
    let alreadyExists = false
    let index = null
    for (let j = 0; j < pyramidMetadata.length; j++) {
      const c =
        pyramidMetadata[j].TotalPixelMatrixColumns || pyramidMetadata[j].Columns
      const r =
        pyramidMetadata[j].TotalPixelMatrixRows || pyramidMetadata[j].Rows
      if (r === rows && c === cols) {
        alreadyExists = true
        index = j
      }
    }
    if (alreadyExists) {
      Object.assign(pyramidFrameMappings[index], frameMapping)
      /*
       * Create a new SOP Instance with metadata updated from current
       * concatentation part.
       */
      const rawMetadata = pyramidMetadata[index].json
      rawMetadata['00280008'].Value[0] += numberOfFrames
      if ('PerFrameFunctionalGroupsSequence' in metadata[index]) {
        rawMetadata['52009230'].Value.push(
          ...metadata[index].PerFrameFunctionalGroupsSequence,
        )
      }
      if (!('SOPInstanceUIDOfConcatenationSource' in metadata[i])) {
        throw new Error(
          'Multiple image instances for the same channel and ' +
            'focal plane have identical dimensions, but the instances ' +
            'are not part of a concatenation either. ' +
            'The image metadata is probably incorrect.',
        )
      }
      const sopInstanceUID = metadata[i].SOPInstanceUIDOfConcatenationSource
      rawMetadata['00080018'].Value[0] = sopInstanceUID
      delete rawMetadata['00200242'] // SOPInstanceUIDOfConcatenationSource
      delete rawMetadata['00209161'] // ConcatentationUID
      delete rawMetadata['00209162'] // InConcatenationNumber
      delete rawMetadata['00209228'] // ConcatenationFrameOffsetNumber
      pyramidMetadata[index] = new VLWholeSlideMicroscopyImage({
        metadata: rawMetadata,
      })
    } else {
      pyramidMetadata.push(metadata[i])
      pyramidFrameMappings.push(frameMapping)
    }
  }

  const nLevels = pyramidMetadata.length
  if (nLevels === 0) {
    console.error('empty pyramid - no levels found')
  }
  const pyramidBaseMetadata = pyramidMetadata[nLevels - 1]

  /*
   * Collect relevant information from DICOM metadata for each pyramid
   * level to construct the Openlayers map.
   */
  const pyramidTileSizes = []
  const pyramidGridSizes = []
  const pyramidResolutions = []
  const pyramidOrigins = []
  const pyramidPixelSpacings = []
  const offset = [0, -1]
  const baseTotalPixelMatrixColumns =
    pyramidBaseMetadata.TotalPixelMatrixColumns
  const baseTotalPixelMatrixRows = pyramidBaseMetadata.TotalPixelMatrixRows
  for (let j = nLevels - 1; j >= 0; j--) {
    const columns = pyramidMetadata[j].Columns
    const rows = pyramidMetadata[j].Rows
    const totalPixelMatrixColumns = pyramidMetadata[j].TotalPixelMatrixColumns
    const totalPixelMatrixRows = pyramidMetadata[j].TotalPixelMatrixRows
    const pixelSpacing = getPixelSpacing(pyramidMetadata[j])
    const nColumns = Math.ceil(totalPixelMatrixColumns / columns)
    const nRows = Math.ceil(totalPixelMatrixRows / rows)
    pyramidTileSizes.push([columns, rows])
    pyramidGridSizes.push([nColumns, nRows])
    pyramidPixelSpacings.push(pixelSpacing)

    let zoomFactor = baseTotalPixelMatrixColumns / totalPixelMatrixColumns
    const roundedZoomFactor = Math.round(zoomFactor)
    /*
     * Compute the resolution at each pyramid level, since the zoom
     * factor may not be the same between adjacent pyramid levels.
     *
     * Round is conditional to avoid openlayers resolutions error.
     * The resolutions array should be composed of unique values in descending order.
     */
    if (pyramidResolutions.includes(roundedZoomFactor)) {
      console.warn(
        'resolution conflict rounding zoom factor (baseTotalPixelMatrixColumns / totalPixelMatrixColumns): ',
        zoomFactor,
      )
      zoomFactor = parseFloat(zoomFactor.toFixed(2))
    } else {
      zoomFactor = roundedZoomFactor
    }
    pyramidResolutions.push(zoomFactor)
    pyramidOrigins.push(offset)
  }
  pyramidResolutions.reverse()
  pyramidTileSizes.reverse()
  pyramidGridSizes.reverse()
  pyramidOrigins.reverse()
  pyramidPixelSpacings.reverse()

  /**
   * Frames may extend beyond the size of the total pixel matrix.
   * The excess pixels may contain garbage and should not be displayed.
   * We set the extent to the size of the actual image without taken
   * excess pixels into account.
   * Note that the vertical axis is flipped in the used tile source,
   * i.e., values on the axis lie in the range [-n, -1], where n is the
   * number of rows in the total pixel matrix.
   */
  const extent = [
    0, // min X
    -(baseTotalPixelMatrixRows + 1), // min Y
    baseTotalPixelMatrixColumns, // max X
    -1, // max Y
  ]

  return {
    extent,
    origins: pyramidOrigins,
    resolutions: pyramidResolutions,
    gridSizes: pyramidGridSizes,
    tileSizes: pyramidTileSizes,
    pixelSpacings: pyramidPixelSpacings,
    metadata: pyramidMetadata,
    frameMappings: pyramidFrameMappings,
    numberOfChannels: pyramidNumberOfChannels,
  }
}

function _areImagePyramidsEqual(pyramid, refPyramid) {
  // Check that all the channels have the same pyramid parameters
  if (!are1DArraysAlmostEqual(pyramid.extent, refPyramid.extent)) {
    console.warn(
      'pyramid has different extent as reference pyramid: ',
      pyramid.extent,
      refPyramid.extent,
    )
    return false
  }
  if (!are2DArraysAlmostEqual(pyramid.origins, refPyramid.origins)) {
    console.warn(
      'pyramid has different origins as reference pyramid: ',
      pyramid.origins,
      refPyramid.origins,
    )
    return false
  }
  if (!are1DArraysAlmostEqual(pyramid.resolutions, refPyramid.resolutions)) {
    console.warn(
      'pyramid has different resolutions as reference pyramid: ',
      pyramid.resolutions,
      refPyramid.resolutions,
    )
    return false
  }
  if (!are2DArraysAlmostEqual(pyramid.gridSizes, refPyramid.gridSizes)) {
    console.warn(
      'pyramid has different grid sizes as reference pyramid: ',
      pyramid.gridSizes,
      refPyramid.gridSizes,
    )
    return false
  }
  if (!are2DArraysAlmostEqual(pyramid.tileSizes, refPyramid.tileSizes)) {
    console.warn(
      'pyramid has different tile sizes as reference pyramid: ',
      pyramid.tileSizes,
      refPyramid.tileSizes,
    )
    return false
  }
  if (
    !are2DArraysAlmostEqual(pyramid.pixelSpacings, refPyramid.pixelSpacings)
  ) {
    console.warn(
      'pyramid has different pixel spacings as reference pyramid: ',
      pyramid.pixelSpacings,
      refPyramid.pixelSpacings,
    )
    return false
  }
  return true
}

function _createEmptyTile({
  columns,
  rows,
  samplesPerPixel,
  bitsAllocated,
  photometricInterpretation,
}) {
  let pixelArray
  if (bitsAllocated <= 8) {
    pixelArray = new Uint8Array(columns * rows * samplesPerPixel)
  } else {
    pixelArray = new Float32Array(columns * rows * samplesPerPixel)
  }

  // Fill white in case of color and black in case of monochrome.
  let fillValue = 2 ** bitsAllocated - 1
  if (photometricInterpretation === 'MONOCHROME2') {
    if (bitsAllocated <= 16) {
      fillValue = 0
    } else {
      // Float pixel data
      fillValue = -(2 ** bitsAllocated - 1) / 2
    }
  }
  for (let i = 0; i < pixelArray.length; i++) {
    pixelArray[i] = fillValue
  }
  return pixelArray
}

function _createTileLoadFunction({
  pyramid,
  client,
  channel,
  iccProfiles,
  targetElement,
}) {
  return async (z, y, x) => {
    let index = `${x + 1}-${y + 1}`
    index += `-${channel}`

    if (pyramid.metadata[z] === undefined) {
      throw new Error(
        `Could not load tile for channel "${channel}" ` +
          `at position (${x + 1}, ${y + 1}) at zoom level ${z} ` +
          ` because level ${z} does not exist.`,
      )
    }

    const studyInstanceUID = pyramid.metadata[z].StudyInstanceUID
    const seriesInstanceUID = pyramid.metadata[z].SeriesInstanceUID
    const path = pyramid.frameMappings[z][index]
    let src
    if (path != null) {
      src = ''
      if (client.wadoURL !== undefined) {
        src += client.wadoURL
      }
      src +=
        '/studies/' +
        studyInstanceUID +
        '/series/' +
        seriesInstanceUID +
        '/instances/' +
        path
    }

    const refImage = pyramid.metadata[z]
    const columns = refImage.Columns
    const rows = refImage.Rows
    const bitsAllocated = refImage.BitsAllocated
    const pixelRepresentation = refImage.PixelRepresentation
    const samplesPerPixel = refImage.SamplesPerPixel
    const photometricInterpretation = refImage.PhotometricInterpretation
    const sopClassUID = refImage.SOPClassUID

    if (src != null) {
      const sopInstanceUID = dwc.utils.getSOPInstanceUIDFromUri(src)
      const frameNumbers = dwc.utils.getFrameNumbersFromUri(src)

      if (samplesPerPixel === 1) {
        console.info(
          `retrieve frame ${frameNumbers} of monochrome image ` +
            `for channel "${channel}" at tile position (${x + 1}, ${y + 1}) ` +
            `at zoom level ${z}`,
        )
      } else {
        console.info(
          `retrieve frame ${frameNumbers} of color image ` +
            `at tile position (${x + 1}, ${y + 1}) at zoom level ${z}`,
        )
      }

      const octetStreamMediaType = 'application/octet-stream'
      /*
       * Use of the "*" transfer syntax is a hack to work around standard
       * compliance issues of the Google Cloud Healthcare API.
       * It will return bulkdata encoded with the transfer syntax of the
       * stored data set (uncompressed or compressed). The decoder can then not
       * rely on the media type specified by the "Content-Type" header in the
       * response message, but will need to determine it from the payload.
       * Only application/octet-stream with "*" is requested here; decoders
       * determine the actual compression format (e.g. JPEG, JPEG-LS, JPEG 2000)
       * from the payload when processing the frames.
       */
      const octetStreamTransferSyntaxUID = '*'

      const mediaTypes = []
      mediaTypes.push(
        ...[
          {
            mediaType: octetStreamMediaType,
            transferSyntaxUID: octetStreamTransferSyntaxUID,
          },
        ],
      )

      const frameInfo = {
        studyInstanceUID,
        seriesInstanceUID,
        sopInstanceUID,
        sopClassUID,
        frameNumber: frameNumbers[0],
        channelIdentifier: String(channel),
      }
      publish(targetElement, EVENT.FRAME_LOADING_STARTED, frameInfo)

      const retrieveOptions = {
        studyInstanceUID,
        seriesInstanceUID,
        sopInstanceUID,
        frameNumbers,
        mediaTypes,
      }
      return client
        .retrieveInstanceFrames(retrieveOptions)
        .then((rawFrames) => {
          return _decodeAndTransformFrame({
            frame: rawFrames[0],
            frameNumber: frameNumbers[0],
            bitsAllocated,
            pixelRepresentation,
            columns,
            rows,
            samplesPerPixel,
            sopInstanceUID,
            metadata: pyramid.metadata,
            iccProfiles,
          }).then((pixelArray) => {
            if (pixelArray.constructor === Float64Array) {
              // TODO: handle Float64Array using LUT
              throw new Error('Double Float Pixel Data is not (yet) supported.')
            }
            publish(targetElement, EVENT.FRAME_LOADING_ENDED, {
              pixelArray,
              ...frameInfo,
            })
            if (samplesPerPixel === 3 && bitsAllocated === 8) {
              // Rendering of color images requires unsigned 8-bit integers
              return pixelArray
            }
            // Rendering of grayscale images requires floating point values
            return new Float32Array(
              pixelArray,
              pixelArray.byteOffset,
              pixelArray.byteLength / pixelArray.BYTES_PER_ELEMENT,
            )
          })
        })
        .catch((error) => {
          publish(targetElement, EVENT.FRAME_LOADING_ENDED, frameInfo)
          publish(targetElement, EVENT.FRAME_LOADING_ERROR, frameInfo)
          return Promise.reject(
            new Error(
              `Failed to load frames ${frameNumbers} ` +
                `of SOP instance "${sopInstanceUID}" ` +
                `for channel "${channel}" ` +
                `at tile position (${x + 1}, ${y + 1}) ` +
                `at zoom level ${z}: `,
              error,
            ),
          )
        })
    } else {
      console.warn(
        `could not load tile "${index}" at level ${z}, ` +
          'this tile does not exist',
      )
      return _createEmptyTile({
        columns,
        rows,
        samplesPerPixel,
        bitsAllocated,
        photometricInterpretation,
      })
    }
  }
}

function _fitImagePyramid(pyramid, refPyramid) {
  /** Get the matching levels between the two pyramids */
  const matchingLevelIndices = []
  for (let i = 0; i < refPyramid.metadata.length; i++) {
    for (let j = 0; j < pyramid.metadata.length; j++) {
      const doOriginsMatch = are1DArraysAlmostEqual(
        refPyramid.origins[i],
        pyramid.origins[j],
      )
      const doPixelSpacingsMatch = are1DArraysAlmostEqual(
        refPyramid.pixelSpacings[i],
        pyramid.pixelSpacings[j],
      )
      if (doOriginsMatch && doPixelSpacingsMatch) {
        matchingLevelIndices.push([i, j])
      }
    }
  }

  /** Create a new pyramid that fits the reference pyramid */
  const fittedPyramid = {
    extent: [...refPyramid.extent],
    origins: [],
    resolutions: [],
    gridSizes: [],
    tileSizes: [],
    pixelSpacings: [],
    metadata: [],
    frameMappings: [],
  }

  if (matchingLevelIndices.length === 0) {
    console.warn(
      'No matching pyramid levels found, handling fixed pixel spacing case...',
    )

    const refBaseLevel = refPyramid.metadata[refPyramid.metadata.length - 1]

    for (let j = 0; j < pyramid.metadata.length; j++) {
      const segmentation = pyramid.metadata[j]
      const refBasePixelSpacing = getPixelSpacing(refBaseLevel)
      const segPixelSpacing = getPixelSpacing(segmentation)

      /** Calculate resolution based on ratio of pixel spacings */
      const resolution = segPixelSpacing[0] / refBasePixelSpacing[0]
      const roundedResolution = Math.round(resolution)

      /** Handle resolution conflicts similar to _computeImagePyramid */
      const finalResolution = fittedPyramid.resolutions.includes(
        roundedResolution,
      )
        ? parseFloat(resolution.toFixed(2))
        : roundedResolution

      fittedPyramid.origins.push([...pyramid.origins[j]])
      fittedPyramid.gridSizes.push([...pyramid.gridSizes[j]])
      fittedPyramid.tileSizes.push([...pyramid.tileSizes[j]])
      fittedPyramid.resolutions.push(finalResolution)
      fittedPyramid.pixelSpacings.push([...pyramid.pixelSpacings[j]])
      fittedPyramid.metadata.push(pyramid.metadata[j])
      fittedPyramid.frameMappings.push(pyramid.frameMappings[j])
    }
  } else {
    /**
     * Fit the pyramid levels to the reference image pyramid.
     * Use the matching levels found in the matchingLevelIndices array.
     */
    for (let i = 0; i < refPyramid.metadata.length; i++) {
      const index = matchingLevelIndices.find((element) => element[0] === i)
      if (index) {
        const j = index[1]
        fittedPyramid.origins.push([...pyramid.origins[j]])
        fittedPyramid.gridSizes.push([...pyramid.gridSizes[j]])
        fittedPyramid.tileSizes.push([...pyramid.tileSizes[j]])
        fittedPyramid.resolutions.push(refPyramid.resolutions[i])
        fittedPyramid.pixelSpacings.push([...pyramid.pixelSpacings[j]])
        fittedPyramid.metadata.push(pyramid.metadata[j])
        fittedPyramid.frameMappings.push(pyramid.frameMappings[j])
      }
    }
  }

  let minZoom = 0
  for (let i = 0; i < refPyramid.resolutions.length; i++) {
    for (let j = 0; j < fittedPyramid.resolutions.length; j++) {
      if (refPyramid.resolutions[i] === fittedPyramid.resolutions[j]) {
        minZoom = i
        break
      }
    }
  }
  let maxZoom = refPyramid.resolutions.length - 1
  for (let i = refPyramid.resolutions.length - 1; i >= minZoom; i--) {
    for (let j = fittedPyramid.resolutions.length - 1; j >= 0; j--) {
      if (refPyramid.resolutions[i] === fittedPyramid.resolutions[j]) {
        maxZoom = i
        break
      }
    }
  }

  const hasMatchingLevels = matchingLevelIndices.length > 0

  return [fittedPyramid, minZoom, maxZoom, hasMatchingLevels]
}

export {
  _areImagePyramidsEqual,
  _computeImagePyramid,
  _createTileLoadFunction,
  _fitImagePyramid,
  _getIccProfiles,
}