@gmod/cram/src/cramFile/slice/index.ts

read CRAM files with pure Javascript

import {
  CramArgumentError,
  CramBufferOverrunError,
  CramMalformedError,
} from '../../errors.ts'
import ByteArrayStopCodec from '../codecs/byteArrayStop.ts'
import ExternalCodec, {
  batchDecodeItf8,
  parseItf8,
} from '../codecs/external.ts'
import Constants from '../constants.ts'
import decodeRecord, { buildRFSchema } from './decodeRecord.ts'
import { dataSeriesTypes } from '../container/compressionScheme.ts'
import { type CramFileBlock } from '../file.ts'
import CramRecord, { defaultDecodeOptions } from '../record.ts'
import { getSectionParsers, isMappedSliceHeader } from '../sectionParsers.ts'
import { decodeUtf8, parseItem, sequenceMD5 } from '../util.ts'

import type { DecodeOptions } from '../record.ts'
import type {
  MappedSliceHeader,
  UnmappedSliceHeader,
} from '../sectionParsers.ts'
import type { BoundDecoders, BulkByteRawDecoder } from './decodeRecord.ts'
import type { Cursor, Cursors, PreDecodedIntBlock } from '../codecs/_base.ts'
import type { DataSeriesEncodingKey } from '../codecs/dataSeriesTypes.ts'
import type CramContainer from '../container/index.ts'
import type { CramEncoding } from '../encoding.ts'
import type CramFile from '../file.ts'

// shared zero-length sentinel returned by bound tag decoders when length=0
const EMPTY_BYTES = new Uint8Array(0)

export type SliceHeader = CramFileBlock & {
  parsedContent: MappedSliceHeader | UnmappedSliceHeader
}

interface RefRegion {
  id: number
  start: number
  end: number
  seq: string | null
}

/**
 * Try to estimate the template length from a bunch of interrelated
 * multi-segment reads.
 */
function calculateMultiSegmentMatedTemplateLength(
  allRecords: CramRecord[],
  _currentRecordNumber: number,
  thisRecord: CramRecord,
) {
  const matedRecords: CramRecord[] = [thisRecord]
  let cur = thisRecord
  while (cur.mateRecordNumber !== undefined && cur.mateRecordNumber >= 0) {
    const mateRecord = allRecords[cur.mateRecordNumber]
    if (!mateRecord) {
      throw new CramMalformedError(
        'intra-slice mate record not found, this file seems malformed',
      )
    }
    matedRecords.push(mateRecord)
    cur = mateRecord
  }

  let minStart = matedRecords[0]!.alignmentStart
  let maxEnd = minStart + matedRecords[0]!.readLength - 1
  for (let i = 1; i < matedRecords.length; i++) {
    const r = matedRecords[i]!
    if (r.alignmentStart < minStart) {
      minStart = r.alignmentStart
    }
    const end = r.alignmentStart + r.readLength - 1
    if (end > maxEnd) {
      maxEnd = end
    }
  }
  const estimatedTemplateLength = maxEnd - minStart + 1
  if (estimatedTemplateLength >= 0) {
    matedRecords.forEach(r => {
      if (r.templateLength !== undefined) {
        throw new CramMalformedError(
          'mate pair group has some members that have template lengths already, this file seems malformed',
        )
      }
      // sign per SAM spec: positive for leftmost, negative for rightmost
      r.templateLength =
        r.alignmentStart === minStart
          ? estimatedTemplateLength
          : -estimatedTemplateLength
    })
  }
}

/**
 * Attempt to calculate the `templateLength` for a pair of intra-slice paired
 * reads. Ported from htslib. Algorithm is imperfect.
 */
function calculateIntraSliceMatePairTemplateLength(
  thisRecord: CramRecord,
  mateRecord: CramRecord,
) {
  // this just estimates the template length by using the simple (non-gapped)
  // end coordinate of each read, because gapping in the alignment doesn't mean
  // the template is longer or shorter
  const start = Math.min(thisRecord.alignmentStart, mateRecord.alignmentStart)
  const end = Math.max(
    thisRecord.alignmentStart + thisRecord.readLength - 1,
    mateRecord.alignmentStart + mateRecord.readLength - 1,
  )
  const lengthEstimate = end - start + 1
  // sign per SAM spec: positive for leftmost, negative for rightmost
  thisRecord.templateLength =
    thisRecord.alignmentStart <= mateRecord.alignmentStart
      ? lengthEstimate
      : -lengthEstimate
  mateRecord.templateLength =
    mateRecord.alignmentStart <= thisRecord.alignmentStart
      ? lengthEstimate
      : -lengthEstimate
}

/**
 * establishes a mate-pair relationship between two records in the same slice.
 * CRAM compresses mate-pair relationships between records in the same slice
 * down into just one record having the index in the slice of its mate
 */
function associateIntraSliceMate(
  allRecords: CramRecord[],
  currentRecordNumber: number,
  thisRecord: CramRecord,
  mateRecord: CramRecord,
) {
  const complicatedMultiSegment = !!(
    mateRecord.mate ||
    (mateRecord.mateRecordNumber !== undefined &&
      mateRecord.mateRecordNumber !== currentRecordNumber)
  )

  // Deal with lossy read names — assign a synthetic name from uniqueId
  // so that paired records share the same name
  if (!thisRecord.readName) {
    const syntheticName = String(thisRecord.uniqueId)
    thisRecord._syntheticReadName = syntheticName
    mateRecord._syntheticReadName = syntheticName
  }

  thisRecord.mate = {
    sequenceId: mateRecord.sequenceId,
    alignmentStart: mateRecord.alignmentStart,
    uniqueId: mateRecord.uniqueId,
  }
  if (mateRecord.readName) {
    thisRecord.mate.readName = mateRecord.readName
  }

  // the mate record might have its own mate pointer, if this is some kind of
  // multi-segment (more than paired) scheme, so only relate that one back to this one
  // if it does not have any other relationship
  if (!mateRecord.mate && mateRecord.mateRecordNumber === undefined) {
    mateRecord.mate = {
      sequenceId: thisRecord.sequenceId,
      alignmentStart: thisRecord.alignmentStart,
      uniqueId: thisRecord.uniqueId,
    }
    if (thisRecord.readName) {
      mateRecord.mate.readName = thisRecord.readName
    }
  }

  // make sure the proper flags and cramFlags are set on both records
  // paired
  thisRecord.flags |= Constants.BAM_FPAIRED

  // set mate unmapped if needed
  if (mateRecord.flags & Constants.BAM_FUNMAP) {
    thisRecord.flags |= Constants.BAM_FMUNMAP
    // thisRecord.templateLength = 0
  }
  if (thisRecord.flags & Constants.BAM_FUNMAP) {
    // thisRecord.templateLength = 0
    mateRecord.flags |= Constants.BAM_FMUNMAP
  }

  // set mate reversed if needed
  if (mateRecord.flags & Constants.BAM_FREVERSE) {
    thisRecord.flags |= Constants.BAM_FMREVERSE
  }
  if (thisRecord.flags & Constants.BAM_FREVERSE) {
    mateRecord.flags |= Constants.BAM_FMREVERSE
  }

  if (thisRecord.templateLength === undefined) {
    if (complicatedMultiSegment) {
      calculateMultiSegmentMatedTemplateLength(
        allRecords,
        currentRecordNumber,
        thisRecord,
      )
    } else {
      calculateIntraSliceMatePairTemplateLength(thisRecord, mateRecord)
    }
  }

  // delete this last because it's used by the
  // complicated template length estimation
  thisRecord.mateRecordNumber = undefined
}

export default class CramSlice {
  private file: CramFile
  container: CramContainer
  containerPosition: number
  sliceSize: number
  private _headerResult?: ReturnType<CramSlice['_fetchHeader']>
  private _blocksResult?: ReturnType<CramSlice['_fetchBlocks']>
  private _blocksContentIdIndexResult?: ReturnType<
    CramSlice['_fetchBlocksContentIdIndex']
  >

  constructor(
    container: CramContainer,
    containerPosition: number,
    sliceSize: number,
  ) {
    this.file = container.file
    this.container = container
    this.containerPosition = containerPosition
    this.sliceSize = sliceSize
  }

  getHeader() {
    if (this._headerResult === undefined) {
      this._headerResult = this._fetchHeader()
      this._headerResult.catch(() => {
        this._headerResult = undefined
      })
    }
    return this._headerResult
  }

  private async _fetchHeader() {
    // fetch and parse the slice header
    const { majorVersion } = await this.file.getDefinition()
    const sectionParsers = getSectionParsers(majorVersion)
    const containerHeader = await this.container.getHeader()

    const header = await this.file.readBlock(
      containerHeader._endPosition + this.containerPosition,
    )
    const parser =
      header.contentType === 'MAPPED_SLICE_HEADER'
        ? sectionParsers.cramMappedSliceHeader.parser
        : header.contentType === 'UNMAPPED_SLICE_HEADER'
          ? sectionParsers.cramUnmappedSliceHeader.parser
          : undefined
    if (parser) {
      const content = parseItem(
        header.content,
        parser,
        0,
        containerHeader._endPosition,
      )
      return { ...header, parsedContent: content }
    } else {
      throw new CramMalformedError(
        `error reading slice header block, invalid content type ${header.contentType}`,
      )
    }
  }

  getBlocks() {
    if (this._blocksResult === undefined) {
      this._blocksResult = this._fetchBlocks()
      this._blocksResult.catch(() => {
        this._blocksResult = undefined
      })
    }
    return this._blocksResult
  }

  private async _fetchBlocks() {
    const header = await this.getHeader()

    if (this.sliceSize) {
      // if we know the slice size (from the index), do one big read for all
      // blocks and parse from the in-memory buffer
      const containerHeader = await this.container.getHeader()
      const sliceFilePosition =
        containerHeader._endPosition + this.containerPosition
      const blocksFilePosition = header._endPosition
      const headerSize = blocksFilePosition - sliceFilePosition
      const remainingBytes = this.sliceSize - headerSize

      const allBlocksBuffer = await this.file.read(
        remainingBytes,
        blocksFilePosition,
      )

      const blocks: CramFileBlock[] = new Array(header.parsedContent.numBlocks)
      let bufferOffset = 0
      for (let i = 0; i < blocks.length; i++) {
        const block = await this.file.readBlockFromBuffer(
          allBlocksBuffer,
          bufferOffset,
          blocksFilePosition + bufferOffset,
        )
        blocks[i] = block
        bufferOffset = block._endPosition - blocksFilePosition
      }
      return blocks
    }

    // fallback: read blocks one at a time (non-indexed access)
    let blockPosition = header._endPosition
    const blocks: CramFileBlock[] = new Array(header.parsedContent.numBlocks)
    for (let i = 0; i < blocks.length; i++) {
      const block = await this.file.readBlock(blockPosition)
      blocks[i] = block
      blockPosition = block._endPosition
    }
    return blocks
  }

  // no memoize
  async getCoreDataBlock() {
    const blocks = await this.getBlocks()
    return blocks[0]
  }

  _getBlocksContentIdIndex() {
    if (this._blocksContentIdIndexResult === undefined) {
      this._blocksContentIdIndexResult = this._fetchBlocksContentIdIndex()
      this._blocksContentIdIndexResult.catch(() => {
        this._blocksContentIdIndexResult = undefined
      })
    }
    return this._blocksContentIdIndexResult
  }

  private async _fetchBlocksContentIdIndex(): Promise<
    Record<number, CramFileBlock>
  > {
    const blocks = await this.getBlocks()
    const blocksByContentId: Record<number, CramFileBlock> = {}
    blocks.forEach(block => {
      if (block.contentType === 'EXTERNAL_DATA') {
        blocksByContentId[block.contentId] = block
      }
    })
    return blocksByContentId
  }

  async getBlockByContentId(id: number) {
    const blocksByContentId = await this._getBlocksContentIdIndex()
    return blocksByContentId[id]
  }

  async getReferenceRegion() {
    // read the slice header
    const sliceHeader = (await this.getHeader()).parsedContent
    if (!isMappedSliceHeader(sliceHeader)) {
      throw new Error('slice header not mapped')
    }

    if (sliceHeader.refSeqId < 0) {
      return undefined
    }

    const compressionScheme = await this.container.getCompressionScheme()
    if (compressionScheme === undefined) {
      throw new Error('compression scheme undefined')
    }

    if (sliceHeader.refBaseBlockId >= 0) {
      const refBlock = await this.getBlockByContentId(
        sliceHeader.refBaseBlockId,
      )
      if (!refBlock) {
        throw new CramMalformedError(
          'embedded reference specified, but reference block does not exist',
        )
      }

      // TODO: we do not read anything named 'span'
      // if (sliceHeader.span > refBlock.uncompressedSize) {
      //   throw new CramMalformedError('Embedded reference is too small')
      // }

      // TODO verify
      return {
        seq: decodeUtf8(refBlock.content),
        start: sliceHeader.refSeqStart,
        end: sliceHeader.refSeqStart + sliceHeader.refSeqSpan - 1,
        span: sliceHeader.refSeqSpan,
      }
    }
    if (
      compressionScheme.referenceRequired ||
      this.file.fetchReferenceSequenceCallback
    ) {
      if (!this.file.fetchReferenceSequenceCallback) {
        throw new Error(
          'reference sequence not embedded, and seqFetch callback not provided, cannot fetch reference sequence',
        )
      }

      const seq = await this.file.fetchReferenceSequenceCallback(
        sliceHeader.refSeqId,
        sliceHeader.refSeqStart,
        sliceHeader.refSeqStart + sliceHeader.refSeqSpan - 1,
      )

      if (seq.length !== sliceHeader.refSeqSpan) {
        throw new CramArgumentError(
          'seqFetch callback returned a reference sequence of the wrong length',
        )
      }

      return {
        seq,
        start: sliceHeader.refSeqStart,
        end: sliceHeader.refSeqStart + sliceHeader.refSeqSpan - 1,
        span: sliceHeader.refSeqSpan,
      }
    }

    return undefined
  }

  getAllRecords() {
    return this.getRecords(() => true)
  }

  async _fetchRecords(decodeOptions: Required<DecodeOptions>) {
    const { majorVersion } = await this.file.getDefinition()

    const compressionScheme = await this.container.getCompressionScheme()
    if (compressionScheme === undefined) {
      throw new Error('compression scheme undefined')
    }

    const sliceHeader = await this.getHeader()
    const blocksByContentId = await this._getBlocksContentIdIndex()

    // check MD5 of reference if available
    if (
      majorVersion > 1 &&
      this.file.options.checkSequenceMD5 &&
      isMappedSliceHeader(sliceHeader.parsedContent) &&
      sliceHeader.parsedContent.refSeqId >= 0 &&
      sliceHeader.parsedContent.md5?.join('') !== '0000000000000000'
    ) {
      const refRegion = await this.getReferenceRegion()
      if (refRegion) {
        const { seq, start, end } = refRegion
        const seqMd5 = sequenceMD5(seq)
        const storedMd5 = sliceHeader.parsedContent.md5
          ?.map(byte => (byte < 16 ? '0' : '') + byte.toString(16))
          .join('')
        if (seqMd5 !== storedMd5) {
          throw new CramMalformedError(
            `MD5 checksum reference mismatch for ref ${sliceHeader.parsedContent.refSeqId} pos ${start}..${end}. recorded MD5: ${storedMd5}, calculated MD5: ${seqMd5}`,
          )
        }
      }
    }

    // tracks the read position within the block. codec.decode() methods
    // advance the byte and bit positions in the cursor as they decode
    // data note that we are only decoding a single block here, the core
    // data block
    const coreDataBlock = await this.getCoreDataBlock()
    const externalCursorMap = new Map<number, Cursor>()
    const cursors: Cursors = {
      lastAlignmentStart: isMappedSliceHeader(sliceHeader.parsedContent)
        ? sliceHeader.parsedContent.refSeqStart
        : 0,
      coreBlock: { bitPosition: 7, bytePosition: 0 },
      externalBlocks: {
        getCursor(contentId: number) {
          let r = externalCursorMap.get(contentId)
          if (r === undefined) {
            r = { bitPosition: 7, bytePosition: 0 }
            externalCursorMap.set(contentId, r)
          }
          return r
        },
      },
    }

    // Pre-decode external int blocks: batch ITF8 decode via WASM so that
    // ExternalCodec.decode() becomes a simple array index read.
    // A block can only be pre-decoded if ALL accessors use int type.
    // If any byte-type accessor shares the same block, skip it.
    const externalIntBlockIds = new Set<number>()
    const externalByteBlockIds = new Set<number>()

    // Recurse through codec encodings to find which external block IDs are
    // used as int vs byte. codecId 1 = EXTERNAL, 4 = BYTE_ARRAY_LENGTH
    // (whose lengths sub-codec is int, values sub-codec is byte),
    // 5 = BYTE_ARRAY_STOP (always byte).
    function collectExternalBlockIds(
      enc: CramEncoding | undefined,
      isInt: boolean,
    ) {
      if (!enc) {
        return
      }
      if (enc.codecId === 1) {
        if (isInt) {
          externalIntBlockIds.add(enc.parameters.blockContentId)
        } else {
          externalByteBlockIds.add(enc.parameters.blockContentId)
        }
      } else if (enc.codecId === 4) {
        collectExternalBlockIds(enc.parameters.lengthsEncoding, true)
        collectExternalBlockIds(enc.parameters.valuesEncoding, false)
      } else if (enc.codecId === 5) {
        externalByteBlockIds.add(enc.parameters.blockContentId)
      }
    }

    for (const [ds, enc] of Object.entries(
      compressionScheme.dataSeriesEncoding,
    )) {
      const dsType = dataSeriesTypes[ds as keyof typeof dataSeriesTypes]
      collectExternalBlockIds(enc, dsType === 'int')
    }
    for (const tagEnc of Object.values(compressionScheme.tagEncoding)) {
      collectExternalBlockIds(tagEnc, false)
    }

    // Remove any int block that is also used as byte
    for (const id of externalByteBlockIds) {
      externalIntBlockIds.delete(id)
    }

    const preDecodedIntBlocks = new Map<number, PreDecodedIntBlock>()
    for (const contentId of externalIntBlockIds) {
      const block = blocksByContentId[contentId]
      if (block?.content.length) {
        const values = batchDecodeItf8(block.content)
        preDecodedIntBlocks.set(contentId, { values, index: 0 })
      }
    }
    cursors.preDecodedIntBlocks = preDecodedIntBlocks

    // Build bound decode functions per data series. For ExternalCodec this
    // captures the content buffer and cursor directly, eliminating per-call
    // Record/Map lookup overhead. The bound decoders are assembled into a
    // single object literal with all data series present so V8 sees a stable
    // hidden class — call sites in decodeRecord then become direct property
    // accesses with monomorphic inline caches.
    const bind = (dataSeriesName: string) => {
      const codec = compressionScheme.getCodecForDataSeries(
        dataSeriesName as DataSeriesEncodingKey,
      )
      if (!codec) {
        return () => {
          throw new CramMalformedError(
            `no codec defined for ${dataSeriesName} data series`,
          )
        }
      }
      if (codec instanceof ExternalCodec) {
        const bid = codec.parameters.blockContentId
        const preDecoded = preDecodedIntBlocks.get(bid)
        if (preDecoded) {
          const { values } = preDecoded
          return () => values[preDecoded.index++]
        }
        const contentBlock = blocksByContentId[bid]
        if (!contentBlock) {
          return () => {
            throw new CramMalformedError(
              `no block found with content ID ${bid}`,
            )
          }
        }
        const cursor = cursors.externalBlocks.getCursor(bid)
        const content = contentBlock.content
        if (codec.dataType === 'int') {
          return () => parseItf8(content, cursor)
        }
        // Mirror the bounds check in ExternalCodec.decode — without it,
        // a truncated/corrupt block silently yields `undefined` for byte
        // reads, which downstream propagates as NaN/0 (silent data
        // corruption) rather than a clear error.
        return () => {
          if (cursor.bytePosition >= content.length) {
            throw new CramBufferOverrunError(
              'attempted to read beyond end of block. this file seems truncated.',
            )
          }
          return content[cursor.bytePosition++]
        }
      }
      if (codec instanceof ByteArrayStopCodec) {
        const { blockContentId, stopByte } = codec.parameters
        const contentBlock = blocksByContentId[blockContentId]
        if (!contentBlock) {
          return () => {
            throw new CramMalformedError(
              `no block found with content ID ${blockContentId}`,
            )
          }
        }
        const content = contentBlock.content
        const cursor = cursors.externalBlocks.getCursor(blockContentId)
        return () => {
          const start = cursor.bytePosition
          const len = content.length
          let pos = start
          while (pos < len && content[pos] !== stopByte) {
            pos++
          }
          if (pos >= len) {
            throw new CramBufferOverrunError(
              'byteArrayStop reading beyond length of data buffer?',
            )
          }
          cursor.bytePosition = pos + 1
          return content.subarray(start, pos)
        }
      }
      return () =>
        codec.decode(this, coreDataBlock!, blocksByContentId, cursors)
    }

    const bd: BoundDecoders = {
      BF: bind('BF'),
      CF: bind('CF'),
      RI: bind('RI'),
      RL: bind('RL'),
      AP: bind('AP'),
      RG: bind('RG'),
      RN: bind('RN'),
      MF: bind('MF'),
      NS: bind('NS'),
      NP: bind('NP'),
      TS: bind('TS'),
      NF: bind('NF'),
      TL: bind('TL'),
      FN: bind('FN'),
      FC: bind('FC'),
      FP: bind('FP'),
      DL: bind('DL'),
      BB: bind('BB'),
      QQ: bind('QQ'),
      BS: bind('BS'),
      IN: bind('IN'),
      RS: bind('RS'),
      PD: bind('PD'),
      HC: bind('HC'),
      SC: bind('SC'),
      MQ: bind('MQ'),
      BA: bind('BA'),
      QS: bind('QS'),
      TC: bind('TC'),
      TN: bind('TN'),
    } as BoundDecoders

    // Bulk byte decoder for QS and BA — getBytesSubarray returns a subarray
    // view when the codec supports it (e.g. ExternalCodec), or undefined otherwise
    const qsCodec = compressionScheme.getCodecForDataSeries('QS')
    const baCodec = compressionScheme.getCodecForDataSeries('BA')
    const decodeBulkBytesRaw: BulkByteRawDecoder | undefined =
      qsCodec || baCodec
        ? (dataSeriesName, length) => {
            const codec = dataSeriesName === 'QS' ? qsCodec : baCodec
            return codec?.getBytesSubarray(blocksByContentId, cursors, length)
          }
        : undefined

    // Bound tag decoders — tags are typically encoded as byteArrayLength
    // (codecId=4) wrapping External-int lengths + External-byte values. We
    // build a fast closure per tagId that inlines the length read and value
    // subarray, eliminating per-call dispatch through ByteArrayLengthCodec
    // and the inner codecs. Other encodings fall back to the generic dispatch.
    const boundTagDecoders: Record<
      string,
      () => Uint8Array | number | undefined
    > = {}
    const bindTagFallback = (tagId: string) => {
      const codec = compressionScheme.getCodecForTag(tagId)
      return () =>
        codec.decode(this, coreDataBlock!, blocksByContentId, cursors)
    }
    for (const tagId of Object.keys(compressionScheme.tagEncoding)) {
      const enc = compressionScheme.tagEncoding[tagId]!
      if (
        enc.codecId === 4 &&
        enc.parameters.lengthsEncoding.codecId === 1 &&
        enc.parameters.valuesEncoding.codecId === 1
      ) {
        const lenBid = enc.parameters.lengthsEncoding.parameters.blockContentId
        const valBid = enc.parameters.valuesEncoding.parameters.blockContentId
        const lenContentBlock = blocksByContentId[lenBid]
        const valContentBlock = blocksByContentId[valBid]
        if (!lenContentBlock || !valContentBlock) {
          boundTagDecoders[tagId] = bindTagFallback(tagId)
          continue
        }
        const valContent = valContentBlock.content
        const valCursor = cursors.externalBlocks.getCursor(valBid)
        const lenPreDecoded = preDecodedIntBlocks.get(lenBid)
        const lenContent = lenContentBlock.content
        const lenCursor = cursors.externalBlocks.getCursor(lenBid)
        const readTagLen = lenPreDecoded
          ? () => lenPreDecoded.values[lenPreDecoded.index++]!
          : () => parseItf8(lenContent, lenCursor)
        boundTagDecoders[tagId] = () => {
          const length = readTagLen()
          if (length === 0) {
            return EMPTY_BYTES
          }
          const start = valCursor.bytePosition
          const end = start + length
          if (end > valContent.length) {
            throw new CramBufferOverrunError(
              'attempted to read beyond end of block. this file seems truncated.',
            )
          }
          valCursor.bytePosition = end
          return valContent.subarray(start, end)
        }
      } else {
        boundTagDecoders[tagId] = bindTagFallback(tagId)
      }
    }

    const records: CramRecord[] = new Array(
      sliceHeader.parsedContent.numRecords,
    )
    const rfSchema = buildRFSchema(bd, majorVersion)
    for (let i = 0; i < records.length; i += 1) {
      try {
        records[i] = new CramRecord(
          decodeRecord(
            this,
            bd,
            rfSchema,
            boundTagDecoders,
            compressionScheme,
            sliceHeader,
            coreDataBlock!,
            blocksByContentId,
            cursors,
            majorVersion,
            i,
            sliceHeader.contentPosition +
              sliceHeader.parsedContent.recordCounter +
              i +
              1,
            decodeOptions,
            decodeBulkBytesRaw,
          ),
        )
      } catch (e) {
        const err = e as { code?: string; message?: string }
        if (err.code === 'CRAM_BUFFER_OVERRUN') {
          const recordsDecoded = i
          const recordsExpected = sliceHeader.parsedContent.numRecords
          throw new CramMalformedError(
            `Failed to decode all records in slice. Decoded ${recordsDecoded} of ${recordsExpected} expected records. ` +
              `Buffer overrun suggests either: (1) file is truncated/corrupted, (2) compression scheme is incorrect, ` +
              `or (3) there's a bug in the decoder. Original error: ${err.message}`,
          )
        } else {
          throw e
        }
      }
    }

    // interpret `recordsToNextFragment` attributes to make standard `mate`
    // objects. The records loop above fills every slot or throws — by the
    // time we get here, records[i] is always defined. The records[mate]
    // guard protects against malformed mateRecordNumber pointing past the
    // slice.
    for (let i = 0; i < records.length; i += 1) {
      const r = records[i]!
      const { mateRecordNumber } = r
      if (
        mateRecordNumber !== undefined &&
        mateRecordNumber >= 0 &&
        records[mateRecordNumber]
      ) {
        associateIntraSliceMate(records, i, r, records[mateRecordNumber])
      }
    }

    return records
  }

  async getRecords(
    filterFunction: (r: CramRecord) => boolean,
    decodeOptions?: DecodeOptions,
  ) {
    // Merge with defaults
    const opts = { ...defaultDecodeOptions, ...decodeOptions }

    // fetch the features if necessary, using the file-level feature cache
    // Include decode options in cache key so different decode configs are cached separately
    const optionsKey = `${opts.decodeTags ? 1 : 0}`
    const cacheKey = `${this.container.filePosition}:${this.containerPosition}:${optionsKey}`
    let recordsPromise = this.file.featureCache.get(cacheKey)
    if (!recordsPromise) {
      recordsPromise = this._fetchRecords(opts)
      this.file.featureCache.set(cacheKey, recordsPromise)
    }

    const unfiltered = await recordsPromise
    const records = unfiltered.filter(filterFunction)

    // if we can fetch reference sequence, add the reference sequence to the records
    if (records.length && this.file.fetchReferenceSequenceCallback) {
      const sliceHeader = await this.getHeader()
      if (
        isMappedSliceHeader(sliceHeader.parsedContent) &&
        (sliceHeader.parsedContent.refSeqId >= 0 || // single-ref slice
          sliceHeader.parsedContent.refSeqId === -2) // multi-ref slice
      ) {
        const singleRefId =
          sliceHeader.parsedContent.refSeqId >= 0
            ? sliceHeader.parsedContent.refSeqId
            : undefined
        const compressionScheme = await this.container.getCompressionScheme()
        if (compressionScheme === undefined) {
          throw new Error('compression scheme undefined')
        }
        const refRegions: Record<string, RefRegion> = {}

        // iterate over the records to find the spans of the reference
        // sequences we need to fetch
        for (const record of records) {
          const seqId =
            singleRefId !== undefined ? singleRefId : record.sequenceId
          let refRegion = refRegions[seqId]
          if (!refRegion) {
            refRegion = {
              id: seqId,
              start: record.alignmentStart,
              end: Number.NEGATIVE_INFINITY,
              seq: null,
            }
            refRegions[seqId] = refRegion
          }

          const end =
            record.alignmentStart +
            (record.lengthOnRef || record.readLength) -
            1
          if (end > refRegion.end) {
            refRegion.end = end
          }
          if (record.alignmentStart < refRegion.start) {
            refRegion.start = record.alignmentStart
          }
        }

        // fetch the `seq` for all of the ref regions
        await Promise.all(
          Object.values(refRegions).map(async refRegion => {
            if (
              refRegion.id !== -1 &&
              refRegion.start <= refRegion.end &&
              this.file.fetchReferenceSequenceCallback
            ) {
              refRegion.seq = await this.file.fetchReferenceSequenceCallback(
                refRegion.id,
                refRegion.start,
                refRegion.end,
              )
            }
          }),
        )

        // now decorate all the records with them
        for (const record of records) {
          const seqId =
            singleRefId !== undefined ? singleRefId : record.sequenceId
          const refRegion = refRegions[seqId]
          if (refRegion?.seq) {
            const seq = refRegion.seq
            record.addReferenceSequence(
              { ...refRegion, seq },
              compressionScheme,
            )
          }
        }
      }
    }

    return records
  }
}