UNPKG

@gmod/cram

Version:

read CRAM files with pure Javascript

673 lines 31.7 kB
import { CramArgumentError, CramBufferOverrunError, CramMalformedError, } from "../../errors.js"; import ByteArrayStopCodec from "../codecs/byteArrayStop.js"; import ExternalCodec, { batchDecodeItf8, parseItf8, } from "../codecs/external.js"; import Constants from "../constants.js"; import decodeRecord, { buildRFSchema } from "./decodeRecord.js"; import { dataSeriesTypes } from "../container/compressionScheme.js"; import CramRecord, { defaultDecodeOptions } from "../record.js"; import { getSectionParsers, isMappedSliceHeader } from "../sectionParsers.js"; import { decodeUtf8, parseItem, sequenceMD5 } from "../util.js"; // shared zero-length sentinel returned by bound tag decoders when length=0 const EMPTY_BYTES = new Uint8Array(0); /** * Try to estimate the template length from a bunch of interrelated * multi-segment reads. */ function calculateMultiSegmentMatedTemplateLength(allRecords, _currentRecordNumber, thisRecord) { const matedRecords = [thisRecord]; let cur = thisRecord; while (cur.mateRecordNumber !== undefined && cur.mateRecordNumber >= 0) { const mateRecord = allRecords[cur.mateRecordNumber]; if (!mateRecord) { throw new CramMalformedError('intra-slice mate record not found, this file seems malformed'); } matedRecords.push(mateRecord); cur = mateRecord; } let minStart = matedRecords[0].alignmentStart; let maxEnd = minStart + matedRecords[0].readLength - 1; for (let i = 1; i < matedRecords.length; i++) { const r = matedRecords[i]; if (r.alignmentStart < minStart) { minStart = r.alignmentStart; } const end = r.alignmentStart + r.readLength - 1; if (end > maxEnd) { maxEnd = end; } } const estimatedTemplateLength = maxEnd - minStart + 1; if (estimatedTemplateLength >= 0) { matedRecords.forEach(r => { if (r.templateLength !== undefined) { throw new CramMalformedError('mate pair group has some members that have template lengths already, this file seems malformed'); } // sign per SAM spec: positive for leftmost, negative for rightmost r.templateLength = r.alignmentStart === minStart ? estimatedTemplateLength : -estimatedTemplateLength; }); } } /** * Attempt to calculate the `templateLength` for a pair of intra-slice paired * reads. Ported from htslib. Algorithm is imperfect. */ function calculateIntraSliceMatePairTemplateLength(thisRecord, mateRecord) { // this just estimates the template length by using the simple (non-gapped) // end coordinate of each read, because gapping in the alignment doesn't mean // the template is longer or shorter const start = Math.min(thisRecord.alignmentStart, mateRecord.alignmentStart); const end = Math.max(thisRecord.alignmentStart + thisRecord.readLength - 1, mateRecord.alignmentStart + mateRecord.readLength - 1); const lengthEstimate = end - start + 1; // sign per SAM spec: positive for leftmost, negative for rightmost thisRecord.templateLength = thisRecord.alignmentStart <= mateRecord.alignmentStart ? lengthEstimate : -lengthEstimate; mateRecord.templateLength = mateRecord.alignmentStart <= thisRecord.alignmentStart ? lengthEstimate : -lengthEstimate; } /** * establishes a mate-pair relationship between two records in the same slice. * CRAM compresses mate-pair relationships between records in the same slice * down into just one record having the index in the slice of its mate */ function associateIntraSliceMate(allRecords, currentRecordNumber, thisRecord, mateRecord) { const complicatedMultiSegment = !!(mateRecord.mate || (mateRecord.mateRecordNumber !== undefined && mateRecord.mateRecordNumber !== currentRecordNumber)); // Deal with lossy read names — assign a synthetic name from uniqueId // so that paired records share the same name if (!thisRecord.readName) { const syntheticName = String(thisRecord.uniqueId); thisRecord._syntheticReadName = syntheticName; mateRecord._syntheticReadName = syntheticName; } thisRecord.mate = { sequenceId: mateRecord.sequenceId, alignmentStart: mateRecord.alignmentStart, uniqueId: mateRecord.uniqueId, }; if (mateRecord.readName) { thisRecord.mate.readName = mateRecord.readName; } // the mate record might have its own mate pointer, if this is some kind of // multi-segment (more than paired) scheme, so only relate that one back to this one // if it does not have any other relationship if (!mateRecord.mate && mateRecord.mateRecordNumber === undefined) { mateRecord.mate = { sequenceId: thisRecord.sequenceId, alignmentStart: thisRecord.alignmentStart, uniqueId: thisRecord.uniqueId, }; if (thisRecord.readName) { mateRecord.mate.readName = thisRecord.readName; } } // make sure the proper flags and cramFlags are set on both records // paired thisRecord.flags |= Constants.BAM_FPAIRED; // set mate unmapped if needed if (mateRecord.flags & Constants.BAM_FUNMAP) { thisRecord.flags |= Constants.BAM_FMUNMAP; // thisRecord.templateLength = 0 } if (thisRecord.flags & Constants.BAM_FUNMAP) { // thisRecord.templateLength = 0 mateRecord.flags |= Constants.BAM_FMUNMAP; } // set mate reversed if needed if (mateRecord.flags & Constants.BAM_FREVERSE) { thisRecord.flags |= Constants.BAM_FMREVERSE; } if (thisRecord.flags & Constants.BAM_FREVERSE) { mateRecord.flags |= Constants.BAM_FMREVERSE; } if (thisRecord.templateLength === undefined) { if (complicatedMultiSegment) { calculateMultiSegmentMatedTemplateLength(allRecords, currentRecordNumber, thisRecord); } else { calculateIntraSliceMatePairTemplateLength(thisRecord, mateRecord); } } // delete this last because it's used by the // complicated template length estimation thisRecord.mateRecordNumber = undefined; } export default class CramSlice { file; container; containerPosition; sliceSize; _headerResult; _blocksResult; _blocksContentIdIndexResult; constructor(container, containerPosition, sliceSize) { this.file = container.file; this.container = container; this.containerPosition = containerPosition; this.sliceSize = sliceSize; } getHeader() { if (this._headerResult === undefined) { this._headerResult = this._fetchHeader(); this._headerResult.catch(() => { this._headerResult = undefined; }); } return this._headerResult; } async _fetchHeader() { // fetch and parse the slice header const { majorVersion } = await this.file.getDefinition(); const sectionParsers = getSectionParsers(majorVersion); const containerHeader = await this.container.getHeader(); const header = await this.file.readBlock(containerHeader._endPosition + this.containerPosition); const parser = header.contentType === 'MAPPED_SLICE_HEADER' ? sectionParsers.cramMappedSliceHeader.parser : header.contentType === 'UNMAPPED_SLICE_HEADER' ? sectionParsers.cramUnmappedSliceHeader.parser : undefined; if (parser) { const content = parseItem(header.content, parser, 0, containerHeader._endPosition); return { ...header, parsedContent: content }; } else { throw new CramMalformedError(`error reading slice header block, invalid content type ${header.contentType}`); } } getBlocks() { if (this._blocksResult === undefined) { this._blocksResult = this._fetchBlocks(); this._blocksResult.catch(() => { this._blocksResult = undefined; }); } return this._blocksResult; } async _fetchBlocks() { const header = await this.getHeader(); if (this.sliceSize) { // if we know the slice size (from the index), do one big read for all // blocks and parse from the in-memory buffer const containerHeader = await this.container.getHeader(); const sliceFilePosition = containerHeader._endPosition + this.containerPosition; const blocksFilePosition = header._endPosition; const headerSize = blocksFilePosition - sliceFilePosition; const remainingBytes = this.sliceSize - headerSize; const allBlocksBuffer = await this.file.read(remainingBytes, blocksFilePosition); const blocks = new Array(header.parsedContent.numBlocks); let bufferOffset = 0; for (let i = 0; i < blocks.length; i++) { const block = await this.file.readBlockFromBuffer(allBlocksBuffer, bufferOffset, blocksFilePosition + bufferOffset); blocks[i] = block; bufferOffset = block._endPosition - blocksFilePosition; } return blocks; } // fallback: read blocks one at a time (non-indexed access) let blockPosition = header._endPosition; const blocks = new Array(header.parsedContent.numBlocks); for (let i = 0; i < blocks.length; i++) { const block = await this.file.readBlock(blockPosition); blocks[i] = block; blockPosition = block._endPosition; } return blocks; } // no memoize async getCoreDataBlock() { const blocks = await this.getBlocks(); return blocks[0]; } _getBlocksContentIdIndex() { if (this._blocksContentIdIndexResult === undefined) { this._blocksContentIdIndexResult = this._fetchBlocksContentIdIndex(); this._blocksContentIdIndexResult.catch(() => { this._blocksContentIdIndexResult = undefined; }); } return this._blocksContentIdIndexResult; } async _fetchBlocksContentIdIndex() { const blocks = await this.getBlocks(); const blocksByContentId = {}; blocks.forEach(block => { if (block.contentType === 'EXTERNAL_DATA') { blocksByContentId[block.contentId] = block; } }); return blocksByContentId; } async getBlockByContentId(id) { const blocksByContentId = await this._getBlocksContentIdIndex(); return blocksByContentId[id]; } async getReferenceRegion() { // read the slice header const sliceHeader = (await this.getHeader()).parsedContent; if (!isMappedSliceHeader(sliceHeader)) { throw new Error('slice header not mapped'); } if (sliceHeader.refSeqId < 0) { return undefined; } const compressionScheme = await this.container.getCompressionScheme(); if (compressionScheme === undefined) { throw new Error('compression scheme undefined'); } if (sliceHeader.refBaseBlockId >= 0) { const refBlock = await this.getBlockByContentId(sliceHeader.refBaseBlockId); if (!refBlock) { throw new CramMalformedError('embedded reference specified, but reference block does not exist'); } // TODO: we do not read anything named 'span' // if (sliceHeader.span > refBlock.uncompressedSize) { // throw new CramMalformedError('Embedded reference is too small') // } // TODO verify return { seq: decodeUtf8(refBlock.content), start: sliceHeader.refSeqStart, end: sliceHeader.refSeqStart + sliceHeader.refSeqSpan - 1, span: sliceHeader.refSeqSpan, }; } if (compressionScheme.referenceRequired || this.file.fetchReferenceSequenceCallback) { if (!this.file.fetchReferenceSequenceCallback) { throw new Error('reference sequence not embedded, and seqFetch callback not provided, cannot fetch reference sequence'); } const seq = await this.file.fetchReferenceSequenceCallback(sliceHeader.refSeqId, sliceHeader.refSeqStart, sliceHeader.refSeqStart + sliceHeader.refSeqSpan - 1); if (seq.length !== sliceHeader.refSeqSpan) { throw new CramArgumentError('seqFetch callback returned a reference sequence of the wrong length'); } return { seq, start: sliceHeader.refSeqStart, end: sliceHeader.refSeqStart + sliceHeader.refSeqSpan - 1, span: sliceHeader.refSeqSpan, }; } return undefined; } getAllRecords() { return this.getRecords(() => true); } async _fetchRecords(decodeOptions) { const { majorVersion } = await this.file.getDefinition(); const compressionScheme = await this.container.getCompressionScheme(); if (compressionScheme === undefined) { throw new Error('compression scheme undefined'); } const sliceHeader = await this.getHeader(); const blocksByContentId = await this._getBlocksContentIdIndex(); // check MD5 of reference if available if (majorVersion > 1 && this.file.options.checkSequenceMD5 && isMappedSliceHeader(sliceHeader.parsedContent) && sliceHeader.parsedContent.refSeqId >= 0 && sliceHeader.parsedContent.md5?.join('') !== '0000000000000000') { const refRegion = await this.getReferenceRegion(); if (refRegion) { const { seq, start, end } = refRegion; const seqMd5 = sequenceMD5(seq); const storedMd5 = sliceHeader.parsedContent.md5 ?.map(byte => (byte < 16 ? '0' : '') + byte.toString(16)) .join(''); if (seqMd5 !== storedMd5) { throw new CramMalformedError(`MD5 checksum reference mismatch for ref ${sliceHeader.parsedContent.refSeqId} pos ${start}..${end}. recorded MD5: ${storedMd5}, calculated MD5: ${seqMd5}`); } } } // tracks the read position within the block. codec.decode() methods // advance the byte and bit positions in the cursor as they decode // data note that we are only decoding a single block here, the core // data block const coreDataBlock = await this.getCoreDataBlock(); const externalCursorMap = new Map(); const cursors = { lastAlignmentStart: isMappedSliceHeader(sliceHeader.parsedContent) ? sliceHeader.parsedContent.refSeqStart : 0, coreBlock: { bitPosition: 7, bytePosition: 0 }, externalBlocks: { getCursor(contentId) { let r = externalCursorMap.get(contentId); if (r === undefined) { r = { bitPosition: 7, bytePosition: 0 }; externalCursorMap.set(contentId, r); } return r; }, }, }; // Pre-decode external int blocks: batch ITF8 decode via WASM so that // ExternalCodec.decode() becomes a simple array index read. // A block can only be pre-decoded if ALL accessors use int type. // If any byte-type accessor shares the same block, skip it. const externalIntBlockIds = new Set(); const externalByteBlockIds = new Set(); // Recurse through codec encodings to find which external block IDs are // used as int vs byte. codecId 1 = EXTERNAL, 4 = BYTE_ARRAY_LENGTH // (whose lengths sub-codec is int, values sub-codec is byte), // 5 = BYTE_ARRAY_STOP (always byte). function collectExternalBlockIds(enc, isInt) { if (!enc) { return; } if (enc.codecId === 1) { if (isInt) { externalIntBlockIds.add(enc.parameters.blockContentId); } else { externalByteBlockIds.add(enc.parameters.blockContentId); } } else if (enc.codecId === 4) { collectExternalBlockIds(enc.parameters.lengthsEncoding, true); collectExternalBlockIds(enc.parameters.valuesEncoding, false); } else if (enc.codecId === 5) { externalByteBlockIds.add(enc.parameters.blockContentId); } } for (const [ds, enc] of Object.entries(compressionScheme.dataSeriesEncoding)) { const dsType = dataSeriesTypes[ds]; collectExternalBlockIds(enc, dsType === 'int'); } for (const tagEnc of Object.values(compressionScheme.tagEncoding)) { collectExternalBlockIds(tagEnc, false); } // Remove any int block that is also used as byte for (const id of externalByteBlockIds) { externalIntBlockIds.delete(id); } const preDecodedIntBlocks = new Map(); for (const contentId of externalIntBlockIds) { const block = blocksByContentId[contentId]; if (block?.content.length) { const values = batchDecodeItf8(block.content); preDecodedIntBlocks.set(contentId, { values, index: 0 }); } } cursors.preDecodedIntBlocks = preDecodedIntBlocks; // Build bound decode functions per data series. For ExternalCodec this // captures the content buffer and cursor directly, eliminating per-call // Record/Map lookup overhead. The bound decoders are assembled into a // single object literal with all data series present so V8 sees a stable // hidden class — call sites in decodeRecord then become direct property // accesses with monomorphic inline caches. const bind = (dataSeriesName) => { const codec = compressionScheme.getCodecForDataSeries(dataSeriesName); if (!codec) { return () => { throw new CramMalformedError(`no codec defined for ${dataSeriesName} data series`); }; } if (codec instanceof ExternalCodec) { const bid = codec.parameters.blockContentId; const preDecoded = preDecodedIntBlocks.get(bid); if (preDecoded) { const { values } = preDecoded; return () => values[preDecoded.index++]; } const contentBlock = blocksByContentId[bid]; if (!contentBlock) { return () => { throw new CramMalformedError(`no block found with content ID ${bid}`); }; } const cursor = cursors.externalBlocks.getCursor(bid); const content = contentBlock.content; if (codec.dataType === 'int') { return () => parseItf8(content, cursor); } // Mirror the bounds check in ExternalCodec.decode — without it, // a truncated/corrupt block silently yields `undefined` for byte // reads, which downstream propagates as NaN/0 (silent data // corruption) rather than a clear error. return () => { if (cursor.bytePosition >= content.length) { throw new CramBufferOverrunError('attempted to read beyond end of block. this file seems truncated.'); } return content[cursor.bytePosition++]; }; } if (codec instanceof ByteArrayStopCodec) { const { blockContentId, stopByte } = codec.parameters; const contentBlock = blocksByContentId[blockContentId]; if (!contentBlock) { return () => { throw new CramMalformedError(`no block found with content ID ${blockContentId}`); }; } const content = contentBlock.content; const cursor = cursors.externalBlocks.getCursor(blockContentId); return () => { const start = cursor.bytePosition; const len = content.length; let pos = start; while (pos < len && content[pos] !== stopByte) { pos++; } if (pos >= len) { throw new CramBufferOverrunError('byteArrayStop reading beyond length of data buffer?'); } cursor.bytePosition = pos + 1; return content.subarray(start, pos); }; } return () => codec.decode(this, coreDataBlock, blocksByContentId, cursors); }; const bd = { BF: bind('BF'), CF: bind('CF'), RI: bind('RI'), RL: bind('RL'), AP: bind('AP'), RG: bind('RG'), RN: bind('RN'), MF: bind('MF'), NS: bind('NS'), NP: bind('NP'), TS: bind('TS'), NF: bind('NF'), TL: bind('TL'), FN: bind('FN'), FC: bind('FC'), FP: bind('FP'), DL: bind('DL'), BB: bind('BB'), QQ: bind('QQ'), BS: bind('BS'), IN: bind('IN'), RS: bind('RS'), PD: bind('PD'), HC: bind('HC'), SC: bind('SC'), MQ: bind('MQ'), BA: bind('BA'), QS: bind('QS'), TC: bind('TC'), TN: bind('TN'), }; // Bulk byte decoder for QS and BA — getBytesSubarray returns a subarray // view when the codec supports it (e.g. ExternalCodec), or undefined otherwise const qsCodec = compressionScheme.getCodecForDataSeries('QS'); const baCodec = compressionScheme.getCodecForDataSeries('BA'); const decodeBulkBytesRaw = qsCodec || baCodec ? (dataSeriesName, length) => { const codec = dataSeriesName === 'QS' ? qsCodec : baCodec; return codec?.getBytesSubarray(blocksByContentId, cursors, length); } : undefined; // Bound tag decoders — tags are typically encoded as byteArrayLength // (codecId=4) wrapping External-int lengths + External-byte values. We // build a fast closure per tagId that inlines the length read and value // subarray, eliminating per-call dispatch through ByteArrayLengthCodec // and the inner codecs. Other encodings fall back to the generic dispatch. const boundTagDecoders = {}; const bindTagFallback = (tagId) => { const codec = compressionScheme.getCodecForTag(tagId); return () => codec.decode(this, coreDataBlock, blocksByContentId, cursors); }; for (const tagId of Object.keys(compressionScheme.tagEncoding)) { const enc = compressionScheme.tagEncoding[tagId]; if (enc.codecId === 4 && enc.parameters.lengthsEncoding.codecId === 1 && enc.parameters.valuesEncoding.codecId === 1) { const lenBid = enc.parameters.lengthsEncoding.parameters.blockContentId; const valBid = enc.parameters.valuesEncoding.parameters.blockContentId; const lenContentBlock = blocksByContentId[lenBid]; const valContentBlock = blocksByContentId[valBid]; if (!lenContentBlock || !valContentBlock) { boundTagDecoders[tagId] = bindTagFallback(tagId); continue; } const valContent = valContentBlock.content; const valCursor = cursors.externalBlocks.getCursor(valBid); const lenPreDecoded = preDecodedIntBlocks.get(lenBid); const lenContent = lenContentBlock.content; const lenCursor = cursors.externalBlocks.getCursor(lenBid); const readTagLen = lenPreDecoded ? () => lenPreDecoded.values[lenPreDecoded.index++] : () => parseItf8(lenContent, lenCursor); boundTagDecoders[tagId] = () => { const length = readTagLen(); if (length === 0) { return EMPTY_BYTES; } const start = valCursor.bytePosition; const end = start + length; if (end > valContent.length) { throw new CramBufferOverrunError('attempted to read beyond end of block. this file seems truncated.'); } valCursor.bytePosition = end; return valContent.subarray(start, end); }; } else { boundTagDecoders[tagId] = bindTagFallback(tagId); } } const records = new Array(sliceHeader.parsedContent.numRecords); const rfSchema = buildRFSchema(bd, majorVersion); for (let i = 0; i < records.length; i += 1) { try { records[i] = new CramRecord(decodeRecord(this, bd, rfSchema, boundTagDecoders, compressionScheme, sliceHeader, coreDataBlock, blocksByContentId, cursors, majorVersion, i, sliceHeader.contentPosition + sliceHeader.parsedContent.recordCounter + i + 1, decodeOptions, decodeBulkBytesRaw)); } catch (e) { const err = e; if (err.code === 'CRAM_BUFFER_OVERRUN') { const recordsDecoded = i; const recordsExpected = sliceHeader.parsedContent.numRecords; throw new CramMalformedError(`Failed to decode all records in slice. Decoded ${recordsDecoded} of ${recordsExpected} expected records. ` + `Buffer overrun suggests either: (1) file is truncated/corrupted, (2) compression scheme is incorrect, ` + `or (3) there's a bug in the decoder. Original error: ${err.message}`); } else { throw e; } } } // interpret `recordsToNextFragment` attributes to make standard `mate` // objects. The records loop above fills every slot or throws — by the // time we get here, records[i] is always defined. The records[mate] // guard protects against malformed mateRecordNumber pointing past the // slice. for (let i = 0; i < records.length; i += 1) { const r = records[i]; const { mateRecordNumber } = r; if (mateRecordNumber !== undefined && mateRecordNumber >= 0 && records[mateRecordNumber]) { associateIntraSliceMate(records, i, r, records[mateRecordNumber]); } } return records; } async getRecords(filterFunction, decodeOptions) { // Merge with defaults const opts = { ...defaultDecodeOptions, ...decodeOptions }; // fetch the features if necessary, using the file-level feature cache // Include decode options in cache key so different decode configs are cached separately const optionsKey = `${opts.decodeTags ? 1 : 0}`; const cacheKey = `${this.container.filePosition}:${this.containerPosition}:${optionsKey}`; let recordsPromise = this.file.featureCache.get(cacheKey); if (!recordsPromise) { recordsPromise = this._fetchRecords(opts); this.file.featureCache.set(cacheKey, recordsPromise); } const unfiltered = await recordsPromise; const records = unfiltered.filter(filterFunction); // if we can fetch reference sequence, add the reference sequence to the records if (records.length && this.file.fetchReferenceSequenceCallback) { const sliceHeader = await this.getHeader(); if (isMappedSliceHeader(sliceHeader.parsedContent) && (sliceHeader.parsedContent.refSeqId >= 0 || // single-ref slice sliceHeader.parsedContent.refSeqId === -2) // multi-ref slice ) { const singleRefId = sliceHeader.parsedContent.refSeqId >= 0 ? sliceHeader.parsedContent.refSeqId : undefined; const compressionScheme = await this.container.getCompressionScheme(); if (compressionScheme === undefined) { throw new Error('compression scheme undefined'); } const refRegions = {}; // iterate over the records to find the spans of the reference // sequences we need to fetch for (const record of records) { const seqId = singleRefId !== undefined ? singleRefId : record.sequenceId; let refRegion = refRegions[seqId]; if (!refRegion) { refRegion = { id: seqId, start: record.alignmentStart, end: Number.NEGATIVE_INFINITY, seq: null, }; refRegions[seqId] = refRegion; } const end = record.alignmentStart + (record.lengthOnRef || record.readLength) - 1; if (end > refRegion.end) { refRegion.end = end; } if (record.alignmentStart < refRegion.start) { refRegion.start = record.alignmentStart; } } // fetch the `seq` for all of the ref regions await Promise.all(Object.values(refRegions).map(async (refRegion) => { if (refRegion.id !== -1 && refRegion.start <= refRegion.end && this.file.fetchReferenceSequenceCallback) { refRegion.seq = await this.file.fetchReferenceSequenceCallback(refRegion.id, refRegion.start, refRegion.end); } })); // now decorate all the records with them for (const record of records) { const seqId = singleRefId !== undefined ? singleRefId : record.sequenceId; const refRegion = refRegions[seqId]; if (refRegion?.seq) { const seq = refRegion.seq; record.addReferenceSequence({ ...refRegion, seq }, compressionScheme); } } } } return records; } } //# sourceMappingURL=index.js.map