@gmod/cram/dist/cramFile/sectionParsers.js

read CRAM files with pure Javascript

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.cramFileDefinition = cramFileDefinition;
exports.cramBlockHeader = cramBlockHeader;
exports.cramBlockCrc32 = cramBlockCrc32;
exports.cramTagDictionary = cramTagDictionary;
exports.cramPreservationMap = cramPreservationMap;
exports.isMappedSliceHeader = isMappedSliceHeader;
exports.getSectionParsers = getSectionParsers;
const util_ts_1 = require("./util.js");
const COMPRESSION_METHODS = [
    'raw',
    'gzip',
    'bzip2',
    'lzma',
    'rans',
    'rans4x16',
    'arith',
    'fqzcomp',
    'tok3',
];
const CONTENT_TYPES = [
    'FILE_HEADER',
    'COMPRESSION_HEADER',
    'MAPPED_SLICE_HEADER',
    'UNMAPPED_SLICE_HEADER', // < only used in cram v1
    'EXTERNAL_DATA',
    'CORE_DATA',
];
// Per-version dispatch for the optional `recordCounter` field shared by slice
// headers and container header 1. CRAM v3 uses LTF8, v2 uses ITF8, v1 omits it.
// Called once per slice / container header, not per record — no allocation
// concerns. Returns the same tuple parseItf8/parseLtf8 already allocates.
function readRecordCounter(buffer, offset, majorVersion) {
    if (majorVersion >= 3) {
        return (0, util_ts_1.parseLtf8)(buffer, offset);
    }
    else if (majorVersion === 2) {
        return (0, util_ts_1.parseItf8Sized)(buffer, offset);
    }
    else {
        console.warn('recordCounter=0');
        return [0, 0];
    }
}
function cramFileDefinition() {
    return {
        parser: (b, _startOffset = 0) => {
            const dataView = new DataView(b.buffer, b.byteOffset, b.length);
            let offset = 0;
            const magic = (0, util_ts_1.decodeUtf8)(b.subarray(offset, offset + 4));
            offset += 4;
            const majorVersion = dataView.getUint8(offset);
            offset += 1;
            const minorVersion = dataView.getUint8(offset);
            offset += 1;
            // 20-byte null-padded field
            const fileId = (0, util_ts_1.readNullTerminatedStringFromBuffer)(b.subarray(offset, offset + 20));
            offset += 20;
            return {
                value: {
                    magic,
                    majorVersion,
                    minorVersion,
                    fileId,
                },
                offset,
            };
        },
        maxLength: 26,
    };
}
function cramBlockHeader() {
    const parser = (buffer, _startOffset = 0) => {
        const b = buffer;
        const dataView = new DataView(b.buffer, b.byteOffset, b.length);
        let offset = 0;
        const d = dataView.getUint8(offset);
        const compressionMethod = COMPRESSION_METHODS[d];
        if (!compressionMethod) {
            throw new Error(`compression method number ${d} not implemented`);
        }
        offset += 1;
        const c = dataView.getUint8(offset);
        const contentType = CONTENT_TYPES[c];
        if (!contentType) {
            throw new Error(`invalid block content type id ${c}`);
        }
        offset += 1;
        const [contentId, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += newOffset1;
        const [compressedSize, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += newOffset2;
        const [uncompressedSize, newOffset3] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += newOffset3;
        return {
            offset,
            value: {
                uncompressedSize,
                compressedSize,
                contentId,
                contentType,
                compressionMethod,
            },
        };
    };
    return { parser, maxLength: 17 };
}
function cramBlockCrc32() {
    return {
        parser: (buffer, offset) => {
            const b = buffer;
            const dataView = new DataView(b.buffer, b.byteOffset, b.length);
            const crc32 = dataView.getUint32(offset, true);
            offset += 4;
            return {
                offset,
                value: {
                    crc32,
                },
            };
        },
        maxLength: 4,
    };
}
function makeTagSet(buffer, stringStart, stringEnd) {
    const str = (0, util_ts_1.decodeUtf8)(buffer.subarray(stringStart, stringEnd));
    const tags = [];
    for (let i = 0; i < str.length; i += 3) {
        tags.push(str.slice(i, i + 3));
    }
    return tags;
}
function cramTagDictionary() {
    return {
        parser: (buffer, offset) => {
            const [size, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset1;
            const subbuf = buffer.subarray(offset, offset + size);
            offset += size;
            const tagSets = [];
            let stringStart = 0;
            let i = 0;
            for (; i < subbuf.length; i++) {
                if (!subbuf[i]) {
                    tagSets.push(makeTagSet(subbuf, stringStart, i));
                    stringStart = i + 1;
                }
            }
            if (i > stringStart) {
                tagSets.push(makeTagSet(subbuf, stringStart, i));
            }
            return {
                value: {
                    size,
                    ents: tagSets,
                },
                offset,
            };
        },
    };
}
function cramPreservationMap() {
    return {
        parser: (buffer, offset) => {
            const b = buffer;
            const dataView = new DataView(b.buffer, b.byteOffset, b.length);
            const [mapSize, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset1;
            const [mapCount, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset2;
            const ents = [];
            for (let i = 0; i < mapCount; i++) {
                const key = String.fromCharCode(buffer[offset]) +
                    String.fromCharCode(buffer[offset + 1]);
                offset += 2;
                if (key === 'MI' ||
                    key === 'UI' ||
                    key === 'PI' ||
                    key === 'RN' ||
                    key === 'AP' ||
                    key === 'RR') {
                    ents.push({
                        key,
                        value: !!dataView.getUint8(offset),
                    });
                    offset += 1;
                }
                else if (key === 'SM') {
                    ents.push({
                        key,
                        value: [
                            dataView.getUint8(offset),
                            dataView.getUint8(offset + 1),
                            dataView.getUint8(offset + 2),
                            dataView.getUint8(offset + 3),
                            dataView.getUint8(offset + 4),
                        ],
                    });
                    offset += 5;
                }
                else if (key === 'TD') {
                    const { offset: offsetRet, value } = cramTagDictionary().parser(buffer, offset);
                    ents.push({ key, value: value.ents });
                    offset = offsetRet;
                }
                else {
                    throw new Error(`unknown key ${key}`);
                }
            }
            return {
                value: {
                    mapSize,
                    mapCount,
                    ents,
                },
                offset,
            };
        },
    };
}
function formatMap(data) {
    const map = {};
    for (const { key, value } of data.ents) {
        if (map[key]) {
            console.warn(`duplicate key ${key} in map`);
        }
        map[key] = value;
    }
    return map;
}
function isMappedSliceHeader(header) {
    return typeof header.refSeqId === 'number';
}
// assemble a section parser for the unmapped slice header, with slight
// variations depending on the major version of the cram file
function cramUnmappedSliceHeader(majorVersion) {
    let maxLength = 0;
    maxLength += 5;
    maxLength += 9;
    maxLength += 5 * 2;
    maxLength += 16;
    const parser = (buffer, offset) => {
        const [numRecords, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += newOffset1;
        const [recordCounter, rcAdvance] = readRecordCounter(buffer, offset, majorVersion);
        offset += rcAdvance;
        const [numBlocks, newOffset3] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += newOffset3;
        const [numContentIds, newOffset4] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += newOffset4;
        const contentIds = [];
        for (let i = 0; i < numContentIds; i++) {
            const [id, newOffset5] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset5;
            contentIds.push(id);
        }
        // the md5 sum is missing in cram v1
        let md5;
        if (majorVersion >= 2) {
            md5 = [...buffer.subarray(offset, offset + 16)];
            offset += 16;
        }
        return {
            value: {
                recordCounter,
                md5,
                contentIds,
                numContentIds,
                numBlocks,
                numRecords,
            },
            offset,
        };
    };
    return {
        parser,
        maxLength: (numContentIds) => maxLength + numContentIds * 5,
    };
}
// assembles a section parser for the unmapped slice header, with slight
// variations depending on the major version of the cram file
function cramMappedSliceHeader(majorVersion) {
    let maxLength = 0;
    maxLength += 5 * 4; // EL0
    maxLength += 9; // EL1
    maxLength += 5 * 3; // EL2 ITF8s
    maxLength += 16; // MD5
    return {
        parser: (buffer, offset) => {
            // L0
            const [refSeqId, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset1;
            const [refSeqStart, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset2;
            const [refSeqSpan, newOffset3] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset3;
            const [numRecords, newOffset4] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset4;
            // EL0
            // L1
            const [recordCounter, rcAdvance] = readRecordCounter(buffer, offset, majorVersion);
            offset += rcAdvance;
            // EL1
            // L2
            const [numBlocks, newOffset6] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset6;
            const [numContentIds, newOffset7] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset7;
            const contentIds = [];
            for (let i = 0; i < numContentIds; i++) {
                const [id, newOffset5] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
                offset += newOffset5;
                contentIds.push(id);
            }
            const [refBaseBlockId, newOffset8] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset8;
            // EL2
            // the md5 sum is missing in cram v1
            let md5;
            if (majorVersion >= 2) {
                md5 = [...buffer.subarray(offset, offset + 16)];
                offset += 16;
            }
            return {
                value: {
                    md5,
                    numBlocks,
                    numRecords,
                    numContentIds,
                    refSeqSpan,
                    refSeqId,
                    refSeqStart,
                    recordCounter,
                    refBaseBlockId,
                    contentIds,
                },
                offset,
            };
        },
        maxLength: (numContentIds) => maxLength + numContentIds * 5,
    };
}
function cramEncoding() {
    return {
        parser: (buffer, offset) => cramEncodingSub(buffer, offset),
    };
}
function cramEncodingSub(buffer, offset) {
    const b = buffer;
    const dataView = new DataView(b.buffer, b.byteOffset, b.length);
    const [codecId, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
    offset += newOffset1;
    const [parametersBytes, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
    offset += newOffset2;
    const parameters = {};
    if (codecId === 0) {
        // NULL
    }
    else if (codecId === 1) {
        // EXTERNAL
        const [bc, newOffset3] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.blockContentId = bc;
        offset += newOffset3;
    }
    else if (codecId === 2) {
        // GOLUMB
        const [off, newOffset3] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.offset = off;
        offset += newOffset3;
        const [M2, newOffset4] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.M = M2;
        offset += newOffset4;
    }
    else if (codecId === 3) {
        // HUFFMAN_INT
        const [numCodes, numCodesAdvance] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += numCodesAdvance;
        const symbols = [];
        for (let i = 0; i < numCodes; i++) {
            const [code, codeAdvance] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            symbols.push(code);
            offset += codeAdvance;
        }
        const [numLengths, numLengthsAdvance] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        offset += numLengthsAdvance;
        const bitLengths = [];
        for (let i = 0; i < numLengths; i++) {
            const [len, lenAdvance] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += lenAdvance;
            bitLengths.push(len);
        }
        parameters.numCodes = numCodes;
        parameters.symbols = symbols;
        parameters.numLengths = numLengths;
        parameters.bitLengths = bitLengths;
    }
    else if (codecId === 4) {
        // BYTE_ARRAY_LEN
        const { value: lengthsEncoding, offset: newOffset1 } = cramEncodingSub(buffer, offset);
        parameters.lengthsEncoding = lengthsEncoding;
        offset = newOffset1;
        const { value: valuesEncoding, offset: newOffset2 } = cramEncodingSub(buffer, offset);
        parameters.valuesEncoding = valuesEncoding;
        offset = newOffset2;
    }
    else if (codecId === 5) {
        // BYTE_ARRAY_STOP
        parameters.stopByte = dataView.getUint8(offset);
        offset += 1;
        const [blockContentId, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.blockContentId = blockContentId;
        offset += newOffset1;
    }
    else if (codecId === 6) {
        // BETA
        const [off, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.offset = off;
        offset += newOffset1;
        const [len, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.length = len;
        offset += newOffset2;
    }
    else if (codecId === 7) {
        // SUBEXP
        const [off, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.offset = off;
        offset += newOffset1;
        const [K, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.K = K;
        offset += newOffset2;
    }
    else if (codecId === 8) {
        // GOLOMB_RICE
        const [off, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.offset = off;
        offset += newOffset1;
        const [l2m, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.log2m = l2m;
        offset += newOffset2;
    }
    else if (codecId === 9) {
        // GAMMA
        const [off, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
        parameters.offset = off;
        offset += newOffset1;
    }
    else {
        throw new Error(`unknown codecId ${codecId}`);
    }
    return {
        value: {
            codecId,
            parametersBytes,
            parameters,
        },
        offset,
    };
}
function cramDataSeriesEncodingMap() {
    return {
        parser: (buffer, offset) => {
            const [mapSize, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset1;
            const [mapCount, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset2;
            const ents = [];
            for (let i = 0; i < mapCount; i++) {
                const key = String.fromCharCode(buffer[offset]) +
                    String.fromCharCode(buffer[offset + 1]);
                offset += 2;
                const { value, offset: newOffset4 } = cramEncodingSub(buffer, offset);
                offset = newOffset4;
                ents.push({ key, value });
            }
            return {
                value: {
                    mapSize,
                    ents,
                    mapCount,
                },
                offset,
            };
        },
    };
}
function cramTagEncodingMap() {
    return {
        parser: (buffer, offset) => {
            const [mapSize, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset1;
            const [mapCount, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset2;
            const ents = [];
            for (let i = 0; i < mapCount; i++) {
                const [k0, newOffset3] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
                offset += newOffset3;
                const key = String.fromCharCode((k0 >> 16) & 0xff) +
                    String.fromCharCode((k0 >> 8) & 0xff) +
                    String.fromCharCode(k0 & 0xff);
                const { value, offset: newOffset4 } = cramEncodingSub(buffer, offset);
                offset = newOffset4;
                ents.push({ key, value });
            }
            return {
                value: {
                    mapSize,
                    ents,
                    mapCount,
                },
                offset,
            };
        },
    };
}
function cramCompressionHeader() {
    return {
        parser: (buffer, offset) => {
            // TODO: if we want to support CRAM v1, we will need to refactor
            // compression header into 2 parts to parse the landmarks, like the
            // container header
            const { value: preservation, offset: newOffset1 } = cramPreservationMap().parser(buffer, offset);
            offset = newOffset1;
            const { value: dataSeriesEncoding, offset: newOffset2 } = cramDataSeriesEncodingMap().parser(buffer, offset);
            offset = newOffset2;
            const { value: tagEncoding, offset: newOffset3 } = cramTagEncodingMap().parser(buffer, offset);
            offset = newOffset3;
            return {
                value: {
                    dataSeriesEncoding: formatMap(dataSeriesEncoding),
                    preservation: formatMap(preservation),
                    tagEncoding: formatMap(tagEncoding),
                },
                offset,
            };
        },
    };
}
function cramContainerHeader1(majorVersion) {
    let maxLength = 4;
    maxLength += 5 * 4;
    maxLength += 9;
    maxLength += 9;
    maxLength += 5 + 5;
    return {
        maxLength,
        parser: (buffer, offset) => {
            const b = buffer;
            const dataView = new DataView(b.buffer, b.byteOffset, b.length);
            // byte size of the container data (blocks)
            const length = dataView.getInt32(offset, true);
            offset += 4;
            // reference sequence identifier:
            // -1 for unmapped reads,
            // -2 for multiple reference sequences
            const [refSeqId, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset1;
            const [refSeqStart, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset2;
            const [alignmentSpan, newOffset3] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset3;
            const [numRecords, newOffset4] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset4;
            const [recordCounter, rcAdvance] = readRecordCounter(buffer, offset, majorVersion);
            offset += rcAdvance;
            let numBases;
            if (majorVersion > 1) {
                const [n, newOffset5] = (0, util_ts_1.parseLtf8)(buffer, offset);
                numBases = n;
                offset += newOffset5;
            }
            const [numBlocks, newOffset6] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset6;
            const [numLandmarks, newOffset7] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset7;
            return {
                value: {
                    length,
                    refSeqId,
                    refSeqStart,
                    alignmentSpan,
                    numBlocks,
                    numLandmarks,
                    numBases,
                    recordCounter,
                    numRecords,
                },
                offset,
            };
        },
    };
}
function cramContainerHeader2(majorVersion) {
    return {
        parser: (buffer, offset) => {
            const b = buffer;
            const dataView = new DataView(b.buffer, b.byteOffset, b.length);
            const [numLandmarks, newOffset1] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
            offset += newOffset1;
            const landmarks = [];
            for (let i = 0; i < numLandmarks; i++) {
                const [landmark, newOffset2] = (0, util_ts_1.parseItf8Sized)(buffer, offset);
                offset += newOffset2;
                landmarks.push(landmark);
            }
            let crc32;
            if (majorVersion >= 3) {
                crc32 = dataView.getUint32(offset, true);
                offset += 4;
            }
            return {
                value: {
                    ...(crc32 === undefined ? {} : { crc32 }),
                    numLandmarks,
                    landmarks,
                },
                offset,
            };
        },
        maxLength: (numLandmarks) => 5 + 5 * numLandmarks + 4,
    };
}
function getSectionParsers(majorVersion) {
    return {
        cramFileDefinition: cramFileDefinition(),
        cramBlockHeader: cramBlockHeader(),
        cramBlockCrc32: cramBlockCrc32(),
        cramDataSeriesEncodingMap: cramDataSeriesEncodingMap(),
        cramTagEncodingMap: cramTagEncodingMap(),
        cramCompressionHeader: cramCompressionHeader(),
        cramEncoding: cramEncoding(),
        cramUnmappedSliceHeader: cramUnmappedSliceHeader(majorVersion),
        cramMappedSliceHeader: cramMappedSliceHeader(majorVersion),
        cramContainerHeader1: cramContainerHeader1(majorVersion),
        cramContainerHeader2: cramContainerHeader2(majorVersion),
    };
}
//# sourceMappingURL=sectionParsers.js.map