@gmod/cram
Version:
read CRAM files with pure Javascript
320 lines • 14.4 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const quick_lru_1 = __importDefault(require("@jbrowse/quick-lru"));
const crc32_1 = __importDefault(require("crc/calculators/crc32"));
const errors_ts_1 = require("../errors.js");
const htscodecs = __importStar(require("../htscodecs/index.js"));
const io_ts_1 = require("../io.js");
const sam_ts_1 = require("../sam.js");
const util_ts_1 = require("./util.js");
const unzip_ts_1 = require("../unzip.js");
const index_ts_1 = __importDefault(require("./container/index.js"));
const sectionParsers_ts_1 = require("./sectionParsers.js");
const xz_decompress_ts_1 = require("../xz-decompress/xz-decompress.js");
// source: https://abdulapopoola.com/2019/01/20/check-endianness-with-javascript/
let isLittleEndian;
function checkLittleEndian() {
if (isLittleEndian === undefined) {
isLittleEndian =
new Uint8Array(new Uint32Array([0x11223344]).buffer)[0] === 0x44;
}
return isLittleEndian;
}
class CramFile {
file;
validateChecksums;
fetchReferenceSequenceCallback;
options;
featureCache;
header;
_sectionParsers;
_definitionResult;
_samHeaderResult;
constructor(args) {
this.file = (0, io_ts_1.open)(args.url, args.path, args.filehandle);
this.validateChecksums = args.validateChecksums ?? false;
this.fetchReferenceSequenceCallback = args.seqFetch;
this.options = {
checkSequenceMD5: args.checkSequenceMD5,
cacheSize: args.cacheSize ?? 20000,
};
// cache of features in a slice, keyed by the slice offset. caches all of
// the features in a slice, or none. the cache is actually used by the
// slice object, it's just kept here at the level of the file
this.featureCache = new quick_lru_1.default({
maxSize: this.options.cacheSize,
});
if (!checkLittleEndian()) {
throw new Error('Detected big-endian machine, may be unable to run');
}
}
read(length, position) {
return this.file.read(length, position);
}
async _getSectionParsers() {
if (!this._sectionParsers) {
const { majorVersion } = await this.getDefinition();
this._sectionParsers = (0, sectionParsers_ts_1.getSectionParsers)(majorVersion);
}
return this._sectionParsers;
}
async getDefinition() {
if (this._definitionResult === undefined) {
this._definitionResult = this._fetchDefinition();
this._definitionResult.catch(() => {
this._definitionResult = undefined;
});
}
return this._definitionResult;
}
async _fetchDefinition() {
const { maxLength, parser } = (0, sectionParsers_ts_1.cramFileDefinition)();
const headbytes = await this.file.read(maxLength, 0);
const definition = parser(headbytes).value;
if (definition.magic !== 'CRAM') {
throw new Error('Not a CRAM file, does not match magic string');
}
else if (definition.majorVersion !== 2 && definition.majorVersion !== 3) {
throw new errors_ts_1.CramUnimplementedError(`CRAM version ${definition.majorVersion} not supported`);
}
else {
return definition;
}
}
async getSamHeader() {
if (this._samHeaderResult === undefined) {
this._samHeaderResult = this._fetchSamHeader();
this._samHeaderResult.catch(() => {
this._samHeaderResult = undefined;
});
}
return this._samHeaderResult;
}
async _fetchSamHeader() {
const firstContainer = await this.getContainerById(0);
if (!firstContainer) {
throw new errors_ts_1.CramMalformedError('file contains no containers');
}
const firstBlock = await firstContainer.getFirstBlock();
const content = firstBlock.content;
const dataView = new DataView(content.buffer, content.byteOffset, content.byteLength);
const headerLength = dataView.getInt32(0, true);
const textStart = 4;
const text = (0, util_ts_1.decodeUtf8)(content.subarray(textStart, textStart + headerLength));
this.header = text;
return (0, sam_ts_1.parseHeaderText)(text);
}
async getHeaderText() {
await this.getSamHeader();
return this.header;
}
// Walk containers from the start of the file. Yields each container along
// with its parsed header. The first container's length is recomputed by
// reading all of its blocks because the recorded length cannot be trusted
// (htslib bug); subsequent containers use header._size + header.length.
async *iterContainers() {
const sectionParsers = await this._getSectionParsers();
let position = sectionParsers.cramFileDefinition.maxLength;
let i = 0;
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
const container = this.getContainerAtPosition(position);
const header = await container.getHeader();
yield container;
if (i === 0) {
position = header._endPosition;
for (let j = 0; j < header.numBlocks; j++) {
const block = await this.readBlock(position);
position = block._endPosition;
}
}
else {
position += header._size + header.length;
}
i++;
}
}
async getContainerById(containerNumber) {
let i = 0;
for await (const container of this.iterContainers()) {
if (i === containerNumber) {
return container;
}
i++;
}
return undefined;
}
async checkCrc32(position, length, recordedCrc32, description) {
const b = await this.file.read(length, position);
// this shift >>> 0 is equivalent to crc32(b).unsigned but uses the
// internal calculator of crc32 to avoid accidentally importing buffer
// https://github.com/alexgorbatchev/crc/blob/31fc3853e417b5fb5ec83335428805842575f699/src/define_crc.ts#L5
const calculatedCrc32 = (0, crc32_1.default)(b) >>> 0;
if (calculatedCrc32 !== recordedCrc32) {
throw new errors_ts_1.CramMalformedError(`crc mismatch in ${description}: recorded CRC32 = ${recordedCrc32}, but calculated CRC32 = ${calculatedCrc32}`);
}
}
/**
* @returns {Promise[number]} the number of containers in the file
*
* note: this is currently used only in unit tests, and after removing file
* length check, relies on a try catch to read return an error to break
*/
async containerCount() {
let containerCount = 0;
try {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for await (const _container of this.iterContainers()) {
containerCount += 1;
}
}
catch (e) {
containerCount--;
}
return containerCount;
}
getContainerAtPosition(position) {
return new index_ts_1.default(this, position);
}
async readBlockHeader(position) {
const { cramBlockHeader } = await this._getSectionParsers();
const buffer = await this.file.read(cramBlockHeader.maxLength, position);
return (0, util_ts_1.parseItem)(buffer, cramBlockHeader.parser, 0, position);
}
async _parseSection(section, position, size = section.maxLength, preReadBuffer) {
const buffer = preReadBuffer ?? (await this.file.read(size, position));
const data = (0, util_ts_1.parseItem)(buffer, section.parser, 0, position);
if (data._size !== size) {
throw new errors_ts_1.CramMalformedError(`section read error: requested size ${size} does not equal parsed size ${data._size}`);
}
return data;
}
async _uncompress(compressionMethod, inputBuffer, uncompressedSize) {
let buf;
if (compressionMethod === 'gzip') {
buf = await (0, unzip_ts_1.unzip)(inputBuffer);
}
else if (compressionMethod === 'bzip2') {
buf = await htscodecs.bz2_uncompress(inputBuffer, uncompressedSize);
}
else if (compressionMethod === 'lzma') {
buf = await (0, xz_decompress_ts_1.xzDecompress)(inputBuffer);
}
else if (compressionMethod === 'rans') {
buf = await htscodecs.rans_uncompress(inputBuffer);
}
else if (compressionMethod === 'rans4x16') {
buf = await htscodecs.r4x16_uncompress(inputBuffer);
}
else if (compressionMethod === 'arith') {
buf = await htscodecs.arith_uncompress(inputBuffer);
}
else if (compressionMethod === 'fqzcomp') {
buf = await htscodecs.fqzcomp_uncompress(inputBuffer);
}
else if (compressionMethod === 'tok3') {
buf = await htscodecs.tok3_uncompress(inputBuffer);
}
else {
throw new errors_ts_1.CramUnimplementedError(`${compressionMethod} decompression not yet implemented`);
}
if (buf.length !== uncompressedSize) {
throw new errors_ts_1.CramMalformedError(`${compressionMethod} decompression produced ${buf.length} bytes, expected ${uncompressedSize}`);
}
return buf;
}
async readBlock(position) {
const { majorVersion } = await this.getDefinition();
const { cramBlockHeader, cramBlockCrc32 } = await this._getSectionParsers();
const headerBuf = await this.file.read(cramBlockHeader.maxLength, position);
const blockHeader = (0, util_ts_1.parseItem)(headerBuf, cramBlockHeader.parser, 0, position);
const totalSize = blockHeader._size +
blockHeader.compressedSize +
(majorVersion >= 3 ? cramBlockCrc32.maxLength : 0);
const fullBuffer = await this.file.read(totalSize, position);
return this.readBlockFromBuffer(fullBuffer, 0, position);
}
async readBlockFromBuffer(buffer, bufferOffset, filePosition) {
const { majorVersion } = await this.getDefinition();
const sectionParsers = await this._getSectionParsers();
const { cramBlockHeader } = sectionParsers;
const headerBytes = buffer.subarray(bufferOffset, bufferOffset + cramBlockHeader.maxLength);
const blockHeader = (0, util_ts_1.parseItem)(headerBytes, cramBlockHeader.parser, 0, filePosition);
const blockContentPosition = blockHeader._endPosition;
const contentOffset = bufferOffset + blockHeader._size;
const d = buffer.subarray(contentOffset, contentOffset + blockHeader.compressedSize);
// Per CRAM spec (PR #681), blocks with uncompressed size 0 are treated as
// empty regardless of the method byte — htsjdk has produced invalid empty
// RANS blocks that would otherwise fail to decompress.
const uncompressedData = blockHeader.uncompressedSize === 0
? new Uint8Array(0)
: blockHeader.compressionMethod !== 'raw'
? await this._uncompress(blockHeader.compressionMethod, d, blockHeader.uncompressedSize)
: d;
const block = {
...blockHeader,
_endPosition: blockContentPosition,
contentPosition: blockContentPosition,
content: uncompressedData,
};
if (majorVersion >= 3) {
const crcOffset = contentOffset + blockHeader.compressedSize;
const crcBytes = buffer.subarray(crcOffset, crcOffset + sectionParsers.cramBlockCrc32.maxLength);
const crc = (0, util_ts_1.parseItem)(crcBytes, sectionParsers.cramBlockCrc32.parser, 0, blockContentPosition + blockHeader.compressedSize);
block.crc32 = crc.crc32;
if (this.validateChecksums) {
const blockData = buffer.subarray(bufferOffset, bufferOffset + blockHeader._size + blockHeader.compressedSize);
const calculatedCrc32 = (0, crc32_1.default)(blockData) >>> 0;
if (calculatedCrc32 !== crc.crc32) {
throw new errors_ts_1.CramMalformedError(`crc mismatch in block data: recorded CRC32 = ${crc.crc32}, but calculated CRC32 = ${calculatedCrc32}`);
}
}
block._endPosition = crc._endPosition;
block._size =
block.compressedSize + sectionParsers.cramBlockCrc32.maxLength;
}
else {
block._endPosition = blockContentPosition + block.compressedSize;
block._size = block.compressedSize;
}
return block;
}
}
exports.default = CramFile;
//# sourceMappingURL=file.js.map