@gmod/cram
Version:
read CRAM files with pure Javascript
306 lines • 13.7 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const crc32_1 = __importDefault(require("crc/calculators/crc32"));
const quick_lru_1 = __importDefault(require("quick-lru"));
const xz_decompress_1 = require("xz-decompress");
const errors_ts_1 = require("../errors.js");
const htscodecs = __importStar(require("../htscodecs/index.js"));
const io_ts_1 = require("../io.js");
const index_ts_1 = __importDefault(require("../rans/index.js"));
const sam_ts_1 = require("../sam.js");
const index_ts_2 = require("../seek-bzip/index.js");
const unzip_ts_1 = require("../unzip.js");
const index_ts_3 = __importDefault(require("./container/index.js"));
const sectionParsers_ts_1 = require("./sectionParsers.js");
const util_ts_1 = require("./util.js");
function bufferToStream(buf) {
return new ReadableStream({
start(controller) {
controller.enqueue(buf);
controller.close();
},
});
}
// source: https://abdulapopoola.com/2019/01/20/check-endianness-with-javascript/
function getEndianness() {
const uInt32 = new Uint32Array([0x11223344]);
const uInt8 = new Uint8Array(uInt32.buffer);
if (uInt8[0] === 0x44) {
return 0; // little-endian
}
else if (uInt8[0] === 0x11) {
return 1; // big-endian
}
else {
return 2; // mixed-endian?
}
}
class CramFile {
constructor(args) {
this.file = (0, io_ts_1.open)(args.url, args.path, args.filehandle);
this.validateChecksums = true;
this.fetchReferenceSequenceCallback = args.seqFetch;
this.options = {
checkSequenceMD5: args.checkSequenceMD5,
cacheSize: args.cacheSize ?? 20000,
};
// cache of features in a slice, keyed by the slice offset. caches all of
// the features in a slice, or none. the cache is actually used by the
// slice object, it's just kept here at the level of the file
this.featureCache = new quick_lru_1.default({
maxSize: this.options.cacheSize,
});
if (getEndianness() > 0) {
throw new Error('Detected big-endian machine, may be unable to run');
}
}
read(length, position) {
return this.file.read(length, position);
}
// memoized
async getDefinition() {
const { maxLength, parser } = (0, sectionParsers_ts_1.cramFileDefinition)();
const headbytes = await this.file.read(maxLength, 0);
const definition = parser(headbytes).value;
if (definition.majorVersion !== 2 && definition.majorVersion !== 3) {
throw new errors_ts_1.CramUnimplementedError(`CRAM version ${definition.majorVersion} not supported`);
}
return definition;
}
// memoize
async getSamHeader() {
const firstContainer = await this.getContainerById(0);
if (!firstContainer) {
throw new errors_ts_1.CramMalformedError('file contains no containers');
}
const firstBlock = await firstContainer.getFirstBlock();
const content = firstBlock.content;
const dataView = new DataView(content.buffer);
const headerLength = dataView.getInt32(0, true);
const textStart = 4;
const decoder = new TextDecoder('utf8');
const text = decoder.decode(content.subarray(textStart, textStart + headerLength));
this.header = text;
return (0, sam_ts_1.parseHeaderText)(text);
}
async getHeaderText() {
await this.getSamHeader();
return this.header;
}
async getContainerById(containerNumber) {
const { majorVersion } = await this.getDefinition();
const sectionParsers = (0, sectionParsers_ts_1.getSectionParsers)(majorVersion);
let position = sectionParsers.cramFileDefinition.maxLength;
// skip with a series of reads to the proper container
let currentContainer;
for (let i = 0; i <= containerNumber; i++) {
// if we are about to go off the end of the file
// and have not found that container, it does not exist
// if (position + cramContainerHeader1.maxLength + 8 >= fileSize) {
// return undefined
// }
currentContainer = this.getContainerAtPosition(position);
const currentHeader = await currentContainer.getHeader();
// if this is the first container, read all the blocks in the container
// to determine its length, because we cannot trust the container
// header's given length due to a bug somewhere in htslib
if (i === 0) {
position = currentHeader._endPosition;
for (let j = 0; j < currentHeader.numBlocks; j++) {
const block = await this.readBlock(position);
position = block._endPosition;
}
}
else {
// otherwise, just traverse to the next container using the container's
// length
position += currentHeader._size + currentHeader.length;
}
}
return currentContainer;
}
async checkCrc32(position, length, recordedCrc32, description) {
const b = await this.file.read(length, position);
// this shift >>> 0 is equivalent to crc32(b).unsigned but uses the
// internal calculator of crc32 to avoid accidentally importing buffer
// https://github.com/alexgorbatchev/crc/blob/31fc3853e417b5fb5ec83335428805842575f699/src/define_crc.ts#L5
const calculatedCrc32 = (0, crc32_1.default)(b) >>> 0;
if (calculatedCrc32 !== recordedCrc32) {
throw new errors_ts_1.CramMalformedError(`crc mismatch in ${description}: recorded CRC32 = ${recordedCrc32}, but calculated CRC32 = ${calculatedCrc32}`);
}
}
/**
* @returns {Promise[number]} the number of containers in the file
*
* note: this is currently used only in unit tests, and after removing file
* length check, relies on a try catch to read return an error to break
*/
async containerCount() {
const { majorVersion } = await this.getDefinition();
const sectionParsers = (0, sectionParsers_ts_1.getSectionParsers)(majorVersion);
let containerCount = 0;
let position = sectionParsers.cramFileDefinition.maxLength;
try {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
const currentHeader = await this.getContainerAtPosition(position).getHeader();
// if this is the first container, read all the blocks in the container,
// because we cannot trust the container header's given length due to a
// bug somewhere in htslib
if (containerCount === 0) {
position = currentHeader._endPosition;
for (let j = 0; j < currentHeader.numBlocks; j++) {
const block = await this.readBlock(position);
position = block._endPosition;
}
}
else {
// otherwise, just traverse to the next container using the container's
// length
position += currentHeader._size + currentHeader.length;
}
containerCount += 1;
}
}
catch (e) {
containerCount--;
/* do nothing */
}
return containerCount;
}
getContainerAtPosition(position) {
return new index_ts_3.default(this, position);
}
async readBlockHeader(position) {
const { majorVersion } = await this.getDefinition();
const sectionParsers = (0, sectionParsers_ts_1.getSectionParsers)(majorVersion);
const { cramBlockHeader } = sectionParsers;
const buffer = await this.file.read(cramBlockHeader.maxLength, position);
return (0, util_ts_1.parseItem)(buffer, cramBlockHeader.parser, 0, position);
}
async _parseSection(section, position, size = section.maxLength, preReadBuffer) {
const buffer = preReadBuffer ?? (await this.file.read(size, position));
const data = (0, util_ts_1.parseItem)(buffer, section.parser, 0, position);
if (data._size !== size) {
throw new errors_ts_1.CramMalformedError(`section read error: requested size ${size} does not equal parsed size ${data._size}`);
}
return data;
}
async _uncompressPre(compressionMethod, inputBuffer, uncompressedSize) {
// console.log({ compressionMethod })
if (compressionMethod === 'gzip') {
return (0, unzip_ts_1.unzip)(inputBuffer);
}
else if (compressionMethod === 'bzip2') {
return (0, index_ts_2.decode)(inputBuffer);
}
else if (compressionMethod === 'lzma') {
const decompressedResponse = new Response(new xz_decompress_1.XzReadableStream(bufferToStream(inputBuffer)));
return new Uint8Array(await decompressedResponse.arrayBuffer());
}
else if (compressionMethod === 'rans') {
const outputBuffer = new Uint8Array(uncompressedSize);
(0, index_ts_1.default)(inputBuffer, outputBuffer);
return outputBuffer;
}
else if (compressionMethod === 'rans4x16') {
return htscodecs.r4x16_uncompress(inputBuffer);
}
else if (compressionMethod === 'arith') {
return htscodecs.arith_uncompress(inputBuffer);
}
else if (compressionMethod === 'fqzcomp') {
return htscodecs.fqzcomp_uncompress(inputBuffer);
}
else if (compressionMethod === 'tok3') {
return htscodecs.tok3_uncompress(inputBuffer);
}
else {
throw new errors_ts_1.CramUnimplementedError(`${compressionMethod} decompression not yet implemented`);
}
}
async _uncompress(compressionMethod, inputBuffer, uncompressedSize) {
const buf = await this._uncompressPre(compressionMethod, inputBuffer, uncompressedSize);
if (buf.length !== uncompressedSize) {
const ret = new Uint8Array(uncompressedSize);
ret.set(buf, 0);
return ret;
}
return buf;
}
async readBlock(position) {
const { majorVersion } = await this.getDefinition();
const sectionParsers = (0, sectionParsers_ts_1.getSectionParsers)(majorVersion);
const blockHeader = await this.readBlockHeader(position);
const blockContentPosition = blockHeader._endPosition;
const d = await this.file.read(blockHeader.compressedSize, blockContentPosition);
const uncompressedData = blockHeader.compressionMethod !== 'raw'
? await this._uncompress(blockHeader.compressionMethod, d, blockHeader.uncompressedSize)
: d;
const block = {
...blockHeader,
_endPosition: blockContentPosition,
contentPosition: blockContentPosition,
content: uncompressedData,
};
if (majorVersion >= 3) {
// parse the crc32
const crc = await this._parseSection(sectionParsers.cramBlockCrc32, blockContentPosition + blockHeader.compressedSize);
block.crc32 = crc.crc32;
// check the block data crc32
if (this.validateChecksums) {
await this.checkCrc32(position, blockHeader._size + blockHeader.compressedSize, crc.crc32, 'block data');
}
// make the endposition and size reflect the whole block
block._endPosition = crc._endPosition;
block._size =
block.compressedSize + sectionParsers.cramBlockCrc32.maxLength;
}
else {
block._endPosition = blockContentPosition + block.compressedSize;
block._size = block.compressedSize;
}
return block;
}
}
exports.default = CramFile;
'getDefinition getSectionParsers getSamHeader'.split(' ').forEach(method => {
(0, util_ts_1.tinyMemoize)(CramFile, method);
});
//# sourceMappingURL=file.js.map