snappystream
Version:
Framed Snappy streams
185 lines (184 loc) • 5.8 kB
JavaScript
// lib/snappystreams.ts
import { compress, uncompress } from "snappy";
import { crc32c } from "@node-rs/crc32";
import stream from "stream";
var CHUNKS = {
streamIdentifier: 255,
compressedData: 0,
uncompressedData: 1,
padding: 254,
unskippable: (v) => v >= 2 && v <= 127 ? v : -1
};
var STREAM_IDENTIFIER = Buffer.from([
255,
6,
0,
0,
115,
78,
97,
80,
112,
89
]);
var MAX_FRAME_DATA_SIZE = 65536;
var checksumMask = (data) => {
const c = crc32c(data);
const result = new Uint32Array(1);
result[0] = (c >>> 15 | c * Math.pow(2, 17)) + 2726488792;
return result[0];
};
var SnappyStream = class extends stream.Transform {
constructor(options) {
super(options);
this.push(STREAM_IDENTIFIER);
}
// No buffering of data before producing a compressed frame. If the data size
// exceeds the size of a frame, then it will automatically be split across
// frames per the Snappy frame spec.
_transform(data, _, callback) {
const out = Buffer.from(data);
const dataChunks = [];
for (let offset = 0; offset < out.length / MAX_FRAME_DATA_SIZE; offset++) {
const start = offset * MAX_FRAME_DATA_SIZE;
const end = start + MAX_FRAME_DATA_SIZE;
dataChunks.push(out.subarray(start, end));
}
Promise.all(dataChunks.map((chunk) => compress(chunk))).then((compressedDataChunks) => {
const frameChunks = [];
for (let i = 0; i < dataChunks.length; i++) {
const chunkData = dataChunks[i];
const frameData = compressedDataChunks[i];
const frameStart = Buffer.alloc(8);
let headerType = CHUNKS.compressedData;
let payload = frameData;
if (frameData.length >= chunkData.length - chunkData.length / 8) {
headerType = CHUNKS.uncompressedData;
payload = chunkData;
}
frameStart.writeUInt8(headerType, 0);
frameStart.writeUintLE(payload.length + 4, 1, 3);
frameStart.writeUInt32LE(checksumMask(chunkData), 4);
frameChunks.push(frameStart);
frameChunks.push(payload);
}
this.push(Buffer.concat(frameChunks));
return callback();
}).catch((err) => {
callback(err);
});
}
};
var UnsnappyStream = class extends stream.Transform {
constructor(verifyChecksums = false, options = {}) {
super(options);
this.verifyChecksums = verifyChecksums;
this.identifierFound = false;
this.frameBuffer = null;
}
// Returns snappy compressed payload. Throws an error if the checksum fails
// provided stream is checking checksums.
framePayload(frame) {
const frameLength = frame.readUIntLE(1, 3);
return frame.subarray(8, frameLength + 4);
}
frameMask(frame) {
return frame.readUInt32LE(4);
}
// Data contains at least one full frame.
hasFrame(data) {
return data.length > 4 && data.readUIntLE(1, 3) + 4 <= data.length;
}
// Return the buffer starting at the next frame. It assumes that a full frame
// exists within data.
toNextFrame(data) {
const frameLength = data.readUIntLE(1, 3);
return data.subarray(4 + frameLength);
}
verify(mask, data) {
if (this.verifyChecksums && checksumMask(data) !== mask) {
throw new Error("Frame failed checksum");
}
}
async processChunks(chunks, done) {
const verify = this.verify.bind(this);
try {
const uncompressFrames = await Promise.all(
chunks.map(async (chunk) => {
const [frameType, mask, payload] = chunk;
const uncompressedData = frameType === CHUNKS.uncompressedData ? payload : await uncompress(payload);
verify(mask, uncompressedData);
return Buffer.isBuffer(uncompressedData) ? uncompressedData : Buffer.from(uncompressedData);
})
);
this.push(Buffer.concat(uncompressFrames));
if (done) {
done();
}
} catch (err) {
this.emit("error", err);
}
}
_transform(data, encoding, done) {
const chunks = [];
let _data = encoding ? Buffer.from(data, encoding) : data;
if (this.frameBuffer) {
_data = Buffer.concat([this.frameBuffer, _data]);
}
this.frameBuffer = null;
if (!this.identifierFound && _data.readUInt8(0) !== CHUNKS.streamIdentifier) {
return this.emit("error", new Error("Missing snappy stream identifier"));
}
while (this.hasFrame(_data)) {
const frameId = _data.readUInt8(0);
try {
switch (frameId) {
case CHUNKS.streamIdentifier:
if (_data.subarray(0, 10).toString() !== STREAM_IDENTIFIER.toString()) {
return this.emit("error", new Error("Invalid stream identifier"));
}
this.identifierFound = true;
break;
case CHUNKS.compressedData:
chunks.push([
CHUNKS.compressedData,
this.frameMask(_data),
this.framePayload(_data)
]);
break;
case CHUNKS.uncompressedData:
chunks.push([
CHUNKS.uncompressedData,
this.frameMask(_data),
this.framePayload(_data)
]);
break;
case CHUNKS.unskippable(frameId):
return this.emit(
"error",
new Error("Encountered unskippable frame")
);
}
} catch (err) {
return this.emit("error", err);
}
_data = this.toNextFrame(_data);
}
if (_data.length) {
this.frameBuffer = _data;
}
if (chunks.length) {
return this.processChunks(chunks, done);
} else {
return done();
}
}
_flush(done) {
const err = this.frameBuffer?.length ? new Error("Failed to decompress Snappy stream") : void 0;
done(err);
}
};
export {
SnappyStream,
UnsnappyStream
};