app-package-builder/out/ContentDefinedChunker.js

Idea is very simple — in the runtime we don't need to process or understand archive format. Wwe just need to know file data ranges. Where file data begins and where ends.

"use strict";

Object.defineProperty(exports, "__esModule", {
    value: true
});
exports.ContentDefinedChunker = undefined;

var _bluebirdLst;

function _load_bluebirdLst() {
    return _bluebirdLst = require("bluebird-lst");
}

var _fsExtraP;

function _load_fsExtraP() {
    return _fsExtraP = require("fs-extra-p");
}

var _rabinBindings;

function _load_rabinBindings() {
    return _rabinBindings = require("rabin-bindings");
}

class ContentDefinedChunker {
    computeChunks(fd, start, end, name) {
        return (0, (_bluebirdLst || _load_bluebirdLst()).coroutine)(function* () {
            const fileSize = end - start;
            const buffer = Buffer.allocUnsafe(Math.min(4 * 1024 * 1024, fileSize));
            const rabin = (0, (_rabinBindings || _load_rabinBindings()).Rabin)();
            const avgBits = 12;
            const min = 8 * 1024;
            // see note in the nsis.ts about archive dict size
            const max = 32 * 1024;
            rabin.configure(avgBits, min, max);
            const checksums = [];
            const allSizes = [];
            let tailBufferData = null;
            let readOffset = start;
            while (true) {
                const actualBufferSize = Math.min(end - readOffset, buffer.length);
                yield (0, (_fsExtraP || _load_fsExtraP()).read)(fd, buffer, 0, actualBufferSize, readOffset);
                const dataBuffer = buffer.length === actualBufferSize ? buffer : buffer.slice(0, actualBufferSize);
                const sizes = [];
                rabin.fingerprint([dataBuffer], sizes);
                let chunkStart = 0;
                for (const size of sizes) {
                    allSizes.push(size);
                    let chunkEnd = chunkStart + size;
                    const hash = new Blake2s(CHECKSUM_OUTPUT_LENGTH);
                    if (tailBufferData !== null) {
                        hash.update(tailBufferData);
                        // if there is the tail data (already processed by rabin data), first size includes it
                        chunkEnd -= tailBufferData.length;
                        tailBufferData = null;
                    }
                    hash.update(dataBuffer, chunkStart, size);
                    checksums.push(digest(hash));
                    chunkStart = chunkEnd;
                }
                const tailSize = actualBufferSize - chunkStart;
                if (tailSize !== 0) {
                    if (tailBufferData !== null) {
                        throw new Error(`Internal error (${name}): tailBufferData must be null`);
                    }
                    tailBufferData = dataBuffer.slice(chunkStart, chunkStart + tailSize);
                }
                readOffset += actualBufferSize;
                if (readOffset >= end) {
                    if (tailBufferData !== null) {
                        allSizes.push(tailSize);
                        checksums.push(computeChecksum(tailBufferData));
                    }
                    break;
                } else if (tailBufferData !== null) {
                    // copy data
                    tailBufferData = Buffer.from(tailBufferData);
                }
            }
            const totalSize = allSizes.reduce(function (accumulator, currentValue) {
                return accumulator + currentValue;
            });
            if (totalSize !== fileSize) {
                throw new Error(`Internal error (${name}): size mismatch: expected: ${fileSize}, got: ${totalSize}`);
            }
            return { checksums, sizes: allSizes };
        })();
    }
}
exports.ContentDefinedChunker = ContentDefinedChunker; // base64 - should be divisible by 3 to avoid paddings

const CHECKSUM_OUTPUT_LENGTH = 18;
const Blake2s = require("../blake2s.js");
function computeChecksum(chunk) {
    const hash = new Blake2s(CHECKSUM_OUTPUT_LENGTH);
    hash.update(chunk);
    // node-base91 doesn't make a lot of sense - 29KB vs 30KB Because for base64 string value in the yml never escaped, but node-base91 often escaped (single quotes) and it adds extra 2 symbols.
    return digest(hash);
}
function digest(hash) {
    return hash.digest().toString("base64");
}
//# sourceMappingURL=ContentDefinedChunker.js.map