@dpkit/file/build/stream/load.js

Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames

import { createReadStream } from "node:fs";
import { Readable, Transform } from "node:stream";
import { isRemotePath } from "@dpkit/core";
export async function loadFileStream(pathOrPaths, options) {
    const index = options?.index ?? 0;
    const paths = Array.isArray(pathOrPaths) ? pathOrPaths : [pathOrPaths];
    const path = paths[index];
    if (!path) {
        throw new Error(`Cannot stream resource ${path} at index ${index}`);
    }
    const isRemote = isRemotePath(path);
    const stream = isRemote
        ? await loadRemoteFileStream(path, options)
        : await loadLocalFileStream(path, options);
    return stream;
}
async function loadRemoteFileStream(path, options) {
    const response = await fetch(path);
    if (!response.body) {
        throw new Error(`Cannot stream remote resource: ${path}`);
    }
    let stream = Readable.fromWeb(response.body);
    if (options?.maxBytes) {
        stream = limitBytesStream(stream, options.maxBytes);
    }
    return stream;
}
async function loadLocalFileStream(path, options) {
    const end = options?.maxBytes ? options.maxBytes - 1 : undefined;
    return createReadStream(path, { end });
}
function limitBytesStream(inputStream, maxBytes) {
    let total = 0;
    return inputStream.pipe(new Transform({
        transform(chunk, _encoding, callback) {
            if (total >= maxBytes) {
                this.push(null);
                callback();
                return;
            }
            total += chunk.length;
            callback(null, chunk);
        },
    }));
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9hZC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3N0cmVhbS9sb2FkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLFNBQVMsQ0FBQTtBQUMxQyxPQUFPLEVBQUUsUUFBUSxFQUFFLFNBQVMsRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUNqRCxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sYUFBYSxDQUFBO0FBRTFDLE1BQU0sQ0FBQyxLQUFLLFVBQVUsY0FBYyxDQUNsQyxXQUE4QixFQUM5QixPQUdDO0lBRUQsTUFBTSxLQUFLLEdBQUcsT0FBTyxFQUFFLEtBQUssSUFBSSxDQUFDLENBQUE7SUFFakMsTUFBTSxLQUFLLEdBQUcsS0FBSyxDQUFDLE9BQU8sQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLFdBQVcsQ0FBQyxDQUFBO0lBQ3RFLE1BQU0sSUFBSSxHQUFHLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQTtJQUV6QixJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDVixNQUFNLElBQUksS0FBSyxDQUFDLDBCQUEwQixJQUFJLGFBQWEsS0FBSyxFQUFFLENBQUMsQ0FBQTtJQUNyRSxDQUFDO0lBRUQsTUFBTSxRQUFRLEdBQUcsWUFBWSxDQUFDLElBQUksQ0FBQyxDQUFBO0lBQ25DLE1BQU0sTUFBTSxHQUFHLFFBQVE7UUFDckIsQ0FBQyxDQUFDLE1BQU0sb0JBQW9CLENBQUMsSUFBSSxFQUFFLE9BQU8sQ0FBQztRQUMzQyxDQUFDLENBQUMsTUFBTSxtQkFBbUIsQ0FBQyxJQUFJLEVBQUUsT0FBTyxDQUFDLENBQUE7SUFFNUMsT0FBTyxNQUFNLENBQUE7QUFDZixDQUFDO0FBRUQsS0FBSyxVQUFVLG9CQUFvQixDQUNqQyxJQUFZLEVBQ1osT0FBK0I7SUFFL0IsTUFBTSxRQUFRLEdBQUcsTUFBTSxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUE7SUFDbEMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUNuQixNQUFNLElBQUksS0FBSyxDQUFDLGtDQUFrQyxJQUFJLEVBQUUsQ0FBQyxDQUFBO0lBQzNELENBQUM7SUFFRCxJQUFJLE1BQU0sR0FBRyxRQUFRLENBQUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQTtJQUU1QyxJQUFJLE9BQU8sRUFBRSxRQUFRLEVBQUUsQ0FBQztRQUN0QixNQUFNLEdBQUcsZ0JBQWdCLENBQUMsTUFBTSxFQUFFLE9BQU8sQ0FBQyxRQUFRLENBQUMsQ0FBQTtJQUNyRCxDQUFDO0lBRUQsT0FBTyxNQUFNLENBQUE7QUFDZixDQUFDO0FBRUQsS0FBSyxVQUFVLG1CQUFtQixDQUNoQyxJQUFZLEVBQ1osT0FBK0I7SUFFL0IsTUFBTSxHQUFHLEdBQUcsT0FBTyxFQUFFLFFBQVEsQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFDLFFBQVEsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQTtJQUNoRSxPQUFPLGdCQUFnQixDQUFDLElBQUksRUFBRSxFQUFFLEdBQUcsRUFBRSxDQUFDLENBQUE7QUFDeEMsQ0FBQztBQUVELFNBQVMsZ0JBQWdCLENBQUMsV0FBcUIsRUFBRSxRQUFnQjtJQUMvRCxJQUFJLEtBQUssR0FBRyxDQUFDLENBQUE7SUFDYixPQUFPLFdBQVcsQ0FBQyxJQUFJLENBQ3JCLElBQUksU0FBUyxDQUFDO1FBQ1osU0FBUyxDQUFDLEtBQUssRUFBRSxTQUFTLEVBQUUsUUFBUTtZQUNsQyxJQUFJLEtBQUssSUFBSSxRQUFRLEVBQUUsQ0FBQztnQkFDdEIsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQTtnQkFDZixRQUFRLEVBQUUsQ0FBQTtnQkFDVixPQUFNO1lBQ1IsQ0FBQztZQUVELEtBQUssSUFBSSxLQUFLLENBQUMsTUFBTSxDQUFBO1lBQ3JCLFFBQVEsQ0FBQyxJQUFJLEVBQUUsS0FBSyxDQUFDLENBQUE7UUFDdkIsQ0FBQztLQUNGLENBQUMsQ0FDSCxDQUFBO0FBQ0gsQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbImltcG9ydCB7IGNyZWF0ZVJlYWRTdHJlYW0gfSBmcm9tIFwibm9kZTpmc1wiXG5pbXBvcnQgeyBSZWFkYWJsZSwgVHJhbnNmb3JtIH0gZnJvbSBcIm5vZGU6c3RyZWFtXCJcbmltcG9ydCB7IGlzUmVtb3RlUGF0aCB9IGZyb20gXCJAZHBraXQvY29yZVwiXG5cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBsb2FkRmlsZVN0cmVhbShcbiAgcGF0aE9yUGF0aHM6IHN0cmluZyB8IHN0cmluZ1tdLFxuICBvcHRpb25zPzoge1xuICAgIGluZGV4PzogbnVtYmVyXG4gICAgbWF4Qnl0ZXM/OiBudW1iZXJcbiAgfSxcbikge1xuICBjb25zdCBpbmRleCA9IG9wdGlvbnM/LmluZGV4ID8/IDBcblxuICBjb25zdCBwYXRocyA9IEFycmF5LmlzQXJyYXkocGF0aE9yUGF0aHMpID8gcGF0aE9yUGF0aHMgOiBbcGF0aE9yUGF0aHNdXG4gIGNvbnN0IHBhdGggPSBwYXRoc1tpbmRleF1cblxuICBpZiAoIXBhdGgpIHtcbiAgICB0aHJvdyBuZXcgRXJyb3IoYENhbm5vdCBzdHJlYW0gcmVzb3VyY2UgJHtwYXRofSBhdCBpbmRleCAke2luZGV4fWApXG4gIH1cblxuICBjb25zdCBpc1JlbW90ZSA9IGlzUmVtb3RlUGF0aChwYXRoKVxuICBjb25zdCBzdHJlYW0gPSBpc1JlbW90ZVxuICAgID8gYXdhaXQgbG9hZFJlbW90ZUZpbGVTdHJlYW0ocGF0aCwgb3B0aW9ucylcbiAgICA6IGF3YWl0IGxvYWRMb2NhbEZpbGVTdHJlYW0ocGF0aCwgb3B0aW9ucylcblxuICByZXR1cm4gc3RyZWFtXG59XG5cbmFzeW5jIGZ1bmN0aW9uIGxvYWRSZW1vdGVGaWxlU3RyZWFtKFxuICBwYXRoOiBzdHJpbmcsXG4gIG9wdGlvbnM/OiB7IG1heEJ5dGVzPzogbnVtYmVyIH0sXG4pIHtcbiAgY29uc3QgcmVzcG9uc2UgPSBhd2FpdCBmZXRjaChwYXRoKVxuICBpZiAoIXJlc3BvbnNlLmJvZHkpIHtcbiAgICB0aHJvdyBuZXcgRXJyb3IoYENhbm5vdCBzdHJlYW0gcmVtb3RlIHJlc291cmNlOiAke3BhdGh9YClcbiAgfVxuXG4gIGxldCBzdHJlYW0gPSBSZWFkYWJsZS5mcm9tV2ViKHJlc3BvbnNlLmJvZHkpXG5cbiAgaWYgKG9wdGlvbnM/Lm1heEJ5dGVzKSB7XG4gICAgc3RyZWFtID0gbGltaXRCeXRlc1N0cmVhbShzdHJlYW0sIG9wdGlvbnMubWF4Qnl0ZXMpXG4gIH1cblxuICByZXR1cm4gc3RyZWFtXG59XG5cbmFzeW5jIGZ1bmN0aW9uIGxvYWRMb2NhbEZpbGVTdHJlYW0oXG4gIHBhdGg6IHN0cmluZyxcbiAgb3B0aW9ucz86IHsgbWF4Qnl0ZXM/OiBudW1iZXIgfSxcbikge1xuICBjb25zdCBlbmQgPSBvcHRpb25zPy5tYXhCeXRlcyA/IG9wdGlvbnMubWF4Qnl0ZXMgLSAxIDogdW5kZWZpbmVkXG4gIHJldHVybiBjcmVhdGVSZWFkU3RyZWFtKHBhdGgsIHsgZW5kIH0pXG59XG5cbmZ1bmN0aW9uIGxpbWl0Qnl0ZXNTdHJlYW0oaW5wdXRTdHJlYW06IFJlYWRhYmxlLCBtYXhCeXRlczogbnVtYmVyKSB7XG4gIGxldCB0b3RhbCA9IDBcbiAgcmV0dXJuIGlucHV0U3RyZWFtLnBpcGUoXG4gICAgbmV3IFRyYW5zZm9ybSh7XG4gICAgICB0cmFuc2Zvcm0oY2h1bmssIF9lbmNvZGluZywgY2FsbGJhY2spIHtcbiAgICAgICAgaWYgKHRvdGFsID49IG1heEJ5dGVzKSB7XG4gICAgICAgICAgdGhpcy5wdXNoKG51bGwpXG4gICAgICAgICAgY2FsbGJhY2soKVxuICAgICAgICAgIHJldHVyblxuICAgICAgICB9XG5cbiAgICAgICAgdG90YWwgKz0gY2h1bmsubGVuZ3RoXG4gICAgICAgIGNhbGxiYWNrKG51bGwsIGNodW5rKVxuICAgICAgfSxcbiAgICB9KSxcbiAgKVxufVxuIl19