@dpkit/file
Version:
Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames
46 lines • 6.16 kB
JavaScript
import { createReadStream } from "node:fs";
import { Readable, Transform } from "node:stream";
import { isRemotePath } from "@dpkit/core";
export async function loadFileStream(pathOrPaths, options) {
const index = options?.index ?? 0;
const paths = Array.isArray(pathOrPaths) ? pathOrPaths : [pathOrPaths];
const path = paths[index];
if (!path) {
throw new Error(`Cannot stream resource ${path} at index ${index}`);
}
const isRemote = isRemotePath(path);
const stream = isRemote
? await loadRemoteFileStream(path)
: await loadLocalFileStream(path);
return stream;
}
// TODO: support maxBytes
async function loadRemoteFileStream(path, options) {
const response = await fetch(path);
if (!response.body) {
throw new Error(`Cannot stream remote resource: ${path}`);
}
let stream = Readable.fromWeb(response.body);
if (options?.maxBytes) {
stream = limitBytesStream(stream, options.maxBytes);
}
return stream;
}
async function loadLocalFileStream(path, options) {
return createReadStream(path, { end: options?.maxBytes });
}
function limitBytesStream(inputStream, maxBytes) {
let total = 0;
return inputStream.pipe(new Transform({
transform(chunk, _encoding, callback) {
total += chunk.length;
if (total > maxBytes) {
callback(new Error("Byte limit exceeded"));
}
else {
callback(null, chunk);
}
},
}));
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9hZC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3N0cmVhbS9sb2FkLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLFNBQVMsQ0FBQTtBQUMxQyxPQUFPLEVBQUUsUUFBUSxFQUFFLFNBQVMsRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUNqRCxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sYUFBYSxDQUFBO0FBRTFDLE1BQU0sQ0FBQyxLQUFLLFVBQVUsY0FBYyxDQUNsQyxXQUE4QixFQUM5QixPQUdDO0lBRUQsTUFBTSxLQUFLLEdBQUcsT0FBTyxFQUFFLEtBQUssSUFBSSxDQUFDLENBQUE7SUFFakMsTUFBTSxLQUFLLEdBQUcsS0FBSyxDQUFDLE9BQU8sQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLFdBQVcsQ0FBQyxDQUFBO0lBQ3RFLE1BQU0sSUFBSSxHQUFHLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQTtJQUV6QixJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDVixNQUFNLElBQUksS0FBSyxDQUFDLDBCQUEwQixJQUFJLGFBQWEsS0FBSyxFQUFFLENBQUMsQ0FBQTtJQUNyRSxDQUFDO0lBRUQsTUFBTSxRQUFRLEdBQUcsWUFBWSxDQUFDLElBQUksQ0FBQyxDQUFBO0lBQ25DLE1BQU0sTUFBTSxHQUFHLFFBQVE7UUFDckIsQ0FBQyxDQUFDLE1BQU0sb0JBQW9CLENBQUMsSUFBSSxDQUFDO1FBQ2xDLENBQUMsQ0FBQyxNQUFNLG1CQUFtQixDQUFDLElBQUksQ0FBQyxDQUFBO0lBRW5DLE9BQU8sTUFBTSxDQUFBO0FBQ2YsQ0FBQztBQUVELHlCQUF5QjtBQUN6QixLQUFLLFVBQVUsb0JBQW9CLENBQ2pDLElBQVksRUFDWixPQUErQjtJQUUvQixNQUFNLFFBQVEsR0FBRyxNQUFNLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQTtJQUNsQyxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ25CLE1BQU0sSUFBSSxLQUFLLENBQUMsa0NBQWtDLElBQUksRUFBRSxDQUFDLENBQUE7SUFDM0QsQ0FBQztJQUVELElBQUksTUFBTSxHQUFHLFFBQVEsQ0FBQyxPQUFPLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFBO0lBRTVDLElBQUksT0FBTyxFQUFFLFFBQVEsRUFBRSxDQUFDO1FBQ3RCLE1BQU0sR0FBRyxnQkFBZ0IsQ0FBQyxNQUFNLEVBQUUsT0FBTyxDQUFDLFFBQVEsQ0FBQyxDQUFBO0lBQ3JELENBQUM7SUFFRCxPQUFPLE1BQU0sQ0FBQTtBQUNmLENBQUM7QUFFRCxLQUFLLFVBQVUsbUJBQW1CLENBQ2hDLElBQVksRUFDWixPQUErQjtJQUUvQixPQUFPLGdCQUFnQixDQUFDLElBQUksRUFBRSxFQUFFLEdBQUcsRUFBRSxPQUFPLEVBQUUsUUFBUSxFQUFFLENBQUMsQ0FBQTtBQUMzRCxDQUFDO0FBRUQsU0FBUyxnQkFBZ0IsQ0FBQyxXQUFxQixFQUFFLFFBQWdCO0lBQy9ELElBQUksS0FBSyxHQUFHLENBQUMsQ0FBQTtJQUNiLE9BQU8sV0FBVyxDQUFDLElBQUksQ0FDckIsSUFBSSxTQUFTLENBQUM7UUFDWixTQUFTLENBQUMsS0FBSyxFQUFFLFNBQVMsRUFBRSxRQUFRO1lBQ2xDLEtBQUssSUFBSSxLQUFLLENBQUMsTUFBTSxDQUFBO1lBQ3JCLElBQUksS0FBSyxHQUFHLFFBQVEsRUFBRSxDQUFDO2dCQUNyQixRQUFRLENBQUMsSUFBSSxLQUFLLENBQUMscUJBQXFCLENBQUMsQ0FBQyxDQUFBO1lBQzVDLENBQUM7aUJBQU0sQ0FBQztnQkFDTixRQUFRLENBQUMsSUFBSSxFQUFFLEtBQUssQ0FBQyxDQUFBO1lBQ3ZCLENBQUM7UUFDSCxDQUFDO0tBQ0YsQ0FBQyxDQUNILENBQUE7QUFDSCxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHsgY3JlYXRlUmVhZFN0cmVhbSB9IGZyb20gXCJub2RlOmZzXCJcbmltcG9ydCB7IFJlYWRhYmxlLCBUcmFuc2Zvcm0gfSBmcm9tIFwibm9kZTpzdHJlYW1cIlxuaW1wb3J0IHsgaXNSZW1vdGVQYXRoIH0gZnJvbSBcIkBkcGtpdC9jb3JlXCJcblxuZXhwb3J0IGFzeW5jIGZ1bmN0aW9uIGxvYWRGaWxlU3RyZWFtKFxuICBwYXRoT3JQYXRoczogc3RyaW5nIHwgc3RyaW5nW10sXG4gIG9wdGlvbnM/OiB7XG4gICAgaW5kZXg/OiBudW1iZXJcbiAgICBtYXhCeXRlcz86IG51bWJlclxuICB9LFxuKSB7XG4gIGNvbnN0IGluZGV4ID0gb3B0aW9ucz8uaW5kZXggPz8gMFxuXG4gIGNvbnN0IHBhdGhzID0gQXJyYXkuaXNBcnJheShwYXRoT3JQYXRocykgPyBwYXRoT3JQYXRocyA6IFtwYXRoT3JQYXRoc11cbiAgY29uc3QgcGF0aCA9IHBhdGhzW2luZGV4XVxuXG4gIGlmICghcGF0aCkge1xuICAgIHRocm93IG5ldyBFcnJvcihgQ2Fubm90IHN0cmVhbSByZXNvdXJjZSAke3BhdGh9IGF0IGluZGV4ICR7aW5kZXh9YClcbiAgfVxuXG4gIGNvbnN0IGlzUmVtb3RlID0gaXNSZW1vdGVQYXRoKHBhdGgpXG4gIGNvbnN0IHN0cmVhbSA9IGlzUmVtb3RlXG4gICAgPyBhd2FpdCBsb2FkUmVtb3RlRmlsZVN0cmVhbShwYXRoKVxuICAgIDogYXdhaXQgbG9hZExvY2FsRmlsZVN0cmVhbShwYXRoKVxuXG4gIHJldHVybiBzdHJlYW1cbn1cblxuLy8gVE9ETzogc3VwcG9ydCBtYXhCeXRlc1xuYXN5bmMgZnVuY3Rpb24gbG9hZFJlbW90ZUZpbGVTdHJlYW0oXG4gIHBhdGg6IHN0cmluZyxcbiAgb3B0aW9ucz86IHsgbWF4Qnl0ZXM/OiBudW1iZXIgfSxcbikge1xuICBjb25zdCByZXNwb25zZSA9IGF3YWl0IGZldGNoKHBhdGgpXG4gIGlmICghcmVzcG9uc2UuYm9keSkge1xuICAgIHRocm93IG5ldyBFcnJvcihgQ2Fubm90IHN0cmVhbSByZW1vdGUgcmVzb3VyY2U6ICR7cGF0aH1gKVxuICB9XG5cbiAgbGV0IHN0cmVhbSA9IFJlYWRhYmxlLmZyb21XZWIocmVzcG9uc2UuYm9keSlcblxuICBpZiAob3B0aW9ucz8ubWF4Qnl0ZXMpIHtcbiAgICBzdHJlYW0gPSBsaW1pdEJ5dGVzU3RyZWFtKHN0cmVhbSwgb3B0aW9ucy5tYXhCeXRlcylcbiAgfVxuXG4gIHJldHVybiBzdHJlYW1cbn1cblxuYXN5bmMgZnVuY3Rpb24gbG9hZExvY2FsRmlsZVN0cmVhbShcbiAgcGF0aDogc3RyaW5nLFxuICBvcHRpb25zPzogeyBtYXhCeXRlcz86IG51bWJlciB9LFxuKSB7XG4gIHJldHVybiBjcmVhdGVSZWFkU3RyZWFtKHBhdGgsIHsgZW5kOiBvcHRpb25zPy5tYXhCeXRlcyB9KVxufVxuXG5mdW5jdGlvbiBsaW1pdEJ5dGVzU3RyZWFtKGlucHV0U3RyZWFtOiBSZWFkYWJsZSwgbWF4Qnl0ZXM6IG51bWJlcikge1xuICBsZXQgdG90YWwgPSAwXG4gIHJldHVybiBpbnB1dFN0cmVhbS5waXBlKFxuICAgIG5ldyBUcmFuc2Zvcm0oe1xuICAgICAgdHJhbnNmb3JtKGNodW5rLCBfZW5jb2RpbmcsIGNhbGxiYWNrKSB7XG4gICAgICAgIHRvdGFsICs9IGNodW5rLmxlbmd0aFxuICAgICAgICBpZiAodG90YWwgPiBtYXhCeXRlcykge1xuICAgICAgICAgIGNhbGxiYWNrKG5ldyBFcnJvcihcIkJ5dGUgbGltaXQgZXhjZWVkZWRcIikpXG4gICAgICAgIH0gZWxzZSB7XG4gICAgICAgICAgY2FsbGJhY2sobnVsbCwgY2h1bmspXG4gICAgICAgIH1cbiAgICAgIH0sXG4gICAgfSksXG4gIClcbn1cbiJdfQ==