azure-kusto-ingest/dist-esm/src/streamUtils.js

Azure Data Explorer Ingestion SDK

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
import fs from "fs";
import { PassThrough } from "stream";
import streamify from "stream-array";
import { CompressionType, StreamDescriptor } from "./descriptors.js";
// Returns fs.ReadStream for node and NodeJS.ReadableStream in browser
export const fileToStream = (fileDescriptor) => {
    const streamFs = fs.createReadStream(fileDescriptor.file);
    const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;
    return Promise.resolve(new StreamDescriptor(streamFs, fileDescriptor.sourceId, compressionType));
};
// Used in managed streaming where we buffer the file to memory for retries
export const tryFileToBuffer = async (fileDescriptor) => {
    try {
        const buffer = fs.readFileSync(fileDescriptor.file);
        const compressionType = fileDescriptor.zipped ? CompressionType.GZIP : CompressionType.None;
        return new StreamDescriptor(buffer, fileDescriptor.sourceId, compressionType);
    }
    catch (error) {
        return await fileToStream(fileDescriptor);
    }
};
const mergeStreams = (...streams) => {
    let pass = new PassThrough();
    let waiting = streams.length;
    for (const stream of streams) {
        pass = stream.pipe(pass, { end: false });
        stream.once("end", () => --waiting === 0 && pass.emit("end"));
    }
    return pass;
};
export const tryStreamToArray = async (stream, maxBufferSize) => {
    if (stream instanceof Buffer) {
        return stream;
    }
    return await new Promise((resolve, reject) => {
        const result = [];
        const endListener = () => resolve(Buffer.concat(result));
        const dataHandler = (chunk) => {
            try {
                result.push(chunk);
                if (result.reduce((sum, b) => sum + b.length, 0) > maxBufferSize) {
                    stream.removeListener("data", dataHandler);
                    stream.removeListener("end", endListener);
                    resolve(mergeStreams(streamify(result), stream));
                }
            }
            catch (e) {
                reject(e);
            }
        };
        stream.on("data", dataHandler);
        stream.on("end", endListener);
    });
};
export const readableToStream = (stream) => {
    return streamify([stream]);
};
//# sourceMappingURL=streamUtils.js.map