azure-kusto-ingest/dist-esm/src/fileDescriptor.js

Azure Data Explorer Ingestion SDK

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
import zlib from "zlib";
import pathlib from "path";
import fs from "fs";
import { file as tmpFile } from "tmp-promise";
import { promisify } from "util";
import { AbstractDescriptor, CompressionType, shouldCompressFileByExtension } from "./descriptors.js";
import { shouldCompressFileByFormat } from "./ingestionProperties.js";
/**
 * Describes a file to be ingested. Use string to describe a local path in Node.JS and Blob object in browsers
 */
export class FileDescriptor extends AbstractDescriptor {
    constructor(
    /**
     * Use string in Node.JS and Blob in browser
     */
    file, sourceId = null, size = null, compressionType = CompressionType.None, extension, // Extracted from file name by default
    name) {
        super(sourceId, size);
        this.file = file;
        this.extension = extension;
        this.name = name;
        this.compressionType = compressionType;
        this.name = name ? name : pathlib.basename(this.file);
        this.extension = extension ? extension : pathlib.extname(this.file).toLowerCase();
        this.zipped = compressionType !== CompressionType.None || this.extension === ".gz" || this.extension === ".zip";
        this.shouldNotCompress = !shouldCompressFileByExtension(this.extension);
    }
    async _gzip() {
        const { path, cleanup } = await tmpFile({ postfix: ".gz", keep: false });
        this.cleanupTmp = cleanup;
        const zipper = zlib.createGzip();
        const input = fs.createReadStream(this.file, { autoClose: true });
        const output = fs.createWriteStream(path);
        await new Promise((resolve, reject) => {
            input
                .pipe(zipper)
                .pipe(output)
                .on("error", (err) => {
                reject(err);
            });
            output.once("close", () => {
                resolve(null);
            });
        });
        return path;
    }
    async prepare(ingestionProperties) {
        const shouldNotCompressByFormat = !shouldCompressFileByFormat(ingestionProperties);
        if (this.zipped || this.shouldNotCompress || shouldNotCompressByFormat) {
            const estimatedCompressionModifier = 11;
            await this._calculateSize(estimatedCompressionModifier);
            return this.file;
        }
        const path = await this._gzip();
        await this._calculateSize();
        return path;
    }
    async _calculateSize(modifier = 1) {
        if (this.size == null || this.size <= 0) {
            const asyncStat = promisify(fs.stat);
            this.size = (await asyncStat(this.file)).size * modifier;
        }
    }
    async cleanup() {
        if (this.cleanupTmp) {
            await this.cleanupTmp();
        }
    }
    getCompressionSuffix() {
        return this.compressionType ? `.${this.compressionType}` : ".gz";
    }
}
//# sourceMappingURL=fileDescriptor.js.map