azure-kusto-ingest
Version:
Azure Data Explorer Ingestion SDK
76 lines • 2.82 kB
JavaScript
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
import { v4 as uuidv4 } from "uuid";
import uuidValidate from "uuid-validate";
import { BlobClient } from "@azure/storage-blob";
export var CompressionType;
(function (CompressionType) {
CompressionType["ZIP"] = ".zip";
CompressionType["GZIP"] = ".gz";
CompressionType["None"] = "";
})(CompressionType || (CompressionType = {}));
export const getSourceId = (sourceId) => {
if (sourceId) {
if (!uuidValidate(sourceId, 4)) {
throw Error("sourceId is not a valid uuid/v4");
}
return sourceId;
}
return uuidv4();
};
export function shouldCompressFileByExtension(extension) {
return !(extension === ".avro" || extension === ".apacheavro" || extension === ".parquet" || extension === ".sstream" || extension === ".orc");
}
export class AbstractDescriptor {
constructor(sourceId = null, size = null) {
this.sourceId = sourceId;
this.size = size;
this.sourceId = getSourceId(sourceId);
}
}
export class StreamDescriptor extends AbstractDescriptor {
/**
* Use Readable for Node.js and ArrayBuffer in browser
*/
constructor(stream, sourceId = null, compressionType = CompressionType.None, size = null) {
super(sourceId, size);
this.stream = stream;
this.compressionType = compressionType;
}
merge(other) {
this.size = other.size;
this.compressionType = other.compressionType;
this.sourceId = other.sourceId;
return this;
}
// Currently streams are not compressed by us
getCompressionSuffix() {
return this.compressionType ? `.${this.compressionType}` : "";
}
}
export class BlobDescriptor extends AbstractDescriptor {
constructor(path, size = null, sourceId = null) {
super(sourceId, size);
this.path = path;
}
async fillSize() {
if (!this.size) {
const blobClient = new BlobClient(this.path);
const blobProps = await blobClient.getProperties();
const length = blobProps.contentLength;
if (length !== undefined) {
if (length === 0) {
throw new Error("Empty blob.");
}
this.size = length;
}
}
}
}
export const generateBlobName = (desc, props) => {
const extension = desc instanceof StreamDescriptor ? null : `${desc.name ? "__" + desc.name : `${desc.extension ? "." + desc.extension : ""}`}`;
const formatSuffix = props.format ? `.${props.format}` : ".csv";
const compressionString = desc.getCompressionSuffix();
return `${props.database}__${props.table}__${desc.sourceId}${extension || formatSuffix}${compressionString}`;
};
//# sourceMappingURL=descriptors.js.map