azure-kusto-ingest
Version:
Azure Data Explorer Ingestion SDK
249 lines • 9.71 kB
JavaScript
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
import { IngestionPropertiesValidationError } from "./errors.js";
/**
* Data formats supported for Kusto ingestion.
*/
export var DataFormat;
(function (DataFormat) {
/**
* Comma-separated value.
*/
DataFormat["CSV"] = "csv";
/**
* Tab-separated value.
*/
DataFormat["TSV"] = "tsv";
/**
* Semicolon-separated value (the unique Azure Storage log format).
*/
DataFormat["SCSV"] = "scsv";
/**
* Start-Of-Header (CTRL-A)-separated value.
*/
DataFormat["SOHSV"] = "sohsv";
/**
* Pipeline-separated value (used by Cosmos).
*/
DataFormat["PSV"] = "psv";
/**
* Each record is a line and has just one field.
*/
DataFormat["TXT"] = "txt";
/**
* Whole stream is a single record with a single field.
*/
DataFormat["RAW"] = "raw";
/**
* Tab-separated value with '\' escaping character.
*/
DataFormat["TSVE"] = "tsve";
/**
* Data is in a JSON format, each line is record with a single JSON value.
*/
DataFormat["JSON"] = "json";
/**
* Data stream holds a single JSON value -- newlines are regular whitespace.
*/
DataFormat["SINGLEJSON"] = "singlejson";
/**
* The data stream is a concatenation of JSON documents (property bags all).
*/
DataFormat["MULTIJSON"] = "multijson";
/**
* Avro format.
*/
DataFormat["AVRO"] = "avro";
/**
* Parquet format.
*/
DataFormat["PARQUET"] = "parquet";
/**
* Microsoft Cosmos structured streams format
*/
DataFormat["SSTREAM"] = "sstream";
/**
* The Optimized Row Columnar (ORC)
*/
DataFormat["ORC"] = "orc";
/**
* Avro format for ingesting through avro2json.
*/
DataFormat["APACHEAVRO"] = "apacheavro";
/**
* W3C Extended Log File format.
*/
DataFormat["W3CLogFile"] = "w3clogfile";
})(DataFormat || (DataFormat = {}));
export var IngestionMappingKind;
(function (IngestionMappingKind) {
IngestionMappingKind["CSV"] = "Csv";
IngestionMappingKind["JSON"] = "Json";
IngestionMappingKind["AVRO"] = "Avro";
IngestionMappingKind["PARQUET"] = "Parquet";
IngestionMappingKind["SSTREAM"] = "SStream";
IngestionMappingKind["ORC"] = "orc";
IngestionMappingKind["APACHEAVRO"] = "ApacheAvro";
IngestionMappingKind["W3CLOGFILE"] = "W3CLogFile";
})(IngestionMappingKind || (IngestionMappingKind = {}));
export const dataFormatMappingKind = (dataFormat) => {
switch (dataFormat) {
case DataFormat.CSV:
return IngestionMappingKind.CSV;
case DataFormat.TSV:
return IngestionMappingKind.CSV;
case DataFormat.SCSV:
return IngestionMappingKind.CSV;
case DataFormat.SOHSV:
return IngestionMappingKind.CSV;
case DataFormat.PSV:
return IngestionMappingKind.CSV;
case DataFormat.TXT:
return IngestionMappingKind.CSV;
case DataFormat.RAW:
return IngestionMappingKind.CSV;
case DataFormat.TSVE:
return IngestionMappingKind.CSV;
case DataFormat.JSON:
return IngestionMappingKind.JSON;
case DataFormat.SINGLEJSON:
return IngestionMappingKind.JSON;
case DataFormat.MULTIJSON:
return IngestionMappingKind.JSON;
case DataFormat.AVRO:
return IngestionMappingKind.AVRO;
case DataFormat.PARQUET:
return IngestionMappingKind.PARQUET;
case DataFormat.SSTREAM:
return IngestionMappingKind.SSTREAM;
case DataFormat.ORC:
return IngestionMappingKind.ORC;
case DataFormat.APACHEAVRO:
return IngestionMappingKind.APACHEAVRO;
case DataFormat.W3CLogFile:
return IngestionMappingKind.W3CLOGFILE;
default:
throw new IngestionPropertiesValidationError(`Unsupported data format: ${dataFormat}`);
}
};
export var ValidationOptions;
(function (ValidationOptions) {
ValidationOptions[ValidationOptions["DoNotValidate"] = 0] = "DoNotValidate";
ValidationOptions[ValidationOptions["ValidateCsvInputConstantColumns"] = 1] = "ValidateCsvInputConstantColumns";
ValidationOptions[ValidationOptions["ValidateCsvInputColumnLevelOnly"] = 2] = "ValidateCsvInputColumnLevelOnly";
})(ValidationOptions || (ValidationOptions = {}));
export var ValidationImplications;
(function (ValidationImplications) {
ValidationImplications[ValidationImplications["Fail"] = 0] = "Fail";
ValidationImplications[ValidationImplications["BestEffort"] = 1] = "BestEffort";
})(ValidationImplications || (ValidationImplications = {}));
export class ValidationPolicy {
constructor(validationOptions = ValidationOptions.DoNotValidate, validationImplications = ValidationImplications.BestEffort) {
this.validationOptions = validationOptions;
this.validationImplications = validationImplications;
}
toJSON() {
return {
ValidationOptions: this.validationOptions,
ValidationImplications: this.validationImplications,
};
}
}
export var ReportLevel;
(function (ReportLevel) {
ReportLevel[ReportLevel["FailuresOnly"] = 0] = "FailuresOnly";
ReportLevel[ReportLevel["DoNotReport"] = 1] = "DoNotReport";
ReportLevel[ReportLevel["FailuresAndSuccesses"] = 2] = "FailuresAndSuccesses";
})(ReportLevel || (ReportLevel = {}));
export var ReportMethod;
(function (ReportMethod) {
ReportMethod[ReportMethod["Queue"] = 0] = "Queue";
ReportMethod[ReportMethod["Table"] = 1] = "Table";
ReportMethod[ReportMethod["QueueAndTable"] = 2] = "QueueAndTable";
})(ReportMethod || (ReportMethod = {}));
// eslint-disable-next-line no-redeclare
export class IngestionProperties {
constructor(data) {
Object.assign(this, data);
}
validate() {
if (!this.database)
throw new IngestionPropertiesValidationError("Must define a target database");
if (!this.table)
throw new IngestionPropertiesValidationError("Must define a target table");
if (!this.format)
throw new IngestionPropertiesValidationError("Must define a data format");
if (this.ingestionMappingType && !this.ingestionMappingKind) {
this.ingestionMappingKind = this.ingestionMappingType;
}
if (this.ingestionMapping && !this.ingestionMappingColumns) {
this.ingestionMappingColumns = this.ingestionMapping;
}
if (!this.ingestionMappingColumns && !this.ingestionMappingReference) {
if (this.ingestionMappingKind) {
throw new IngestionPropertiesValidationError("Cannot define ingestionMappingKind without either ingestionMappingColumns or" + " ingestionMappingReference");
}
}
else {
const mappingKind = dataFormatMappingKind(this.format);
if (this.ingestionMappingKind && this.ingestionMappingKind !== mappingKind) {
throw new IngestionPropertiesValidationError(`Mapping kind '${this.ingestionMappingKind}' does not match format '${this.format}' (should be '${mappingKind}')`);
}
if (this.ingestionMappingColumns) {
if (this.ingestionMappingReference) {
throw new IngestionPropertiesValidationError("Cannot define both ingestionMappingColumns and ingestionMappingReference");
}
if (this.ingestionMappingColumns.length === 0) {
throw new IngestionPropertiesValidationError("Must define at least one column mapping");
}
const wrongMappings = this.ingestionMappingColumns
.filter((m) => m.mappingKind !== mappingKind)
.map((m) => `Mapping kind mismatch for column '${m.columnName}' - expected data format kind - '${mappingKind}', but was '${m.mappingKind}'`);
if (wrongMappings.length > 0) {
throw new IngestionPropertiesValidationError(`Invalid columns:\n${wrongMappings.join("\n")}`);
}
}
}
if (this.reportMethod !== ReportMethod.Queue && this.reportLevel === ReportLevel.FailuresOnly) {
throw new IngestionPropertiesValidationError("ReportLevel.FailuresOnly is not supported with ReportMethod.Table");
}
}
merge(extraProps) {
const merged = new IngestionProperties(this);
if (!extraProps) {
return merged;
}
const assign = (obj, prop, value) => {
obj[prop] = value;
};
for (const key of Object.keys(extraProps)) {
if (extraProps[key]) {
assign(merged, key, extraProps[key]);
}
}
return merged;
}
setDefaults() {
if (!this.format) {
this.format = DataFormat.CSV;
}
if (!this.reportLevel) {
this.reportLevel = ReportLevel.FailuresOnly;
}
if (!this.reportMethod) {
this.reportMethod = ReportMethod.Queue;
}
}
}
export function shouldCompressFileByFormat(ingestionProperties) {
if (!ingestionProperties) {
ingestionProperties = {};
}
return !(ingestionProperties.format === "avro" ||
ingestionProperties.format === "parquet" ||
ingestionProperties.format === "orc" ||
ingestionProperties.format === "apacheavro" ||
ingestionProperties.format === "sstream");
}
export default IngestionProperties;
//# sourceMappingURL=ingestionProperties.js.map