azure-kusto-ingest
Version:
Azure Data Explorer Ingestion SDK
133 lines • 7.99 kB
JavaScript
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
import { isNodeLike } from "@azure/core-util";
import { KustoConnectionStringBuilder } from "azure-kusto-data";
import { AbstractKustoClient } from "./abstractKustoClient.js";
import { BlobDescriptor, StreamDescriptor } from "./descriptors.js";
import { FileDescriptor } from "./fileDescriptor.js";
import IngestClient from "./ingestClient.js";
import { ExponentialRetry } from "./retry.js";
import { readableToStream, tryFileToBuffer, tryStreamToArray } from "./streamUtils.js";
import StreamingIngestClient from "./streamingIngestClient.js";
const maxStreamSize = 1024 * 1024 * 4;
const attemptCount = 3;
const ingestPrefix = "https://ingest-";
class KustoManagedStreamingIngestClient extends AbstractKustoClient {
/**
* Creates a KustoManagedStreamingIngestClient from a DM connection string.
* This method infers the engine connection string.
* For advanced usage, use the constructor that takes a DM connection string and an engine connection string.
*
* @param dmConnectionString The DM connection string.
* @param defaultProps The default ingestion properties.
*/
static fromDmConnectionString(dmConnectionString, defaultProps) {
var _a;
if (dmConnectionString.dataSource == null || !dmConnectionString.dataSource.startsWith(ingestPrefix)) {
throw new Error(`DM connection string must include the prefix '${ingestPrefix}'`);
}
const engineConnectionString = KustoConnectionStringBuilder.fromExisting(dmConnectionString);
engineConnectionString.dataSource = (_a = engineConnectionString.dataSource) === null || _a === void 0 ? void 0 : _a.replace(ingestPrefix, "https://");
return new KustoManagedStreamingIngestClient(engineConnectionString, dmConnectionString, defaultProps);
}
/**
* Creates a KustoManagedStreamingIngestClient from a engine connection string.
* This method infers the engine connection string.
* For advanced usage, use the constructor that takes an engine connection string and an engine connection string.
*
* @param engineConnectionString The engine connection string.
* @param defaultProps The default ingestion properties.
*/
static fromEngineConnectionString(engineConnectionString, defaultProps) {
var _a;
if (engineConnectionString.dataSource == null || engineConnectionString.dataSource.startsWith(ingestPrefix)) {
throw new Error(`Engine connection string must not include the prefix '${ingestPrefix}'`);
}
const dmConnectionString = KustoConnectionStringBuilder.fromExisting(engineConnectionString);
dmConnectionString.dataSource = (_a = dmConnectionString.dataSource) === null || _a === void 0 ? void 0 : _a.replace("https://", ingestPrefix);
return new KustoManagedStreamingIngestClient(engineConnectionString, dmConnectionString, defaultProps);
}
constructor(engineKcsb, dmKcsb, defaultProps, autoCorrectEndpoint = true) {
super(defaultProps);
this.baseSleepTimeSecs = 1;
this.baseJitterSecs = 1;
this.streamingIngestClient = new StreamingIngestClient(engineKcsb, defaultProps, autoCorrectEndpoint);
this.queuedIngestClient = new IngestClient(dmKcsb, defaultProps, autoCorrectEndpoint);
if (this.streamingIngestClient.defaultDatabase && this.streamingIngestClient.defaultDatabase !== this.queuedIngestClient.defaultDatabase) {
throw new Error(`Default database for streaming ingest client (${this.streamingIngestClient.defaultDatabase}) must match default database for queued ingest client (${this.queuedIngestClient.defaultDatabase})`);
}
this.defaultDatabase = this.streamingIngestClient.defaultDatabase;
}
/**
* Use Readable for Node.js and ArrayBuffer in browser
*/
async ingestFromStream(stream, ingestionProperties, clientRequestId) {
var _a;
this.ensureOpen();
const props = this._getMergedProps(ingestionProperties);
let descriptor = stream instanceof StreamDescriptor ? stream : new StreamDescriptor(stream);
let result = isNodeLike ? await tryStreamToArray(descriptor.stream, maxStreamSize) : descriptor.stream;
descriptor = new StreamDescriptor(result).merge(descriptor);
let streamingResult = null;
// tryStreamToArray returns a Buffer in NodeJS impl if stream size is small enouph
if ((isNodeLike && result instanceof Buffer) || !isNodeLike) {
streamingResult = await this.streamWithRetries(isNodeLike ? ((_a = descriptor.size) !== null && _a !== void 0 ? _a : 0) : descriptor.stream.byteLength, descriptor, props, clientRequestId, result);
result = isNodeLike ? readableToStream(result) : descriptor.stream;
}
return streamingResult !== null && streamingResult !== void 0 ? streamingResult : this.queuedIngestClient.ingestFromStream(new StreamDescriptor(result).merge(descriptor), props);
}
/**
* Use string for Node.js and Blob in browser
*/
async ingestFromFile(file, ingestionProperties) {
this.ensureOpen();
const stream = file instanceof FileDescriptor ? await tryFileToBuffer(file) : await tryFileToBuffer(new FileDescriptor(file));
return await this.ingestFromStream(stream, ingestionProperties);
}
async ingestFromBlob(blob, ingestionProperties, clientRequestId) {
var _a;
const props = this._getMergedProps(ingestionProperties);
const descriptor = blob instanceof BlobDescriptor ? blob : new BlobDescriptor(blob);
// No need to check blob size if it was given to us that it's not empty
await descriptor.fillSize();
const streamingResult = await this.streamWithRetries((_a = descriptor.size) !== null && _a !== void 0 ? _a : 0, descriptor, props, clientRequestId);
return streamingResult !== null && streamingResult !== void 0 ? streamingResult : this.queuedIngestClient.ingestFromBlob(descriptor, props);
}
async streamWithRetries(length, descriptor, props, clientRequestId, stream) {
const isBlob = descriptor instanceof BlobDescriptor;
if (length <= maxStreamSize) {
// If we get buffer that means it was less than the max size, so we can do streamingIngestion
const retry = new ExponentialRetry(attemptCount, this.baseSleepTimeSecs, this.baseJitterSecs);
while (retry.shouldTry()) {
try {
const sourceId = clientRequestId !== null && clientRequestId !== void 0 ? clientRequestId : `KNC.executeManagedStreamingIngest${isBlob ? "FromBlob" : "FromStream"};${descriptor.sourceId};${retry.currentAttempt}`;
if (isBlob) {
return await this.streamingIngestClient.ingestFromBlob(descriptor, props, sourceId);
}
if (isNodeLike) {
return await this.streamingIngestClient.ingestFromStream(new StreamDescriptor(readableToStream(stream)).merge(descriptor), props, sourceId);
}
return await this.streamingIngestClient.ingestFromStream(descriptor, props, sourceId);
}
catch (err) {
const oneApiError = err;
if (oneApiError["@permanent"]) {
throw err;
}
await retry.backoff();
}
}
stream = isBlob ? undefined : isNodeLike && stream ? readableToStream(stream) : descriptor.stream;
}
return null;
}
close() {
if (!this._isClosed) {
this.streamingIngestClient.close();
this.queuedIngestClient.close();
}
super.close();
}
}
export default KustoManagedStreamingIngestClient;
//# sourceMappingURL=managedStreamingIngestClient.js.map