@azure/search-documents
Version:
Azure client library to use AI Search for node.js and browser.
335 lines (334 loc) • 10.9 kB
JavaScript
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var searchIndexingBufferedSender_exports = {};
__export(searchIndexingBufferedSender_exports, {
DEFAULT_BATCH_SIZE: () => DEFAULT_BATCH_SIZE,
DEFAULT_FLUSH_WINDOW: () => DEFAULT_FLUSH_WINDOW,
DEFAULT_RETRY_COUNT: () => DEFAULT_RETRY_COUNT,
SearchIndexingBufferedSender: () => SearchIndexingBufferedSender
});
module.exports = __toCommonJS(searchIndexingBufferedSender_exports);
var import_core_util = require("@azure/core-util");
var import_node_events = __toESM(require("node:events"));
var import_timers = require("./timers.js");
var import_indexDocumentsBatch = require("./indexDocumentsBatch.js");
var import_tracing = require("./tracing.js");
const DEFAULT_BATCH_SIZE = 512;
const DEFAULT_FLUSH_WINDOW = 6e4;
const DEFAULT_RETRY_COUNT = 3;
const DEFAULT_MAX_RETRY_DELAY = 6e4;
class SearchIndexingBufferedSender {
/**
* Search Client used to call the underlying IndexBatch operations.
*/
client;
/**
* Indicates if autoFlush is enabled.
*/
autoFlush;
/**
* Interval between flushes (in milliseconds).
*/
flushWindowInMs;
/**
* Delay between retries
*/
throttlingDelayInMs;
/**
* Maximum number of Retries
*/
maxRetriesPerAction;
/**
* Max Delay between retries
*/
maxThrottlingDelayInMs;
/**
* Size of the batch.
*/
initialBatchActionCount;
/**
* Batch object used to complete the service call.
*/
batchObject;
/**
* Clean up for the timer
*/
cleanupTimer;
/**
* Event emitter/publisher used in the Buffered Sender
*/
emitter = new import_node_events.default();
/**
* Method to retrieve the document key
*/
documentKeyRetriever;
/**
* Creates a new instance of SearchIndexingBufferedSender.
*
* @param client - Search Client used to call the underlying IndexBatch operations.
* @param options - Options to modify auto flush.
*
*/
constructor(client, documentKeyRetriever, options = {}) {
this.client = client;
this.documentKeyRetriever = documentKeyRetriever;
this.autoFlush = options.autoFlush ?? true;
this.initialBatchActionCount = options.initialBatchActionCount ?? DEFAULT_BATCH_SIZE;
this.flushWindowInMs = options.flushWindowInMs ?? DEFAULT_FLUSH_WINDOW;
this.throttlingDelayInMs = options.throttlingDelayInMs ?? DEFAULT_FLUSH_WINDOW;
this.maxRetriesPerAction = options.maxRetriesPerAction ?? DEFAULT_RETRY_COUNT;
this.maxThrottlingDelayInMs = options.maxThrottlingDelayInMs ?? DEFAULT_MAX_RETRY_DELAY;
this.batchObject = new import_indexDocumentsBatch.IndexDocumentsBatch();
if (this.autoFlush) {
this.cleanupTimer = (0, import_timers.createInterval)(() => this.flush(), this.flushWindowInMs);
}
}
/**
* Uploads the documents/Adds the documents to the upload queue.
*
* @param documents - Documents to be uploaded.
* @param options - Upload options.
*/
async uploadDocuments(documents, options = {}) {
const { span, updatedOptions } = (0, import_tracing.createSpan)(
"SearchIndexingBufferedSender-uploadDocuments",
options
);
try {
this.batchObject.upload(documents);
this.emitter.emit("batchAdded", {
action: "upload",
documents
});
return this.internalFlush(false, updatedOptions);
} catch (e) {
span.setStatus({
status: "error",
error: e.message
});
throw e;
} finally {
span.end();
}
}
/**
* Merges the documents/Adds the documents to the merge queue.
*
* @param documents - Documents to be merged.
* @param options - Upload options.
*/
async mergeDocuments(documents, options = {}) {
const { span, updatedOptions } = (0, import_tracing.createSpan)(
"SearchIndexingBufferedSender-mergeDocuments",
options
);
try {
this.batchObject.merge(documents);
this.emitter.emit("batchAdded", {
action: "merge",
documents
});
return this.internalFlush(false, updatedOptions);
} catch (e) {
span.setStatus({
status: "error",
error: e.message
});
throw e;
} finally {
span.end();
}
}
/**
* Merges/Uploads the documents/Adds the documents to the merge/upload queue.
*
* @param documents - Documents to be merged/uploaded.
* @param options - Upload options.
*/
async mergeOrUploadDocuments(documents, options = {}) {
const { span, updatedOptions } = (0, import_tracing.createSpan)(
"SearchIndexingBufferedSender-mergeOrUploadDocuments",
options
);
try {
this.batchObject.mergeOrUpload(documents);
this.emitter.emit("batchAdded", {
action: "mergeOrUpload",
documents
});
return this.internalFlush(false, updatedOptions);
} catch (e) {
span.setStatus({
status: "error",
error: e.message
});
throw e;
} finally {
span.end();
}
}
/**
* Deletes the documents/Adds the documents to the delete queue.
*
* @param documents - Documents to be deleted.
* @param options - Upload options.
*/
async deleteDocuments(documents, options = {}) {
const { span, updatedOptions } = (0, import_tracing.createSpan)(
"SearchIndexingBufferedSender-deleteDocuments",
options
);
try {
this.batchObject.delete(documents);
this.emitter.emit("batchAdded", {
action: "delete",
documents
});
return this.internalFlush(false, updatedOptions);
} catch (e) {
span.setStatus({
status: "error",
error: e.message
});
throw e;
} finally {
span.end();
}
}
/**
* Flushes the queue manually.
*
* @param options - Flush options.
*/
async flush(options = {}) {
const { span, updatedOptions } = (0, import_tracing.createSpan)("SearchIndexingBufferedSender-flush", options);
try {
if (this.batchObject.actions.length > 0) {
return this.internalFlush(true, updatedOptions);
}
} catch (e) {
span.setStatus({
status: "error",
error: e.message
});
throw e;
} finally {
span.end();
}
}
/**
* If using autoFlush: true, call this to cleanup the autoflush timer.
*/
async dispose() {
if (this.batchObject.actions.length > 0) {
await this.internalFlush(true);
}
if (this.cleanupTimer) {
this.cleanupTimer();
}
}
on(event, listener) {
this.emitter.on(event, listener);
}
off(event, listener) {
this.emitter.removeListener(event, listener);
}
isBatchReady() {
return this.batchObject.actions.length >= this.initialBatchActionCount;
}
async internalFlush(force, options = {}) {
if (force || this.autoFlush && this.isBatchReady()) {
const actions = this.batchObject.actions;
this.batchObject = new import_indexDocumentsBatch.IndexDocumentsBatch();
while (actions.length > 0) {
const actionsToSend = actions.splice(0, this.initialBatchActionCount);
const { batchToSubmit, submitLater } = this.pruneActions(actionsToSend);
actions.unshift(...submitLater);
await this.submitDocuments(batchToSubmit, options);
}
}
}
pruneActions(batch) {
const hashSet = /* @__PURE__ */ new Set();
const resultBatch = [];
const pruned = [];
for (const document of batch) {
const key = this.documentKeyRetriever(document);
if (hashSet.has(key)) {
pruned.push(document);
} else {
hashSet.add(key);
resultBatch.push(document);
}
}
return { batchToSubmit: resultBatch, submitLater: pruned };
}
async submitDocuments(actionsToSend, options, retryAttempt = 1) {
try {
for (const action of actionsToSend) {
this.emitter.emit("beforeDocumentSent", action);
}
const result = await this.client.indexDocuments(
new import_indexDocumentsBatch.IndexDocumentsBatch(actionsToSend),
options
);
this.emitter.emit("batchSucceeded", result);
} catch (e) {
if (e.statusCode && e.statusCode === 413 && actionsToSend.length > 1) {
const splitActionsArray = [
actionsToSend.slice(0, actionsToSend.length / 2),
actionsToSend.slice(actionsToSend.length / 2, actionsToSend.length)
];
this.initialBatchActionCount = splitActionsArray[0].length;
for (const actions of splitActionsArray) {
await this.submitDocuments(actions, options);
}
} else if (this.isRetryAbleError(e) && retryAttempt <= this.maxRetriesPerAction) {
const exponentialDelay = this.throttlingDelayInMs * Math.pow(2, retryAttempt);
const clampedExponentialDelay = Math.min(this.maxThrottlingDelayInMs, exponentialDelay);
const delayWithJitter = clampedExponentialDelay / 2 + (0, import_core_util.getRandomIntegerInclusive)(0, clampedExponentialDelay / 2);
await (0, import_core_util.delay)(delayWithJitter);
await this.submitDocuments(actionsToSend, options, retryAttempt + 1);
} else {
this.emitter.emit("batchFailed", e);
throw e;
}
}
}
isRetryAbleError(e) {
return e.statusCode && (e.statusCode === 422 || e.statusCode === 409 || e.statusCode === 503);
}
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
DEFAULT_BATCH_SIZE,
DEFAULT_FLUSH_WINDOW,
DEFAULT_RETRY_COUNT,
SearchIndexingBufferedSender
});
//# sourceMappingURL=searchIndexingBufferedSender.js.map