@zilliz/milvus2-sdk-node
Version:
Official Node.js SDK for [Milvus](https://github.com/milvus-io/milvus) vector database. Provides gRPC and HTTP clients for vector similarity search, metadata filtering, and full collection/index/user management.
374 lines • 17.1 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.extractRequestMetadata = exports.isValidClientRequestId = exports.getValidDataArray = exports.getSparseDim = exports.getPKFieldExpr = exports.getQueryIteratorExpr = exports.sleep = exports.findKeyValue = exports.promisify = exports.withTelemetrySuppressed = exports.withTelemetryLogicalOperation = exports.setPoolFailoverHandler = exports.setPoolTelemetryManager = exports.TELEMETRY_MANAGER_KEY = exports.FAILOVER_HANDLER_KEY = void 0;
const __1 = require("../");
const logger_1 = require("./logger");
const grpc_js_1 = require("@grpc/grpc-js");
const async_hooks_1 = require("async_hooks");
/** Well-known property key for attaching a failover handler to a pool. */
exports.FAILOVER_HANDLER_KEY = '__failoverHandler';
exports.TELEMETRY_MANAGER_KEY = '__telemetryManager';
/** Attach the telemetry recorder used by the common RPC path. */
function setPoolTelemetryManager(pool, manager) {
pool[exports.TELEMETRY_MANAGER_KEY] = manager;
}
exports.setPoolTelemetryManager = setPoolTelemetryManager;
/**
* Attach a failover handler to a pool for global cluster support.
*/
function setPoolFailoverHandler(pool, handler) {
pool[exports.FAILOVER_HANDLER_KEY] = handler;
}
exports.setPoolFailoverHandler = setPoolFailoverHandler;
/**
* Check if an error is a gRPC UNAVAILABLE error.
*/
function isUnavailableError(err) {
return err && err.code === grpc_js_1.status.UNAVAILABLE;
}
/**
* Execute a single gRPC call via the pool.
*/
function executeCall(pool, target, params, timeout, requestMetadata) {
const t = timeout === 0 ? 1000 * 60 * 60 * 24 : timeout;
return (() => __awaiter(this, void 0, void 0, function* () {
const client = yield pool.acquire();
let finalRequestMetadata = requestMetadata;
if (!finalRequestMetadata && params) {
finalRequestMetadata = (0, exports.extractRequestMetadata)(params);
}
const clientRequestId = getClientRequestId(finalRequestMetadata);
let metadata;
if (clientRequestId) {
metadata = new grpc_js_1.Metadata();
metadata.add(__1.METADATA.CLIENT_REQUEST_ID, clientRequestId);
}
return new Promise((resolve, reject) => {
try {
const callOptions = { deadline: new Date(Date.now() + t) };
const callback = (err, result) => {
if (err) {
reject(err);
}
else {
resolve(result);
}
if (client) {
pool.release(client);
}
};
// grpc-js unary calls accept either (request, options, callback) or
// (request, metadata, options, callback). Metadata is not an option field:
// putting it inside callOptions silently drops it on a real channel even though
// loose unit-test doubles may appear to accept it.
if (metadata) {
client[target](params, metadata, callOptions, callback);
}
else {
client[target](params, callOptions, callback);
}
}
catch (e) {
reject(e);
if (client) {
pool.release(client);
}
}
});
}))();
}
const TELEMETRY_OPERATIONS = new Set([
'Insert',
'Delete',
'Upsert',
'Search',
'HybridSearch',
'Query',
'RunAnalyzer',
]);
// A process-wide store is safe here: AsyncLocalStorage isolates concurrent promise
// chains, while nested public helpers (for example delete -> deleteEntities) inherit
// the current logical operation and therefore do not emit a second measurement.
const telemetryLogicalScope = new async_hooks_1.AsyncLocalStorage();
function telemetryOperation(target) {
return TELEMETRY_OPERATIONS.has(target) ? target : undefined;
}
function getBusinessError(result) {
const responseStatus = (result === null || result === void 0 ? void 0 : result.status) || result;
if (!responseStatus) {
return undefined;
}
const code = Number(responseStatus.code || 0);
const errorCode = responseStatus.error_code;
const success = code === 0 &&
(errorCode === undefined ||
errorCode === 0 ||
errorCode === '0' ||
errorCode === 'Success' ||
errorCode === 'SUCCESS');
return success
? undefined
: new Error(responseStatus.reason || 'Milvus operation failed');
}
/**
* Measure one complete public SDK operation, including validation, request
* preprocessing, retries, response formatting, and postprocessing.
*
* promisify remains instrumented as a fallback for direct/internal callers, but
* suppresses its RPC-level measurement while the same logical operation is active.
*/
function withTelemetryLogicalOperation(pool, operation, params, call) {
return __awaiter(this, void 0, void 0, function* () {
const activeScope = telemetryLogicalScope.getStore();
if ((activeScope === null || activeScope === void 0 ? void 0 : activeScope.suppress) || (activeScope === null || activeScope === void 0 ? void 0 : activeScope.operation) === operation) {
return call();
}
const telemetry = pool
? pool[exports.TELEMETRY_MANAGER_KEY]
: undefined;
const startTime = performance.now();
const requestId = getTelemetryRequestId((0, exports.extractRequestMetadata)(params));
let result;
let finalError;
let failed = false;
try {
result = yield telemetryLogicalScope.run({ operation }, call);
}
catch (error) {
failed = true;
finalError = error;
}
if (telemetry) {
try {
telemetry.recordOperation({
operation,
collection: operation === 'RunAnalyzer'
? ''
: String((params === null || params === void 0 ? void 0 : params.collection_name) || ''),
startTime,
error: failed ? finalError : getBusinessError(result),
requestId,
});
}
catch (_a) {
// Optional telemetry must never replace the business result, including when a
// thrown value has a hostile coercion hook or logging itself is unavailable.
}
}
if (failed) {
throw finalError;
}
return result;
});
}
exports.withTelemetryLogicalOperation = withTelemetryLogicalOperation;
/**
* Run internal SDK machinery without emitting telemetry. Iterators are not
* logical operations, so neither their setup queries nor their per-page
* internal RPCs may be measured: wrapping them here suppresses both the
* logical-operation wrapper and the promisify RPC-level fallback.
*/
function withTelemetrySuppressed(call) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const activeScope = telemetryLogicalScope.getStore();
if (activeScope === null || activeScope === void 0 ? void 0 : activeScope.suppress) {
return call();
}
return telemetryLogicalScope.run({ operation: (_a = activeScope === null || activeScope === void 0 ? void 0 : activeScope.operation) !== null && _a !== void 0 ? _a : '', suppress: true }, call);
});
}
exports.withTelemetrySuppressed = withTelemetrySuppressed;
/**
* Promisify a function call with optional timeout, metadata, and global cluster failover.
* @param pool - The pool of gRPC clients
* @param target - The name of the target function to call
* @param params - The parameters to pass to the target function (may contain client_request_id or client-request-id)
* @param timeout - Optional timeout in milliseconds
* @param requestMetadata - Optional metadata to include in the request (e.g., client-request-id). If not provided, will be extracted from params automatically.
* @returns A Promise that resolves with the result of the target function call
*/
function promisify(pool, target, params, timeout, requestMetadata) {
return __awaiter(this, void 0, void 0, function* () {
const operation = telemetryOperation(target);
const activeScope = telemetryLogicalScope.getStore();
const recordRpcOperation = operation !== undefined &&
!(activeScope === null || activeScope === void 0 ? void 0 : activeScope.suppress) &&
(activeScope === null || activeScope === void 0 ? void 0 : activeScope.operation) !== operation;
const telemetry = pool[exports.TELEMETRY_MANAGER_KEY];
const startTime = performance.now();
const finalRequestMetadata = requestMetadata || (params ? (0, exports.extractRequestMetadata)(params) : undefined);
const requestId = getTelemetryRequestId(finalRequestMetadata);
let result;
let finalError;
let failed = false;
try {
try {
result = yield executeCall(pool, target, params, timeout, finalRequestMetadata);
}
catch (error) {
// Check for global cluster failover handler. Instrumentation stays outside both
// attempts so one logical SDK call contributes exactly one telemetry outcome.
const handler = pool[exports.FAILOVER_HANDLER_KEY];
if (!handler || !isUnavailableError(error)) {
throw error;
}
logger_1.logger.debug(`\x1b[36m[Global]\x1b[0m UNAVAILABLE error on \x1b[1m${target}\x1b[0m, triggering failover handler`);
const newPool = yield handler(error);
if (!newPool) {
throw error;
}
logger_1.logger.debug(`\x1b[36m[Global]\x1b[0m Failover complete, retrying \x1b[1m${target}\x1b[0m with new pool`);
result = yield executeCall(newPool, target, params, timeout, finalRequestMetadata);
}
}
catch (error) {
failed = true;
finalError = error;
}
if (recordRpcOperation && operation && telemetry) {
const businessError = failed ? finalError : getBusinessError(result);
try {
telemetry.recordOperation({
operation,
// Go intentionally records RunAnalyzer globally even though its request happens to
// carry a collection_name field.
collection: operation === 'RunAnalyzer'
? ''
: String((params === null || params === void 0 ? void 0 : params.collection_name) || ''),
startTime,
error: businessError,
requestId,
});
}
catch (_a) {
// Optional telemetry must never replace the business result, including when a
// thrown value has a hostile coercion hook or logging itself is unavailable.
}
}
if (failed) {
throw finalError;
}
return result;
});
}
exports.promisify = promisify;
const findKeyValue = (obj, key) => { var _a; return (_a = obj.find(v => v.key === key)) === null || _a === void 0 ? void 0 : _a.value; };
exports.findKeyValue = findKeyValue;
const sleep = (time) => {
return new Promise(resolve => setTimeout(resolve, time));
};
exports.sleep = sleep;
/**
* Returns the query iterator expression based on the provided parameters.
*
* @param params - The parameters for generating the query iterator expression.
* @param params.expr - The expression to be combined with the iterator expression.
* @param params.pkField - The primary key field schema.
* @param params.lastPKId - The primary key cursor from the previous batch.
* @param params.lastElementOffset - The struct element cursor from the previous batch.
* @returns The query iterator expression.
*/
const getQueryIteratorExpr = (params) => {
// get params
const { expr, lastPKId, lastElementOffset, pkField } = params;
// If cache does not exist, return expression based on primaryKey type
let compareValue = '';
if (typeof lastPKId === 'undefined') {
// get default value
compareValue =
(pkField === null || pkField === void 0 ? void 0 : pkField.data_type) === __1.DataTypeStringEnum.VarChar
? ''
: `${__1.DEFAULT_MIN_INT64}`;
}
else {
compareValue = lastPKId;
}
// return expr combined with iteratorExpr
return (0, exports.getPKFieldExpr)({
pkField,
value: compareValue,
expr,
condition: typeof lastElementOffset === 'undefined' ? '>' : '>=',
});
};
exports.getQueryIteratorExpr = getQueryIteratorExpr;
// return pk filed != expression based on pk field type, if pk field is string, return pk field != ''
const getPKFieldExpr = (data) => {
const { pkField, value, condition = '!=', expr = '' } = data;
const pkValue = (pkField === null || pkField === void 0 ? void 0 : pkField.data_type) === __1.DataTypeStringEnum.VarChar
? `'${value}'`
: `${value}`;
return `${pkField === null || pkField === void 0 ? void 0 : pkField.name} ${condition} ${pkValue}${expr ? ` && (${expr})` : ''}`;
};
exports.getPKFieldExpr = getPKFieldExpr;
// get biggest size of sparse vector array
const getSparseDim = (data) => {
let dim = 0;
for (const row of data) {
const indices = Object.keys(row).map(Number);
if (indices.length > dim) {
dim = indices.length;
}
}
return dim;
};
exports.getSparseDim = getSparseDim;
// get valid data
// create a length array with valid data, if the data is undefined or null, return false, otherwise return true
const getValidDataArray = (data, length) => {
return Array.from({ length }).map((_, i) => {
return data[i] !== undefined && data[i] !== null;
});
};
exports.getValidDataArray = getValidDataArray;
/**
* Extracts client request ID from metadata object with priority handling.
* Priority: client_request_id > client-request-id (JavaScript/TypeScript convention)
* @param metadata - Metadata object that may contain traceid
* @returns Client request ID as string or undefined if not found
*/
const getClientRequestId = (metadata) => {
var _a;
if (!metadata) {
return undefined;
}
// Preserve the established wire contract: arbitrary non-empty string IDs were
// documented and sent before client telemetry existed. New telemetry correlation has
// stricter OpenTelemetry requirements, but must not silently remove metadata from
// existing applications.
const value = (_a = metadata.client_request_id) !== null && _a !== void 0 ? _a : metadata['client-request-id'];
return typeof value === 'string' && value.length > 0 ? value : undefined;
};
/** Returns whether value is a non-zero lowercase 128-bit OpenTelemetry trace ID. */
const isValidClientRequestId = (value) => typeof value === 'string' &&
/^[0-9a-f]{32}$/.test(value) &&
value !== '00000000000000000000000000000000';
exports.isValidClientRequestId = isValidClientRequestId;
/** Returns a request ID only when it is safe to expose as an OTel trace ID. */
const getTelemetryRequestId = (metadata) => {
const value = getClientRequestId(metadata);
return (0, exports.isValidClientRequestId)(value) ? value : undefined;
};
/**
* Extracts request metadata (traceid) from request data.
* Supports both client_request_id and client-request-id formats.
* Priority: client_request_id > client-request-id (JavaScript/TypeScript convention)
* @param data - Request data that may contain traceid
* @returns Request metadata object or undefined if no traceid provided
*/
const extractRequestMetadata = (data) => {
const clientRequestId = getClientRequestId(data);
return clientRequestId ? { 'client-request-id': clientRequestId } : undefined;
};
exports.extractRequestMetadata = extractRequestMetadata;
//# sourceMappingURL=Function.js.map