@zilliz/milvus2-sdk-node
Version:
Official Node.js SDK for [Milvus](https://github.com/milvus-io/milvus) vector database. Provides gRPC and HTTP clients for vector similarity search, metadata filtering, and full collection/index/user management.
117 lines (116 loc) • 5.55 kB
TypeScript
/**
* Client-side roaring bitmap construction for the `membership_match(field, {blob}, type=roaring)` filter
* expression.
*
* `membership_match(..., type=roaring)` is the exact-membership sibling of
* `membership_match(..., type=bloom)`: the client compresses the
* membership set into a roaring bitmap and ships the blob, so a set that would blow past the
* proxy gRPC receive limit as an `in [...]` list rides the wire in a fraction of the space --
* and, unlike a bloom filter, with no false positives. Only int64-domain fields are
* addressable this way, since a roaring bitmap indexes 64-bit keys.
*
* The blob is an MRB1 envelope wrapping a *portable* Roaring64 serialization -- the
* cross-language format CRoaring calls `portable`, which is what segcore decodes. The proxy
* validates the envelope and embeds the bytes verbatim, so these bytes must match what the
* other SDKs produce exactly. See milvus-io/milvus#51968 and
* `docs/design-docs/design_docs/20260714-roaring-exact-membership-expression.md`.
*/
/** The 4-byte MRB1 envelope magic. */
export declare const ROARING_BITMAP_MAGIC = "MRB1";
/** The MRB1 envelope version implemented here. */
export declare const ROARING_BITMAP_VERSION = 1;
/** Identifies the portable Roaring64 body format. */
export declare const ROARING_BITMAP_FORMAT_PORTABLE_ROARING64 = 1;
/** Size in bytes of the MRB1 envelope header. */
export declare const ROARING_BITMAP_HEADER_SIZE = 32;
/**
* Limits the proxy enforces on a decoded blob, checked here so an oversized set fails at the
* call that built it rather than as a server-side parameter error one request later.
*/
export declare const ROARING_BITMAP_MAX_HIGH_CONTAINERS: number;
export declare const ROARING_BITMAP_MAX_DECODED_BYTES: number;
export declare const ROARING_BITMAP_MAX_BODY_BYTES: number;
/**
* A membership value.
*
* `bigint` and decimal `string` are here because an int64 does not survive a JavaScript
* number: the SDK already carries int64s as strings everywhere else (see Format.ts), and a
* number past 2^53 is rejected rather than silently rounded into a different key.
*/
export type RoaringBitmapMember = number | bigint | string;
/**
* Incrementally collects members and serializes them as an MRB1-wrapped roaring bitmap.
*
* Use it when the membership set arrives in pieces -- a cursor, a paged API -- so a 10M-member
* bitmap never has to exist as a 10M-element array first. {@link buildRoaringBitmap} is the
* one-shot form.
*
* The blob is the expensive part and is meant to be reused: build it once and pass the same
* `Uint8Array` to every query that needs that membership set.
*
* ```ts
* const builder = new RoaringBitmapBuilder();
* for await (const page of pages) builder.addMany(page.ids);
* const blob = builder.build();
* ```
*/
export declare class RoaringBitmapBuilder {
/**
* high key -> (container key -> low 16 bits), populated in insertion order and sorted only
* at build time.
*
* Bucketing on the way in is what keeps BigInt out of the hot path. The normative algorithm
* sorts uint64 keys, which in JavaScript means either BigInt comparisons or an index sort
* over 64-bit pairs; splitting first and sorting the 32-bit high keys and the 16-bit values
* separately is the same total order -- unsigned high key, then container key, then value --
* with nothing wider than a uint32 ever compared.
*/
private readonly highs;
/** Last bucket touched. Members usually arrive clustered, so this skips most outer lookups. */
private lastHighKey;
private lastHigh;
/** Adds one member. Duplicates are allowed and collapse at build time. */
add(value: RoaringBitmapMember): this;
/** Adds every member of `values`. */
addMany(values: ReadonlyArray<RoaringBitmapMember>): this;
/**
* Serializes the current member set into an MRB1 envelope.
*
* The builder stays usable afterwards, and building twice from the same members produces
* the same bytes.
*
* @throws if the bitmap exceeds a limit the proxy would reject (see
* ROARING_BITMAP_MAX_HIGH_CONTAINERS / _MAX_DECODED_BYTES / _MAX_BODY_BYTES)
*/
build(): Uint8Array;
private insert;
/** Sorts and deduplicates every bucket, then decides each container's encoding. */
private finalize;
}
/**
* Builds an MRB1-wrapped roaring bitmap over a membership set, for `membership_match(..., type=roaring)`.
*
* Members are signed integers -- `number`, `bigint`, or a decimal `string` for values a
* number cannot hold exactly. They map by sign extension to int64 and then by two's-
* complement reinterpretation to the bitmap's unsigned 64-bit key space, so `-1` and
* `INT8(-1)` are the same key; duplicates collapse and order does not matter. An empty set
* produces a valid 40-byte blob that matches nothing.
*
* The target field must be an int64-domain field: unlike `membership_match(..., type=bloom)` there is no string
* form, because a roaring bitmap indexes integers.
*
* Building the blob is the expensive part, so keep it and reuse it across queries rather than
* rebuilding per request.
*
* ```ts
* const blob = buildRoaringBitmap(userIds);
* await client.query({
* collection_name: 'docs',
* filter: 'membership_match(user_id, {ids}, type=roaring)',
* exprValues: { ids: blob },
* });
* ```
*
* @param members the membership set
*/
export declare const buildRoaringBitmap: (members: ReadonlyArray<RoaringBitmapMember>) => Uint8Array;