@adonis-agora/agent
Version:
Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.
477 lines • 25.2 kB
JavaScript
import { createHash } from 'node:crypto';
import { EmbeddingRetriever } from './embedding-retriever.js';
import { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, } from './vector-store.js';
const DISTANCE = { cosine: 'Cosine', inner: 'Dot', l2: 'Euclid' };
/**
* Translates a metadata filter (`Record<string, unknown>`) into a {@link QdrantFilter}, preserving
* token-ACL semantics: a scalar is an exact match; an array is match-any (set membership — it mirrors
* pgvector's `jsonb_exists_any`, since Qdrant's `any` tests intersection against a scalar OR an array
* field); an empty array denies everything (`any: []` never matches). Keys target `metadata.<k>` (the
* payload nests metadata under `metadata`). Multiple keys are ANDed into `must`. Empty or absent →
* undefined.
*/
export function buildQdrantFilter(filter) {
if (filter === undefined || Object.keys(filter).length === 0)
return undefined;
const must = Object.entries(filter).map(([key, value]) => {
const k = `metadata.${key}`;
return Array.isArray(value) ? { key: k, match: { any: value } } : { key: k, match: { value } };
});
return { must };
}
/** Defensive page ceiling on the scroll loops — guards against a misbehaving client whose
* `next_page_offset` never becomes null/undefined (pagination that never terminates). */
const MAX_SCROLL_PAGES = 10_000;
/** The library's fixed UUID namespace for deriving deterministic point ids (RFC 4122 §4.3). */
const NAMESPACE = 'b9d5a5f2-1c3e-5e7a-9b2d-6f4c8a1e0d3b';
/** UUIDv5(NAMESPACE, name) via SHA-1 — deterministic, with no external dependency. */
export function chunkIdToPointId(chunkId) {
const ns = Buffer.from(NAMESPACE.replace(/-/g, ''), 'hex');
const hash = createHash('sha1').update(ns).update(chunkId).digest();
const bytes = hash.subarray(0, 16);
// readUInt8/writeUInt8 (not indexed access) so this stays clean under noUncheckedIndexedAccess.
bytes.writeUInt8((bytes.readUInt8(6) & 0x0f) | 0x50, 6); // version 5
bytes.writeUInt8((bytes.readUInt8(8) & 0x3f) | 0x80, 8); // RFC 4122 variant
const hex = bytes.toString('hex');
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
}
/**
* A {@link VectorStore} over Qdrant — the twin of {@link import('./pg-vector-store.js').PgVectorStore}
* for managed vector databases. It talks to the client through the structural {@link QdrantClientLike}
* (so `@qdrant/js-client-rest` stays an optional peer). One collection, scoped by payload filter. The
* chunk id (`${documentId}#<n>`) becomes the point's UUIDv5 and travels back in the payload.
*/
export class QdrantStore {
client;
collection;
dimension;
metric;
upsertBatchSize;
constructor(client, options = {}) {
this.client = client;
this.collection = options.collection ?? 'agent_rag_chunks';
this.dimension = options.dimension ?? 1536;
this.metric = options.metric ?? 'cosine';
this.upsertBatchSize = options.upsertBatchSize ?? 100;
}
/** Idempotent: creates the collection (dimension + metric) if it does not exist yet. */
async ensureCollection() {
const { collections } = await this.client.getCollections();
if (collections.some((c) => c.name === this.collection))
return;
await this.client.createCollection(this.collection, {
vectors: { size: this.dimension, distance: DISTANCE[this.metric] },
});
}
async upsert(records) {
if (records.length === 0)
return;
const points = records.map((r) => ({
id: chunkIdToPointId(r.id),
vector: r.embedding,
payload: {
id: r.id,
documentId: documentIdOf(r.id),
text: r.text,
...(r.source !== undefined ? { source: r.source } : {}),
...(r.metadata !== undefined ? { metadata: r.metadata } : {}),
},
}));
// Slice into batches: one huge request stalls against the client's 300s timeout (see upsertBatchSize).
const step = this.upsertBatchSize > 0 ? this.upsertBatchSize : points.length;
for (let i = 0; i < points.length; i += step) {
await this.client.upsert(this.collection, { points: points.slice(i, i + step) });
}
}
async search(embedding, options) {
const filter = buildQdrantFilter(options.filter);
// Euclid (`l2`) is Qdrant's one "smaller is better" metric — Cosine and Dot both order
// "larger is better", already matching the `VectorStore` contract (`score` higher-is-more-
// relevant), so only `l2` needs translating. Confirmed against Qdrant's docs
// (https://qdrant.tech/documentation/concepts/search/ and the API reference mirror at
// https://qdrant-qdrant-18.mintlify.app/concepts/distance-metrics): Euclid uses `Order::SmallBetter`,
// and `score_threshold` is checked as `score < threshold` for that ordering — i.e. it's an UPPER
// bound on distance (points farther than the threshold are excluded), the mirror image of Cosine's
// lower bound on similarity.
const isL2 = this.metric === 'l2';
const result = await this.client.query(this.collection, {
query: embedding,
limit: options.topK,
with_payload: true,
...(filter !== undefined ? { filter } : {}),
// `minScore` is a floor on the (higher-is-better) Passage.score. For l2, score = -distance, so
// `score >= minScore` <=> `distance <= -minScore` — negate to get the Qdrant (max-distance)
// threshold. Cosine/inner already return higher-is-better scores, so minScore passes through.
...(options.minScore !== undefined
? { score_threshold: isL2 ? -options.minScore : options.minScore }
: {}),
});
return result.points.map((point) => {
const payload = point.payload ?? {};
const source = payload.source;
const metadata = payload.metadata;
return {
id: String(payload.id),
text: String(payload.text),
// Negate Euclid's raw distance (lower-is-better) so `score` stays higher-is-more-relevant,
// matching the `VectorStore` contract and mirroring PgVectorStore's `-(col <-> ?)` for l2.
score: isL2 ? -point.score : point.score,
...(source !== undefined && source !== null ? { source: String(source) } : {}),
...(metadata !== undefined && metadata !== null
? { metadata: metadata }
: {}),
};
});
}
async remove(documentId) {
await this.client.delete(this.collection, {
filter: { must: [{ key: 'documentId', match: { value: documentId } }] },
});
}
/**
* {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
* neither `text` nor the **vector**.
*
* Qdrant is the one adapter here that cannot do the merge server-side. `set_payload` merges at the TOP
* level of a point's payload, and this store nests the caller's metadata one level down (under
* `metadata`, so a filter key is `metadata.<k>` — see {@link buildQdrantFilter}), so a top-level set of
* `metadata` replaces the whole object rather than merging into it. The merge therefore happens in JS:
* scroll the document's points for their current metadata, apply the shared
* {@link import('./vector-store.js').applyMetadataPatch}, write back. Cost: one scroll (payload only,
* **`with_vector: false`**, and only the two keys needed) plus one `set_payload` per distinct resulting
* metadata — which is ONE for any corpus built by `chunkDocuments`, since that stamps the same document
* metadata on every chunk.
*
* The vector is untouched by construction, not by care: `set_payload` has no vector field, and the
* points are never re-`upsert`ed (upsert is the call that would carry a vector). The scroll does not
* even fetch the vectors, so this method never has one in hand to write back.
*
* Point ids are re-derived with {@link chunkIdToPointId} from the chunk id in the payload, so the
* scroll needs no point id from the response and this store keeps working against a client whose
* `scroll` returns only payloads.
*
* `wait: true` because the returned count is a claim that the writes landed; Qdrant's default is to
* acknowledge before applying.
*/
async updateMetadata(documentId, patch) {
if (effectivePatchKeys(patch).length === 0) {
return 0;
}
const setPayload = this.client.setPayload?.bind(this.client);
if (setPayload === undefined) {
throw new Error('QdrantStore.updateMetadata requires a client with `setPayload` (the real ' +
'@qdrant/js-client-rest has it). Refusing to fall back to upsert, which would rewrite the ' +
'embedding this method exists to preserve.');
}
// Group by the RESULTING metadata so identical outcomes share one write — the normal case.
const groups = new Map();
let written = 0;
let offset;
for (let pageCount = 0;; pageCount++) {
if (pageCount >= MAX_SCROLL_PAGES) {
throw new Error('updateMetadata: scroll exceeded MAX_SCROLL_PAGES (the page offset never terminated)');
}
const page = await this.client.scroll(this.collection, {
filter: { must: [{ key: 'documentId', match: { value: documentId } }] },
with_payload: ['id', 'metadata'],
with_vector: false,
limit: 256,
...(offset !== undefined ? { offset } : {}),
});
for (const point of page.points) {
const payload = point.payload ?? {};
const chunkId = payload.id;
if (chunkId === undefined || chunkId === null)
continue;
const current = payload.metadata;
const next = applyMetadataPatch(current !== undefined && current !== null
? current
: undefined, patch);
const key = JSON.stringify(next);
const group = groups.get(key) ?? { metadata: next, points: [] };
group.points.push(chunkIdToPointId(String(chunkId)));
groups.set(key, group);
written += 1;
}
if (page.next_page_offset === undefined || page.next_page_offset === null)
break;
offset = page.next_page_offset;
}
for (const group of groups.values()) {
await setPayload(this.collection, {
payload: { metadata: group.metadata },
points: group.points,
wait: true,
});
}
return written;
}
/**
* {@link VectorStore.listDocumentIds} — the same scroll {@link QdrantStore.listDocuments} does, but
* transferring ONE payload key (`documentId`) instead of the whole payload, so the per-chunk metadata
* blob never crosses the wire or gets parsed. Qdrant has no `DISTINCT`, so the collapse still happens
* here; what this saves is the payload, which is the bulk of it. Filter comes from the same
* {@link buildQdrantFilter} `search` uses.
*/
async listDocumentIds(filter) {
const qFilter = buildQdrantFilter(filter);
const ids = new Set();
let offset;
for (let pageCount = 0;; pageCount++) {
if (pageCount >= MAX_SCROLL_PAGES) {
throw new Error('listDocumentIds: scroll exceeded MAX_SCROLL_PAGES (the page offset never terminated)');
}
const page = await this.client.scroll(this.collection, {
with_payload: ['documentId'],
with_vector: false,
limit: 256,
...(qFilter !== undefined ? { filter: qFilter } : {}),
...(offset !== undefined ? { offset } : {}),
});
for (const point of page.points) {
const docId = point.payload?.documentId;
if (docId === undefined || docId === null)
continue;
ids.add(String(docId));
}
if (page.next_page_offset === undefined || page.next_page_offset === null)
break;
offset = page.next_page_offset;
}
return [...ids];
}
/**
* {@link VectorStore.removeWhere} — ONE filtered `delete`, no enumeration: Qdrant deletes by filter
* server-side, so this never scrolls the points it is about to remove.
*
* Filter parity with {@link QdrantStore.search} is by construction — the same
* {@link buildQdrantFilter} call produces the predicate for both, so `metadata.<k>` keying, scalar
* `match.value`, array `match.any` set-membership and the empty-array deny are the same object here as
* there.
*
* It DOES cost one extra round trip that memory and pgvector do not: Qdrant's delete response carries
* no count, so the chunk count comes from a preceding `count` (an aggregate — no payloads, no vectors,
* not an enumeration). Consequence worth knowing: the count is taken just before the delete, so under
* concurrent writes it is a very slightly stale count of what matched, where memory and pgvector return
* an exact count from the deleting statement itself. The delete is filter-scoped either way; only the
* reported number can drift.
*/
async removeWhere(filter) {
assertRemovalFilter(filter);
if (filterDeniesAll(filter)) {
return 0;
}
const count = this.client.count?.bind(this.client);
if (count === undefined) {
throw new Error('QdrantStore.removeWhere requires a client with `count` (the real @qdrant/js-client-rest has ' +
'it) to report how many chunks it removed, because Qdrant’s delete response carries no count.');
}
const qFilter = buildQdrantFilter(filter);
if (qFilter === undefined) {
// Unreachable: assertRemovalFilter already rejected the only input that yields undefined.
throw new Error('removeWhere: filtro vazio escapou do assertRemovalFilter');
}
const { count: matched } = await count(this.collection, { filter: qFilter, exact: true });
await this.client.delete(this.collection, { filter: qFilter, wait: true });
return matched;
}
async listDocuments(filter) {
const qFilter = buildQdrantFilter(filter);
const seen = new Map();
let offset;
// Page through the scroll cursor until it is exhausted. A ~1500-chunk corpus is a few pages.
for (let pageCount = 0;; pageCount++) {
if (pageCount >= MAX_SCROLL_PAGES) {
throw new Error('listDocuments: scroll exceeded MAX_SCROLL_PAGES (the page offset never terminated)');
}
const page = await this.client.scroll(this.collection, {
with_payload: true,
with_vector: false,
limit: 256,
...(qFilter !== undefined ? { filter: qFilter } : {}),
...(offset !== undefined ? { offset } : {}),
});
for (const point of page.points) {
const payload = point.payload ?? {};
const docId = payload.documentId;
if (docId === undefined || docId === null)
continue;
const id = String(docId);
if (seen.has(id))
continue;
const metadata = payload.metadata;
seen.set(id, {
id,
...(metadata !== undefined && metadata !== null
? { metadata: metadata }
: {}),
});
}
if (page.next_page_offset === undefined || page.next_page_offset === null)
break;
offset = page.next_page_offset;
}
return [...seen.values()];
}
/**
* OPTIONAL capability. Server-side GROUP BY + exact count over one payload field — the
* aggregate an inspection panel reads per document ("chunks per value of `metadata.numeroProcesso`",
* values of the top-level `source`), optionally narrowed by a RAW {@link QdrantFilter} (the
* metadata-`Record` filters of {@link VectorStore.search} cannot express the absence a priority
* chain needs; raw filters can, via `is_empty`/`must_not`/`should`).
*
* Qdrant requires a payload index on the facetted field. When the server complains, this method
* PROVISIONS a `keyword` index (idempotent, `wait: true`) and retries the facet exactly once —
* the first call on a fresh collection pays one index build, every later call is a plain facet.
* If the client lacks `facet` (too old) the error says so; if it lacks `createPayloadIndex` the
* original server error propagates instead of a silent degrade.
*
* Counts are the server's FAST default (per-segment indexed statistics). `exact: true` asks for
* the expensive recount and can cost tens of seconds per call on a corpus of hundreds of
* thousands of points — which for a panel that fans out into a dozen facets is the difference
* between a page load and a hang. It stays opt-in for reconciliation-shaped uses.
*
* `first` is the number of distinct values to return (default 200) — spelled as in
* {@link import('../pagination.js').CursorParams} so page sizes are named the same thing across the
* ecosystem. It is deliberately NOT a {@link import('../pagination.js').CursorPage}: Qdrant's facet
* endpoint returns the top-`first` values in one shot and hands back **no continuation token**, so a
* page envelope here would advertise a `nextCursor` that could only ever be `null`. Faceting is a
* capped aggregate; {@link QdrantStore.scrollChunks} is the paginated surface.
*/
async facetValues(field, options) {
const facet = this.client.facet?.bind(this.client);
if (facet === undefined) {
throw new Error('QdrantStore.facetValues requires a client with `facet` (the real @qdrant/js-client-rest ' +
'>= 1.18 against a server >= 1.10 has it).');
}
const args = {
key: field,
limit: options?.first ?? 200,
...(options?.exact === undefined ? {} : { exact: options.exact }),
...(options?.filter !== undefined ? { filter: options.filter } : {}),
};
const request = () => facet(this.collection, args);
try {
const { hits } = await request();
return hits;
}
catch (error) {
const createIndex = this.client.createPayloadIndex?.bind(this.client);
if (createIndex === undefined)
throw error;
await createIndex(this.collection, {
field_name: field,
field_schema: { type: 'keyword' },
wait: true,
});
const { hits } = await request();
return hits;
}
}
/**
* OPTIONAL capability. Count chunks matching a RAW {@link QdrantFilter} without transferring
* any — the aggregate behind a panel row like "chunks with NO key field at all". Requires a
* client with `count` (the real one has it; {@link QdrantStore.removeWhere} already depends on
* it). The count is the server's fast default; `exact: true` forces the expensive recount
* (same trade-off as {@link QdrantStore.facetValues}).
*/
async countChunks(options) {
const count = this.client.count?.bind(this.client);
if (count === undefined) {
throw new Error('QdrantStore.countChunks requires a client with `count` (the real @qdrant/js-client-rest has it).');
}
const { count: total } = await count(this.collection, {
...(options?.filter !== undefined ? { filter: options.filter } : {}),
...(options?.exact === undefined ? {} : { exact: options.exact }),
});
return total;
}
/**
* OPTIONAL capability. Enumerate chunk PAYLOADS matching a RAW {@link QdrantFilter} — the
* read side of the same server-side narrowing {@link QdrantStore.facetValues} counts. Vectors
* never cross the wire; `payloadKeys` narrows the transfer further.
*
* Unlike {@link VectorStore.listDocuments} this is per CHUNK (not collapsed per document) and
* speaks the raw filter language — it exists for inspection/deletion panels, where the caller
* knows the exact payload shape it stored.
*
* **Cursor-paginated**, in the ecosystem's forward-only interface (see
* {@link import('../pagination.js').CursorParams}): `{ after, first }` in, a
* {@link import('../pagination.js').CursorPage} out. ONE round trip per call — `first` (default
* 256) is the Qdrant scroll `limit`, and `nextCursor` wraps Qdrant's own opaque
* `next_page_offset`. Qdrant's scroll only ever resumes *forward* from that offset, so
* `prevCursor` is always `null` and `hasPrev` always `false`.
*
* The cursor is OPAQUE — {@link encodeScrollCursor} base64url's the raw offset precisely so
* callers cannot come to depend on it being a Qdrant point id. Hand it back verbatim.
*
* To drain the whole match, loop while `hasNext`:
*
* ```ts
* const all: Record<string, unknown>[] = [];
* let after: string | undefined;
* do {
* const page = await store.scrollChunks({ filter, ...(after ? { after } : {}) });
* all.push(...page.items);
* after = page.nextCursor ?? undefined;
* } while (after !== undefined);
* ```
*/
async scrollChunks(options) {
const offset = options?.after === undefined ? undefined : decodeScrollCursor(options.after);
const page = await this.client.scroll(this.collection, {
with_payload: options?.payloadKeys ?? true,
with_vector: false,
limit: options?.first ?? 256,
...(options?.filter !== undefined ? { filter: options.filter } : {}),
...(offset !== undefined ? { offset } : {}),
});
const nextOffset = page.next_page_offset;
const hasNext = nextOffset !== undefined && nextOffset !== null;
return {
items: page.points.map((point) => point.payload ?? {}),
nextCursor: hasNext ? encodeScrollCursor(nextOffset) : null,
// Forward-only backend: Qdrant's scroll has no backward token. See `src/pagination.ts`.
prevCursor: null,
hasNext,
hasPrev: false,
};
}
}
/**
* Wrap Qdrant's `next_page_offset` (a point id — string, number or UUID, the server's business) in an
* opaque, URL-safe cursor: base64url of its JSON. Mirrors how `@adonis-agora/filter` encodes its
* keyset cursors, and for the same reason — a cursor a caller can read is a cursor a caller will
* eventually construct.
*/
export function encodeScrollCursor(offset) {
return Buffer.from(JSON.stringify({ o: offset }), 'utf8').toString('base64url');
}
/**
* Unwrap a cursor produced by {@link encodeScrollCursor}. Throws on a malformed one rather than
* silently restarting from the first page: a scroll that quietly rewinds re-emits chunks a
* deletion panel already processed, which is worse than a loud failure.
*/
export function decodeScrollCursor(cursor) {
let parsed;
try {
parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
}
catch {
parsed = undefined;
}
if (parsed === null || typeof parsed !== 'object' || !('o' in parsed)) {
throw new Error('scrollChunks: `after` is not a cursor this store issued. Pass back a `nextCursor` from a ' +
'previous page verbatim (cursors are opaque), or omit it for the first page.');
}
return parsed.o;
}
/** A {@link import('../spi/retriever.js').Retriever} over a {@link QdrantStore}: embeds the query,
* then searches Qdrant. Twin of `PgVectorRetriever`. */
export class QdrantRetriever extends EmbeddingRetriever {
// biome-ignore lint/complexity/noUselessConstructor: not useless — it NARROWS `store` from the base's `VectorStore` to `QdrantStore`, which is this subclass's whole point. Delete it and `new QdrantRetriever(embedder, anyInMemoryStore)` starts compiling.
constructor(embedder, store) {
super(embedder, store);
}
}
//# sourceMappingURL=qdrant-store.js.map