@adonis-agora/agent
Version:
Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.
311 lines • 19.8 kB
TypeScript
import type { EmbeddingProvider } from '../spi/embedding-provider.js';
import type { Passage } from '../spi/retriever.js';
import type { LucidRawRunner } from '../stores/lucid.js';
import { EmbeddingRetriever } from './embedding-retriever.js';
import type { IndexedDocument, LexicalVectorStore, MetadataPatch, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
/**
* The similarity metric — which pgvector distance operator ranks results (and which index opclass the
* migration should build). `cosine` (the default) is scale-invariant and the right choice for most
* embedding models; `l2` (Euclidean) and `inner` (inner product) are offered for models trained for them.
*/
export type PgVectorMetric = 'cosine' | 'l2' | 'inner';
/**
* Physical column names for the pgvector chunk table. Every field defaults to the mirror of the
* in-memory store's record shape (`id`/`text`/`source`/`metadata`/`embedding`); override to point the
* store at an existing table. Each name is validated against a strict identifier regex before it is ever
* spliced into SQL — the ONLY parts of a statement not passed as a positional binding.
*/
export interface PgVectorColumns {
/** Chunk id column (PK). Chunk ids are `${documentId}#<n>`. Default `id`. */
id?: string;
/** Chunk text column. Default `text`. */
text?: string;
/** Citation/source column (nullable). Default `source`. */
source?: string;
/** `jsonb` metadata column (nullable). Default `metadata`. */
metadata?: string;
/** `vector(N)` embedding column. Default `embedding`. */
embedding?: string;
}
export interface PgVectorStoreOptions {
/** Table name. Default `agent_rag_chunks`. Validated against the identifier regex. */
table?: string;
/**
* Embedding width — must match your model (e.g. 1536 for text-embedding-3-small). Default 1536.
*
* Pass a **list** to let several models' vectors share one table (mixed dimensions — say, while
* migrating from a 768-wide local model to a 1536-wide hosted one): the column becomes an untyped
* `vector`, the schema gets one *partial* HNSW index per listed width
* (`((embedding::vector(n))) … WHERE vector_dims(embedding) = n`), and `search` only compares the
* query against vectors of its own width, through that width's index. A width that is not listed is
* still answered — exactly, by a sequential scan. Put the model in the chunk metadata and filter on
* it when two models share a width.
*/
dimension?: number | readonly number[];
/**
* Allow chunks with no embedding (`embedding: []` on upsert is stored as `NULL`): index text first
* and embed later, or keep full-text-only chunks. `search` skips them; the lexical leg of
* {@link PgLexicalVectorStore} still finds them. Default `false` — the column stays `NOT NULL`. Only
* changes the DDL for a table that does not exist yet.
*/
nullableEmbeddings?: boolean;
/**
* pgvector ≥ 0.8 iterative index scans for `search`: keep walking the HNSW graph until enough rows
* pass the metadata filter, instead of returning fewer than `topK` when a selective filter (one
* tenant of many) rejects most of what the index produced. Applied with `SET LOCAL` inside a
* transaction, so it needs a handle with `transaction` (Lucid's `db` has one); silently skipped on an
* older pgvector or a runner without transactions. Default off.
*/
iterativeScan?: 'strict_order' | 'relaxed_order';
/** `hnsw.ef_search` for `search` (candidate list size; pgvector's default is 40). Needs `transaction`. */
efSearch?: number;
/**
* Rows per multi-row `INSERT … ON CONFLICT` statement in `upsert`. Default 100. Duplicate ids in one
* `upsert` call collapse to the last occurrence (as the old row-by-row loop left them).
*/
upsertBatchSize?: number;
/** Similarity metric / distance operator. Default `cosine`. */
metric?: PgVectorMetric;
/** Override the physical column names (each validated). */
columns?: PgVectorColumns;
}
/** pgvector accepts a `'[1,2,3]'` text literal cast to `vector`; the literal itself is a `?` binding. */
export declare function toVectorLiteral(embedding: number[]): string;
/**
* Strip every NUL byte (`0x00`) from a value before it can reach a Postgres `text`/`jsonb` binding.
* Postgres rejects `0x00` in those types outright (`invalid byte sequence for encoding "UTF8": 0x00`),
* while source systems like Qdrant accept it — so text extracted from PDFs that happens to carry a stray
* NUL byte works fine right up until it is written to pgvector. Strings have the byte removed (the rest
* of the text is kept intact); arrays and PLAIN objects are walked recursively, including object KEYS
* (a NUL in a metadata key would otherwise reach `JSON.stringify` and then Postgres just the same); every
* other value (numbers, booleans, `null`, `undefined`, embeddings, a `Date`, a class instance, a
* `Buffer`, …) passes through unchanged, by reference.
*
* Two things a naive `for...in` + assignment walk gets wrong, both handled here:
*
* - **Only plain objects are walked.** The check is `Object.getPrototypeOf(v) === Object.prototype ||
* === null`, never `typeof v === 'object'` alone — a `Date`, a class instance, a `Buffer`, … would
* otherwise be torn down into a plain `{}` of its enumerable own properties, silently losing its
* prototype (and, for a `Date`, every method on it).
* - **The result is built with `Object.fromEntries`, never `result[key] = value` on a fresh `{}`.** A
* metadata key literally named `__proto__`, assigned that way, does not create an own property — it
* hits the inherited `Object.prototype` accessor instead, which (for the ordinary case, a string
* value) silently no-ops, so the key is dropped rather than round-tripped. `Object.fromEntries` uses
* `CreateDataProperty` under the hood, which always creates a real own property, `__proto__` included.
*/
export declare function stripNulBytes<T>(value: T): T;
/**
* Build the metadata `WHERE` fragment for a filter, using Lucid/Knex positional `?` bindings (never
* string interpolation). Two predicate kinds:
*
* - **scalar** filter values collapse into a single jsonb-containment check (`metadata @> ?::jsonb`) —
* exact-match, byte-for-byte the previous behavior so scalar filters stay backward compatible.
* - an **array** filter value is a **match-any** (OR / set membership) check: the record matches when
* its value for that key — scalar or array — shares an element with the filter array. An empty array
* can never match (the `false` deny primitive).
*
* Set membership uses `jsonb_exists_any(jsonb, text[])` — the function form of the `?|` operator, chosen
* because bare `?|` collides with Knex's `?` binding placeholder. The metadata KEY is itself a `?`
* binding (`metadata->?`), so a caller-supplied key can never inject SQL; only the (already validated)
* table/column identifiers are ever spliced in. Returns `{ sql: '', bindings: [] }` when there is no
* filter, preserving the previous unfiltered query shape.
*
* Every binding built here — the key, the scalar containment JSON, each array token — is run through
* {@link stripNulBytes} first, the same as every write path: a filter key or value can carry a NUL byte
* just as easily as stored text can, Postgres rejects it here exactly as it would on a write, and
* stripping it here keeps a filter lookup consistent with the already-stripped data it is matched against.
*/
export declare function buildMetadataWhere(filter: Record<string, unknown> | undefined, metadataColumn: string): {
sql: string;
bindings: unknown[];
};
/** Normalize whatever a Lucid `rawQuery` returns (PG `{rows}`, SQLite array, MySQL `[rows,fields]`). */
export declare function normalizeRows(raw: unknown): Record<string, unknown>[];
/**
* A pgvector-backed {@link VectorStore} — the production RAG adapter, the durable twin of
* {@link import('./memory-vector-store.js').MemoryVectorStore}. It runs over the structural
* {@link LucidRawRunner} (so `@adonisjs/lucid` stays an optional peer — this file imports no Lucid
* types) with raw SQL for the pgvector operators. Similarity ranks via the `<=>`/`<->`/`<#>` operator for
* the configured {@link PgVectorMetric}; the query embedding is always a `?::vector` positional binding
* (NEVER string-interpolated). Scalar metadata filters are an `@> ?::jsonb` containment binding; an
* array filter value is set-membership (`jsonb_exists_any`, the `?|` function form) — the capability-token
* ACL primitive. Only the (validated) table/column identifiers are ever spliced into a statement.
*
* Usually you don't construct this directly: `config/agent.ts` selects it via `retrievers.pgvector({...})`
* and the provider builds it, lazily importing `@adonisjs/lucid` only when the pgvector retriever is
* chosen. Call {@link PgVectorStore.ensureSchema} once (or run the bundled migration) to provision the
* `vector` extension, the chunk table, and the metric's index.
*/
export declare class PgVectorStore implements VectorStore {
protected readonly db: LucidRawRunner;
protected readonly table: string;
/** The single width (fixed-dimension table), or `undefined` for a mixed-dimension table. */
private readonly dimension;
/** The widths with a partial HNSW index, for a mixed-dimension table. */
private readonly indexedDimensions;
private readonly nullableEmbeddings;
private readonly iterativeScan;
private readonly efSearch;
private readonly upsertBatchSize;
private iterativeScanSupport;
private readonly metric;
private readonly metricName;
protected readonly col: Required<PgVectorColumns>;
constructor(db: LucidRawRunner, options?: PgVectorStoreOptions);
/**
* Idempotent DDL — the `vector` extension, the chunk table, and the metric's index. Handy for tests
* and scripts; an AdonisJS app should prefer the bundled migration so the schema is versioned.
*/
ensureSchema(): Promise<void>;
/** The `CREATE EXTENSION` / `CREATE TABLE` / `CREATE INDEX` statements {@link ensureSchema} issues. */
schemaStatements(): string[];
/**
* Every `text`/`jsonb` binding (id, text, source, metadata — including nested metadata values, array
* items and object keys) is run through {@link stripNulBytes} first: Postgres rejects the NUL byte
* (`0x00`) in `text`/`jsonb` outright, while upstream sources (PDF extraction, Qdrant) happily carry
* it, so an unstripped chunk would otherwise fail this INSERT with `invalid byte sequence for encoding
* "UTF8": 0x00`. The embedding is untouched — it is never text/jsonb.
*
* Written as multi-row `INSERT … ON CONFLICT` statements of {@link PgVectorStoreOptions.upsertBatchSize}
* rows each (one round trip per batch instead of per chunk). An empty `embedding` is written as `NULL`.
*/
upsert(records: VectorRecord[]): Promise<void>;
/**
* `documentId` is run through {@link stripNulBytes} before binding — the stored `id` column never
* carries a NUL byte (it went through the same stripping on `upsert`), so a raw NUL in the lookup
* value could only ever fail to match; stripping it here keeps the lookup consistent with the data
* instead of trusting a caller-supplied string to already be clean.
*/
remove(documentId: string): Promise<void>;
/**
* {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
* {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
* into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
* at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
* removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
* lost-update window between a `SELECT` and an `UPDATE`.
*
* The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
* this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
* pgvector never re-parses a vector literal either.
*
* `RETURNING` the id column yields the chunk count without a second query.
*
* The patch is run through {@link stripNulBytes} first (values AND keys) for the same reason as
* {@link PgVectorStore.upsert}: Postgres rejects the NUL byte (`0x00`) in `jsonb`, so an unstripped
* value or key would otherwise fail this `UPDATE`. `documentId` is stripped too, for the same
* lookup-consistency reason as {@link PgVectorStore.remove}.
*/
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
/**
* {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
* {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
* transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
* The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
*/
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
/**
* {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
* same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
*
* Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
* by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
* clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
*/
removeWhere(filter: Record<string, unknown>): Promise<number>;
/**
* Nearest first by the configured metric. On a mixed-dimension table only vectors as wide as the
* query are compared (through that width's partial index when it has one); a chunk without an
* embedding is never returned.
*/
search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
/** Run a read with the configured HNSW settings `SET LOCAL` in one transaction, when possible. */
private withSearchSettings;
/** `hnsw.iterative_scan` exists from pgvector 0.8.0; setting it on an older one is an error. */
private supportsIterativeScan;
}
/** Where the lexical leg of a {@link PgLexicalVectorStore} reads its `tsvector` from. */
export interface PgFullTextOptions {
/**
* Text-search configuration (`simple`, `english`, `portuguese`, …). Default `simple` — no stemming
* or stop words, so it works for any language and matches exact terms. Must be the configuration
* the index was built with, or Postgres cannot use the index and scans the table.
*/
config?: string;
/**
* A `tsvector` column you maintain yourself (typically `GENERATED ALWAYS AS (…) STORED` in a
* migration, e.g. weighting a title above the body) with its own GIN index. When omitted the store
* searches `to_tsvector(config, text)`, and {@link PgVectorStore.schemaStatements} adds a GIN
* expression index over exactly that.
*/
column?: string;
/**
* Search natural-language questions by their meaningful terms (default `true`): the question's
* stop words are dropped (see {@link PgFullTextOptions.stopWords}) and rows holding any remaining
* term (at most 24) are ranked by the sum of the matched terms' IDF among those rows (BM25 without
* term frequencies), `ts_rank_cd` breaking ties; a question with explicit search syntax (a quoted
* phrase, `-word`) is first tried as written. `false`: only `websearch_to_tsquery` on the query as
* written (every word must appear).
*/
anyTermFallback?: boolean;
/**
* Stop word lists dropped from questions, by language; only the list(s) with the most words in the
* question apply, so a word that is a stop word in another language stays a term. Default
* {@link DEFAULT_STOP_WORDS} (English, Portuguese, Spanish: Postgres' Snowball lists). `false` keeps
* every word. With a language `config` (`english`) Postgres drops that language's stop words too.
*/
stopWords?: Readonly<Record<string, ReadonlySet<string>>> | false;
}
/**
* {@link PgVectorStore} plus Postgres full-text search: a {@link LexicalVectorStore}, so a
* {@link import('./lexical-retriever.js').LexicalRetriever} and a
* {@link import('./hybrid-retriever.js').HybridRetriever} can fuse the dense and lexical legs over the
* same rows and ids. `retrievers.pgvector({ fullText: {} })` wires exactly that.
*
* A question is searched by its meaningful terms: its stop words dropped, rows holding any remaining
* term ranked by IDF-weighted coverage, `ts_rank_cd` (cover density) breaking ties (see
* {@link PgFullTextOptions.anyTermFallback}); explicit syntax (quoted phrases, `-exclusions`) goes
* through `websearch_to_tsquery` first.
*
* A separate class rather than a method on {@link PgVectorStore} on purpose: full-text search needs a
* GIN index, which on a populated table belongs in a migration you run and watch (`CREATE INDEX` in
* `ensureSchema` takes a write lock at boot), and a store that silently grew `searchText` on upgrade
* would be picked up by every `isLexicalVectorStore` check and scan an unindexed table.
*/
export declare class PgLexicalVectorStore extends PgVectorStore implements LexicalVectorStore {
private readonly tsConfig;
private readonly tsColumn;
private readonly anyTermFallback;
private readonly stopWords;
constructor(db: LucidRawRunner, options?: PgVectorStoreOptions & {
fullText?: PgFullTextOptions;
});
schemaStatements(): string[];
/**
* Full-text search over the chunk text. `score` is an IDF / `ts_rank_cd` rank — not a similarity,
* and not on the vector leg's scale — so `minScore` is deliberately not applied here.
*/
searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
/** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
private runTextSearch;
/**
* Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
* `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
* once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full of
* common words can match most of the table. Terms are letter/digit runs: space-joined in one
* binding (no array binding), each quoted as a literal lexeme.
*/
private runRankedTermSearch;
/** Must be character-for-character the index expression, or the planner won't use the index. */
private tsvector;
}
/**
* A {@link import('../spi/retriever.js').Retriever} over a {@link PgVectorStore}: embed the query, then
* pgvector-search. The production sibling of the in-memory {@link EmbeddingRetriever} wiring — this is
* what `retrievers.pgvector({...})` builds. Construct it directly for programmatic use, or let the factory.
*/
export declare class PgVectorRetriever extends EmbeddingRetriever {
constructor(embedder: EmbeddingProvider, store: PgVectorStore);
}
//# sourceMappingURL=pg-vector-store.d.ts.map