@adonis-agora/agent/dist/src/rag/pg-vector-store.js

Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.

import { EmbeddingRetriever } from './embedding-retriever.js';
import { anyTermTsquery, DEFAULT_STOP_WORDS, hasSearchSyntax, keywordTerms, } from './lexical-query.js';
import { assertRemovalFilter, effectivePatchKeys, filterDeniesAll } from './vector-store.js';
const METRICS = {
    cosine: {
        operator: '<=>',
        opclass: 'vector_cosine_ops',
        score: (col, type = 'vector') => `1 - (${col} <=> ?::${type})`,
    },
    l2: {
        operator: '<->',
        opclass: 'vector_l2_ops',
        score: (col, type = 'vector') => `-(${col} <-> ?::${type})`,
    },
    inner: {
        // pgvector's `<#>` returns the NEGATIVE inner product, so `-(a <#> b)` recovers the inner product
        // and `ORDER BY a <#> b ASC` ranks the largest inner product first — consistent with the others.
        operator: '<#>',
        opclass: 'vector_ip_ops',
        score: (col, type = 'vector') => `-(${col} <#> ?::${type})`,
    },
};
/** Only unqualified SQL identifiers (letter/underscore start, then letters/digits/underscores) are allowed. */
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
/**
 * Reject any table/column name that is not a bare SQL identifier. Table and column names are the only
 * parts of a pgvector statement that CANNOT be a positional binding (an identifier is not a value), so
 * every one is validated here before it is spliced in — closing the injection vector that raw operator
 * SQL would otherwise open. Embeddings, filters, top-K are always `?` bindings and never reach this path.
 */
function assertIdentifier(name, role) {
    if (!IDENTIFIER.test(name)) {
        throw new Error(`Invalid ${role} "${name}": a pgvector identifier must match ${IDENTIFIER.source}`);
    }
    return name;
}
/** pgvector accepts a `'[1,2,3]'` text literal cast to `vector`; the literal itself is a `?` binding. */
export function toVectorLiteral(embedding) {
    return `[${embedding.join(',')}]`;
}
/**
 * Strip every NUL byte (`0x00`) from a value before it can reach a Postgres `text`/`jsonb` binding.
 * Postgres rejects `0x00` in those types outright (`invalid byte sequence for encoding "UTF8": 0x00`),
 * while source systems like Qdrant accept it — so text extracted from PDFs that happens to carry a stray
 * NUL byte works fine right up until it is written to pgvector. Strings have the byte removed (the rest
 * of the text is kept intact); arrays and PLAIN objects are walked recursively, including object KEYS
 * (a NUL in a metadata key would otherwise reach `JSON.stringify` and then Postgres just the same); every
 * other value (numbers, booleans, `null`, `undefined`, embeddings, a `Date`, a class instance, a
 * `Buffer`, …) passes through unchanged, by reference.
 *
 * Two things a naive `for...in` + assignment walk gets wrong, both handled here:
 *
 * - **Only plain objects are walked.** The check is `Object.getPrototypeOf(v) === Object.prototype ||
 *   === null`, never `typeof v === 'object'` alone — a `Date`, a class instance, a `Buffer`, … would
 *   otherwise be torn down into a plain `{}` of its enumerable own properties, silently losing its
 *   prototype (and, for a `Date`, every method on it).
 * - **The result is built with `Object.fromEntries`, never `result[key] = value` on a fresh `{}`.** A
 *   metadata key literally named `__proto__`, assigned that way, does not create an own property — it
 *   hits the inherited `Object.prototype` accessor instead, which (for the ordinary case, a string
 *   value) silently no-ops, so the key is dropped rather than round-tripped. `Object.fromEntries` uses
 *   `CreateDataProperty` under the hood, which always creates a real own property, `__proto__` included.
 */
export function stripNulBytes(value) {
    if (typeof value === 'string') {
        // split/join rather than a regex literal -- a bare NUL inside a RegExp trips biome's
        // noControlCharactersInRegex lint, and split/join needs no escaping at all.
        return value.split('\u0000').join('');
    }
    if (Array.isArray(value)) {
        return value.map((item) => stripNulBytes(item));
    }
    if (value !== null && typeof value === 'object') {
        const proto = Object.getPrototypeOf(value);
        if (proto !== Object.prototype && proto !== null) {
            // Not a plain object -- a Date, a class instance, a Buffer, a Map, ... -- leave it exactly as is.
            return value;
        }
        return Object.fromEntries(Object.entries(value).map(([key, val]) => [
            stripNulBytes(key),
            stripNulBytes(val),
        ]));
    }
    return value;
}
/**
 * Build the metadata `WHERE` fragment for a filter, using Lucid/Knex positional `?` bindings (never
 * string interpolation). Two predicate kinds:
 *
 * - **scalar** filter values collapse into a single jsonb-containment check (`metadata @> ?::jsonb`) —
 *   exact-match, byte-for-byte the previous behavior so scalar filters stay backward compatible.
 * - an **array** filter value is a **match-any** (OR / set membership) check: the record matches when
 *   its value for that key — scalar or array — shares an element with the filter array. An empty array
 *   can never match (the `false` deny primitive).
 *
 * Set membership uses `jsonb_exists_any(jsonb, text[])` — the function form of the `?|` operator, chosen
 * because bare `?|` collides with Knex's `?` binding placeholder. The metadata KEY is itself a `?`
 * binding (`metadata->?`), so a caller-supplied key can never inject SQL; only the (already validated)
 * table/column identifiers are ever spliced in. Returns `{ sql: '', bindings: [] }` when there is no
 * filter, preserving the previous unfiltered query shape.
 *
 * Every binding built here — the key, the scalar containment JSON, each array token — is run through
 * {@link stripNulBytes} first, the same as every write path: a filter key or value can carry a NUL byte
 * just as easily as stored text can, Postgres rejects it here exactly as it would on a write, and
 * stripping it here keeps a filter lookup consistent with the already-stripped data it is matched against.
 */
export function buildMetadataWhere(filter, metadataColumn) {
    if (filter === undefined || Object.keys(filter).length === 0) {
        return { sql: '', bindings: [] };
    }
    const clauses = [];
    const bindings = [];
    // Accumulated as [key, value] pairs, not `scalar[cleanKey] = value` on a fresh `{}` -- a filter key
    // literally named `__proto__` would otherwise hit the inherited `Object.prototype` accessor instead
    // of creating an own property, and (for the ordinary case, a non-object value) that setter silently
    // no-ops, dropping the key before it ever reaches `JSON.stringify`. `Object.fromEntries` below builds
    // the object with `CreateDataProperty`, the same fix {@link stripNulBytes} makes for its own walk.
    const scalarEntries = [];
    for (const [key, value] of Object.entries(filter)) {
        const cleanKey = stripNulBytes(key);
        if (Array.isArray(value)) {
            if (value.length === 0) {
                clauses.push('false');
                continue;
            }
            // Coerce the record's value to an array (a scalar becomes a one-element array) so `?|` set
            // membership works uniformly, then test overlap with the caller's tokens.
            clauses.push(`jsonb_exists_any(CASE WHEN jsonb_typeof(${metadataColumn}->?) = 'array' ` +
                `THEN ${metadataColumn}->? ELSE jsonb_build_array(${metadataColumn}->?) END, ?::text[])`);
            bindings.push(cleanKey, cleanKey, cleanKey, value.map((token) => stripNulBytes(String(token))));
        }
        else {
            scalarEntries.push([cleanKey, value]);
        }
    }
    if (scalarEntries.length > 0) {
        clauses.push(`${metadataColumn} @> ?::jsonb`);
        bindings.push(JSON.stringify(stripNulBytes(Object.fromEntries(scalarEntries))));
    }
    return { sql: `WHERE ${clauses.join(' AND ')}`, bindings };
}
/** Normalize whatever a Lucid `rawQuery` returns (PG `{rows}`, SQLite array, MySQL `[rows,fields]`). */
export function normalizeRows(raw) {
    if (Array.isArray(raw)) {
        if (raw.length > 0 && Array.isArray(raw[0])) {
            return raw[0];
        }
        return raw;
    }
    if (raw !== null && typeof raw === 'object' && Array.isArray(raw.rows)) {
        return raw.rows;
    }
    return [];
}
/** A `jsonb` column comes back parsed (object) on `pg` or as a string on some drivers — accept both. */
function parseMetadata(value) {
    if (value === null || value === undefined)
        return undefined;
    if (typeof value === 'object')
        return value;
    if (typeof value === 'string' && value.length > 0) {
        try {
            return JSON.parse(value);
        }
        catch {
            return undefined;
        }
    }
    return undefined;
}
/**
 * SQL that collapses a chunk id (`${documentId}#<n>`) back to its source document id — the pgvector
 * mirror of {@link import('./vector-store.js').documentIdOf}. Shared by `remove` and `listDocuments` so
 * both key on the exact same definition of "chunk belongs to document". `col` is a validated identifier.
 */
function documentIdExpr(col) {
    return `regexp_replace(${col}, '#[0-9]+$', '')`;
}
/**
 * A pgvector-backed {@link VectorStore} — the production RAG adapter, the durable twin of
 * {@link import('./memory-vector-store.js').MemoryVectorStore}. It runs over the structural
 * {@link LucidRawRunner} (so `@adonisjs/lucid` stays an optional peer — this file imports no Lucid
 * types) with raw SQL for the pgvector operators. Similarity ranks via the `<=>`/`<->`/`<#>` operator for
 * the configured {@link PgVectorMetric}; the query embedding is always a `?::vector` positional binding
 * (NEVER string-interpolated). Scalar metadata filters are an `@> ?::jsonb` containment binding; an
 * array filter value is set-membership (`jsonb_exists_any`, the `?|` function form) — the capability-token
 * ACL primitive. Only the (validated) table/column identifiers are ever spliced into a statement.
 *
 * Usually you don't construct this directly: `config/agent.ts` selects it via `retrievers.pgvector({...})`
 * and the provider builds it, lazily importing `@adonisjs/lucid` only when the pgvector retriever is
 * chosen. Call {@link PgVectorStore.ensureSchema} once (or run the bundled migration) to provision the
 * `vector` extension, the chunk table, and the metric's index.
 */
export class PgVectorStore {
    db;
    table;
    /** The single width (fixed-dimension table), or `undefined` for a mixed-dimension table. */
    dimension;
    /** The widths with a partial HNSW index, for a mixed-dimension table. */
    indexedDimensions;
    nullableEmbeddings;
    iterativeScan;
    efSearch;
    upsertBatchSize;
    iterativeScanSupport;
    metric;
    metricName;
    col;
    constructor(db, options = {}) {
        this.db = db;
        this.table = assertIdentifier(options.table ?? 'agent_rag_chunks', 'table');
        const dimension = options.dimension ?? 1536;
        if (typeof dimension === 'number') {
            this.dimension = dimension;
            this.indexedDimensions = [];
        }
        else {
            for (const width of dimension) {
                if (!Number.isInteger(width) || width <= 0) {
                    throw new Error(`Invalid pgvector dimension ${String(width)}: must be a positive integer`);
                }
            }
            this.dimension = undefined;
            this.indexedDimensions = [...new Set(dimension)];
        }
        this.nullableEmbeddings = options.nullableEmbeddings ?? false;
        this.iterativeScan = options.iterativeScan;
        this.efSearch = options.efSearch;
        this.upsertBatchSize = Math.max(1, Math.floor(options.upsertBatchSize ?? 100));
        this.metricName = options.metric ?? 'cosine';
        this.metric = METRICS[this.metricName];
        const columns = options.columns ?? {};
        this.col = {
            id: assertIdentifier(columns.id ?? 'id', 'column'),
            text: assertIdentifier(columns.text ?? 'text', 'column'),
            source: assertIdentifier(columns.source ?? 'source', 'column'),
            metadata: assertIdentifier(columns.metadata ?? 'metadata', 'column'),
            embedding: assertIdentifier(columns.embedding ?? 'embedding', 'column'),
        };
    }
    /**
     * Idempotent DDL — the `vector` extension, the chunk table, and the metric's index. Handy for tests
     * and scripts; an AdonisJS app should prefer the bundled migration so the schema is versioned.
     */
    async ensureSchema() {
        for (const statement of this.schemaStatements()) {
            await this.db.rawQuery(statement);
        }
    }
    /** The `CREATE EXTENSION` / `CREATE TABLE` / `CREATE INDEX` statements {@link ensureSchema} issues. */
    schemaStatements() {
        const c = this.col;
        const mixed = this.dimension === undefined;
        const statements = [
            'CREATE EXTENSION IF NOT EXISTS vector',
            `CREATE TABLE IF NOT EXISTS ${this.table} (
        ${c.id} TEXT PRIMARY KEY,
        ${c.text} TEXT NOT NULL,
        ${c.source} TEXT,
        ${c.metadata} JSONB,
        ${c.embedding} ${mixed ? 'vector' : `vector(${this.dimension})`}${this.nullableEmbeddings ? '' : ' NOT NULL'}
      )`,
        ];
        if (mixed) {
            for (const width of this.indexedDimensions) {
                statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_${c.embedding}_${width}_idx
        ON ${this.table} USING hnsw ((${c.embedding}::vector(${width})) ${this.metric.opclass})
        WHERE vector_dims(${c.embedding}) = ${width}`);
            }
        }
        else {
            statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_${c.embedding}_idx
        ON ${this.table} USING hnsw (${c.embedding} ${this.metric.opclass})`);
        }
        return statements;
    }
    /**
     * Every `text`/`jsonb` binding (id, text, source, metadata — including nested metadata values, array
     * items and object keys) is run through {@link stripNulBytes} first: Postgres rejects the NUL byte
     * (`0x00`) in `text`/`jsonb` outright, while upstream sources (PDF extraction, Qdrant) happily carry
     * it, so an unstripped chunk would otherwise fail this INSERT with `invalid byte sequence for encoding
     * "UTF8": 0x00`. The embedding is untouched — it is never text/jsonb.
     *
     * Written as multi-row `INSERT … ON CONFLICT` statements of {@link PgVectorStoreOptions.upsertBatchSize}
     * rows each (one round trip per batch instead of per chunk). An empty `embedding` is written as `NULL`.
     */
    async upsert(records) {
        // The id is stripped before de-duplication, so two ids differing only by a NUL byte are the one
        // row they would be stored as. One statement can't touch a row twice (`ON CONFLICT DO UPDATE
        // command cannot affect row a second time`), so duplicates collapse to the LAST occurrence — the
        // same last-write-wins the old row-by-row loop had.
        const byId = new Map();
        for (const record of records) {
            const id = stripNulBytes(record.id);
            byId.delete(id);
            byId.set(id, { ...record, id });
        }
        const unique = [...byId.values()];
        const c = this.col;
        for (let start = 0; start < unique.length; start += this.upsertBatchSize) {
            const batch = unique.slice(start, start + this.upsertBatchSize);
            const bindings = [];
            for (const record of batch) {
                bindings.push(record.id, stripNulBytes(record.text), stripNulBytes(record.source ?? null), record.metadata !== undefined ? JSON.stringify(stripNulBytes(record.metadata)) : null, 
                // An empty embedding is a chunk with no vector yet — NULL (see `nullableEmbeddings`).
                record.embedding.length > 0 ? toVectorLiteral(record.embedding) : null);
            }
            await this.db.rawQuery(`INSERT INTO ${this.table} (${c.id}, ${c.text}, ${c.source}, ${c.metadata}, ${c.embedding})
         VALUES ${batch.map(() => '(?, ?, ?, ?::jsonb, ?::vector)').join(', ')}
         ON CONFLICT (${c.id}) DO UPDATE SET
           ${c.text} = EXCLUDED.${c.text},
           ${c.source} = EXCLUDED.${c.source},
           ${c.metadata} = EXCLUDED.${c.metadata},
           ${c.embedding} = EXCLUDED.${c.embedding}`, bindings);
        }
    }
    /**
     * `documentId` is run through {@link stripNulBytes} before binding — the stored `id` column never
     * carries a NUL byte (it went through the same stripping on `upsert`), so a raw NUL in the lookup
     * value could only ever fail to match; stripping it here keeps the lookup consistent with the data
     * instead of trusting a caller-supplied string to already be clean.
     */
    async remove(documentId) {
        await this.db.rawQuery(`DELETE FROM ${this.table} WHERE ${documentIdExpr(this.col.id)} = ?`, [
            stripNulBytes(documentId),
        ]);
    }
    /**
     * {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
     * {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
     * into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
     * at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
     * removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
     * lost-update window between a `SELECT` and an `UPDATE`.
     *
     * The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
     * this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
     * pgvector never re-parses a vector literal either.
     *
     * `RETURNING` the id column yields the chunk count without a second query.
     *
     * The patch is run through {@link stripNulBytes} first (values AND keys) for the same reason as
     * {@link PgVectorStore.upsert}: Postgres rejects the NUL byte (`0x00`) in `jsonb`, so an unstripped
     * value or key would otherwise fail this `UPDATE`. `documentId` is stripped too, for the same
     * lookup-consistency reason as {@link PgVectorStore.remove}.
     */
    async updateMetadata(documentId, patch) {
        const cleanPatch = stripNulBytes(patch);
        const keys = effectivePatchKeys(cleanPatch);
        if (keys.length === 0) {
            return 0;
        }
        // Accumulated as [key, value] pairs, not `set[key] = cleanPatch[key]` on a fresh `{}` -- see the
        // matching comment in `buildMetadataWhere`: a patch key literally named `__proto__` would otherwise
        // be silently dropped by the inherited prototype setter instead of surviving as an own property.
        const setEntries = [];
        const removed = [];
        for (const key of keys) {
            if (cleanPatch[key] === null) {
                removed.push(key);
            }
            else {
                setEntries.push([key, cleanPatch[key]]);
            }
        }
        const set = Object.fromEntries(setEntries);
        const c = this.col;
        const raw = await this.db.rawQuery(`UPDATE ${this.table}
          SET ${c.metadata} = (COALESCE(${c.metadata}, '{}'::jsonb) || ?::jsonb) - ?::text[]
        WHERE ${documentIdExpr(c.id)} = ?
        RETURNING ${c.id} AS id`, [JSON.stringify(set), removed, stripNulBytes(documentId)]);
        return normalizeRows(raw).length;
    }
    async listDocuments(filter) {
        const c = this.col;
        const docExpr = documentIdExpr(c.id);
        const where = buildMetadataWhere(filter, c.metadata);
        const raw = await this.db.rawQuery(`SELECT DISTINCT ON (${docExpr}) ${docExpr} AS doc_id, ${c.metadata} AS metadata
       FROM ${this.table}
       ${where.sql}
       ORDER BY ${docExpr}`, where.bindings);
        return normalizeRows(raw).map((row) => {
            const metadata = parseMetadata(row.metadata);
            return {
                id: String(row.doc_id),
                ...(metadata !== undefined ? { metadata } : {}),
            };
        });
    }
    /**
     * {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
     * {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
     * transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
     * The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
     */
    async listDocumentIds(filter) {
        const docExpr = documentIdExpr(this.col.id);
        const where = buildMetadataWhere(filter, this.col.metadata);
        const raw = await this.db.rawQuery(`SELECT DISTINCT ${docExpr} AS doc_id
         FROM ${this.table}
         ${where.sql}
        ORDER BY doc_id`, where.bindings);
        return normalizeRows(raw).map((row) => String(row.doc_id));
    }
    /**
     * {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
     * same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
     *
     * Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
     * by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
     * clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
     */
    async removeWhere(filter) {
        assertRemovalFilter(filter);
        if (filterDeniesAll(filter)) {
            return 0;
        }
        const where = buildMetadataWhere(filter, this.col.metadata);
        const raw = await this.db.rawQuery(`DELETE FROM ${this.table} ${where.sql} RETURNING ${this.col.id} AS id`, where.bindings);
        return normalizeRows(raw).length;
    }
    /**
     * Nearest first by the configured metric. On a mixed-dimension table only vectors as wide as the
     * query are compared (through that width's partial index when it has one); a chunk without an
     * embedding is never returned.
     */
    async search(embedding, options) {
        if (embedding.length === 0) {
            return [];
        }
        const c = this.col;
        const vector = toVectorLiteral(embedding);
        // A fixed-width table keeps the exact expression its index was built on (`embedding`); a mixed one
        // casts both sides to the query's width so the planner can match that width's partial index.
        let vectorType = 'vector';
        const shapeClauses = [];
        if (this.dimension === undefined) {
            const width = embedding.length;
            if (this.indexedDimensions.includes(width)) {
                vectorType = `vector(${width})`;
            }
            shapeClauses.push(`${c.embedding} IS NOT NULL`, `vector_dims(${c.embedding}) = ${width}`);
        }
        else if (this.nullableEmbeddings) {
            shapeClauses.push(`${c.embedding} IS NOT NULL`);
        }
        const column = vectorType === 'vector' ? c.embedding : `(${c.embedding}::${vectorType})`;
        const scoreExpr = this.metric.score(column, vectorType);
        const meta = buildMetadataWhere(options.filter, c.metadata);
        // Combine the metadata predicate with an optional minScore relevance floor. Filtering the score
        // expression in SQL (not in JS after the fact) keeps the floor applied BEFORE `LIMIT`, so the K
        // returned are all above the floor. `buildMetadataWhere` prefixes `WHERE ` (6 chars) — strip it so
        // the clauses can be re-joined with the floor via AND. When neither is present the query shape and
        // binding order are byte-for-byte identical to before.
        const clauses = [];
        const whereBindings = [];
        if (meta.sql !== '') {
            clauses.push(meta.sql.slice('WHERE '.length));
            whereBindings.push(...meta.bindings);
        }
        clauses.push(...shapeClauses);
        if (options.minScore !== undefined) {
            // The score expression carries its own `?::vector`, so the embedding is bound again here.
            clauses.push(`${scoreExpr} >= ?`);
            whereBindings.push(vector, options.minScore);
        }
        const whereSql = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
        // Bindings are consumed positionally, in the exact order the `?`s appear: score expr (embedding),
        // then the optional filter/minScore clause(s), then the ORDER BY embedding, then the LIMIT. The
        // embedding is bound once per `?::vector` because a positional `?` cannot be reused like a `$1`.
        const bindings = [vector, ...whereBindings, vector, options.topK];
        const sql = `SELECT ${c.id} AS id, ${c.text} AS text, ${c.source} AS source, ${c.metadata} AS metadata,
              ${scoreExpr} AS score
       FROM ${this.table}
       ${whereSql}
       ORDER BY ${column} ${this.metric.operator} ?::${vectorType}
       LIMIT ?`;
        const raw = await this.withSearchSettings((runner) => runner.rawQuery(sql, bindings));
        return normalizeRows(raw).map(toPassage);
    }
    /** Run a read with the configured HNSW settings `SET LOCAL` in one transaction, when possible. */
    async withSearchSettings(read) {
        const transaction = this.db.transaction;
        if (typeof transaction !== 'function') {
            return read(this.db);
        }
        const iterative = this.iterativeScan !== undefined && (await this.supportsIterativeScan());
        if (!iterative && this.efSearch === undefined) {
            return read(this.db);
        }
        return transaction.call(this.db, async (trx) => {
            // Lucid's transaction client is a full query client (it has `rawQuery`); the structural type
            // this file sees only promises `from`/`table`, hence the check.
            const runner = trx;
            if (typeof runner.rawQuery !== 'function') {
                return read(this.db);
            }
            const tx = runner;
            if (iterative) {
                await tx.rawQuery(`SELECT set_config('hnsw.iterative_scan', ?, true)`, [
                    this.iterativeScan,
                ]);
            }
            if (this.efSearch !== undefined) {
                await tx.rawQuery(`SELECT set_config('hnsw.ef_search', ?, true)`, [String(this.efSearch)]);
            }
            return read(tx);
        });
    }
    /** `hnsw.iterative_scan` exists from pgvector 0.8.0; setting it on an older one is an error. */
    supportsIterativeScan() {
        this.iterativeScanSupport ??= this.db
            .rawQuery(`SELECT extversion FROM pg_extension WHERE extname = 'vector'`)
            .then((raw) => {
            const version = String(normalizeRows(raw)[0]?.extversion ?? '0.0');
            const [major = 0, minor = 0] = version.split('.').map(Number);
            return major > 0 || minor >= 8;
        })
            .catch(() => false);
        return this.iterativeScanSupport;
    }
}
/**
 * {@link PgVectorStore} plus Postgres full-text search: a {@link LexicalVectorStore}, so a
 * {@link import('./lexical-retriever.js').LexicalRetriever} and a
 * {@link import('./hybrid-retriever.js').HybridRetriever} can fuse the dense and lexical legs over the
 * same rows and ids. `retrievers.pgvector({ fullText: {} })` wires exactly that.
 *
 * A question is searched by its meaningful terms: its stop words dropped, rows holding any remaining
 * term ranked by IDF-weighted coverage, `ts_rank_cd` (cover density) breaking ties (see
 * {@link PgFullTextOptions.anyTermFallback}); explicit syntax (quoted phrases, `-exclusions`) goes
 * through `websearch_to_tsquery` first.
 *
 * A separate class rather than a method on {@link PgVectorStore} on purpose: full-text search needs a
 * GIN index, which on a populated table belongs in a migration you run and watch (`CREATE INDEX` in
 * `ensureSchema` takes a write lock at boot), and a store that silently grew `searchText` on upgrade
 * would be picked up by every `isLexicalVectorStore` check and scan an unindexed table.
 */
export class PgLexicalVectorStore extends PgVectorStore {
    tsConfig;
    tsColumn;
    anyTermFallback;
    stopWords;
    constructor(db, options = {}) {
        super(db, options);
        this.tsConfig = assertIdentifier(options.fullText?.config ?? 'simple', 'text search config');
        this.tsColumn =
            options.fullText?.column !== undefined
                ? assertIdentifier(options.fullText.column, 'column')
                : undefined;
        this.anyTermFallback = options.fullText?.anyTermFallback ?? true;
        const stopWords = options.fullText?.stopWords;
        this.stopWords = stopWords === false ? {} : (stopWords ?? DEFAULT_STOP_WORDS);
    }
    schemaStatements() {
        const statements = super.schemaStatements();
        if (this.tsColumn === undefined) {
            statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_${this.col.text}_tsv_idx
        ON ${this.table} USING gin (${this.tsvector()})`);
        }
        return statements;
    }
    /**
     * Full-text search over the chunk text. `score` is an IDF / `ts_rank_cd` rank — not a similarity,
     * and not on the vector leg's scale — so `minScore` is deliberately not applied here.
     */
    async searchText(query, options) {
        if ((options.filter !== undefined && filterDeniesAll(options.filter)) || query.trim() === '') {
            return [];
        }
        if (!this.anyTermFallback || hasSearchSyntax(query)) {
            const strict = await this.runTextSearch(query, options);
            if (strict.length > 0 || !this.anyTermFallback) {
                return strict;
            }
        }
        const terms = keywordTerms(query, 24, this.stopWords);
        return terms.length === 0 ? [] : this.runRankedTermSearch(terms, options);
    }
    /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
    async runTextSearch(query, options) {
        const c = this.col;
        const where = buildMetadataWhere(options.filter, c.metadata);
        // The parsed query is a lateral relation, so the one `?` binding feeds both the match and the rank.
        const clauses = [`${this.tsvector()} @@ __tsq.q`];
        if (where.sql !== '') {
            clauses.push(where.sql.slice('WHERE '.length));
        }
        const raw = await this.db.rawQuery(`SELECT ${c.id} AS id, ${c.text} AS text, ${c.source} AS source, ${c.metadata} AS metadata,
              ts_rank_cd(${this.tsvector()}, __tsq.q) AS score
       FROM ${this.table}, websearch_to_tsquery('${this.tsConfig}', ?) AS __tsq(q)
       WHERE ${clauses.join(' AND ')}
       ORDER BY score DESC, ${c.id}
       LIMIT ?`, [query, ...where.bindings, options.topK]);
        return normalizeRows(raw).map(toPassage);
    }
    /**
     * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
     * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
     * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full of
     * common words can match most of the table. Terms are letter/digit runs: space-joined in one
     * binding (no array binding), each quoted as a literal lexeme.
     */
    async runRankedTermSearch(terms, options) {
        const c = this.col;
        const cfg = this.tsConfig;
        const where = buildMetadataWhere(options.filter, c.metadata);
        const clauses = [`${this.tsvector()} @@ __any.q`];
        if (where.sql !== '') {
            clauses.push(where.sql.slice('WHERE '.length));
        }
        const raw = await this.db.rawQuery(`SELECT id, text, source, metadata, score FROM (
         WITH __terms AS (
           SELECT DISTINCT to_tsquery('${cfg}', quote_literal(t)) AS q
           FROM regexp_split_to_table(?, ' ') AS t
         ), __cand AS MATERIALIZED (
           SELECT ${c.id} AS id, ${c.text} AS text, ${c.source} AS source, ${c.metadata} AS metadata,
                  ${this.tsvector()} AS __tsv, __any.q AS __q
           FROM ${this.table}, to_tsquery('${cfg}', ?) AS __any(q)
           WHERE ${clauses.join(' AND ')}
         ), __hits AS MATERIALIZED (
           SELECT __cand.id AS __id, __terms.q FROM __cand JOIN __terms ON __cand.__tsv @@ __terms.q
         ), __idf AS (
           SELECT q, ln(1 + ((SELECT count(*) FROM __cand) - count(*) + 0.5) / (count(*) + 0.5)) AS w
           FROM __hits GROUP BY q
         ), __scored AS (
           SELECT __hits.__id, sum(__idf.w) AS s, rank() OVER (ORDER BY sum(__idf.w) DESC) AS r
           FROM __hits JOIN __idf USING (q) GROUP BY __hits.__id
         )
         SELECT __cand.id, __cand.text, __cand.source, __cand.metadata,
                __scored.s AS __s, ts_rank_cd(__cand.__tsv, __cand.__q, 32) AS __tr,
                __scored.s + ts_rank_cd(__cand.__tsv, __cand.__q, 32) AS score
         FROM __scored JOIN __cand ON __cand.id = __scored.__id
         WHERE __scored.r <= ?
       ) AS __ranked
       ORDER BY __s DESC, __tr DESC, id
       LIMIT ?`, [terms.join(' '), anyTermTsquery(terms), ...where.bindings, options.topK, options.topK]);
        return normalizeRows(raw).map(toPassage);
    }
    /** Must be character-for-character the index expression, or the planner won't use the index. */
    tsvector() {
        return this.tsColumn ?? `to_tsvector('${this.tsConfig}'::regconfig, ${this.col.text})`;
    }
}
function toPassage(row) {
    const metadata = parseMetadata(row.metadata);
    const source = row.source;
    return {
        id: String(row.id),
        text: String(row.text),
        score: Number(row.score),
        ...(source !== null && source !== undefined ? { source: String(source) } : {}),
        ...(metadata !== undefined ? { metadata } : {}),
    };
}
/**
 * A {@link import('../spi/retriever.js').Retriever} over a {@link PgVectorStore}: embed the query, then
 * pgvector-search. The production sibling of the in-memory {@link EmbeddingRetriever} wiring — this is
 * what `retrievers.pgvector({...})` builds. Construct it directly for programmatic use, or let the factory.
 */
export class PgVectorRetriever extends EmbeddingRetriever {
    // biome-ignore lint/complexity/noUselessConstructor: not useless — it NARROWS `store` from the base's `VectorStore` to `PgVectorStore`, which is this subclass's whole point. Delete it and `new PgVectorRetriever(embedder, anyInMemoryStore)` starts compiling.
    constructor(embedder, store) {
        super(embedder, store);
    }
}
//# sourceMappingURL=pg-vector-store.js.map