@adonis-agora/agent
Version:
Governed, durable-backed AI agent (chat + tool-calling + governance) for AdonisJS — part of the Agora ecosystem.
171 lines • 7.91 kB
JavaScript
/** A non-2xx answer (or a malformed one) from an embeddings or rerank endpoint. */
export class HttpModelError extends Error {
status;
constructor(
/** HTTP status; `502` when the endpoint answered 2xx with a body that doesn't fit the contract. */
status, message) {
super(message);
this.status = status;
this.name = 'HttpModelError';
}
}
/**
* Did the server refuse this request for its **size** — too many inputs (TEI's
* `--max-client-batch-size`: `batch size 64 > maximum allowed batch size 32`) or too large a
* payload? Those are fixed by sending fewer inputs per request; anything else (auth, a bad model,
* one input over the context window) is not, and is rethrown as is.
*/
export function isBatchTooLarge(error) {
if (!(error instanceof HttpModelError)) {
return false;
}
if (error.status === 413) {
return true;
}
return ((error.status === 400 || error.status === 422) &&
/batch.?size|too many inputs|maximum allowed batch|payload too large|max.*(inputs|batch)/i.test(error.message));
}
/**
* Batch sizes a server accepted after refusing larger ones, per endpoint URL and model, for the
* life of the process — so a second provider built for the same server (one per request, say)
* starts at the size that works instead of re-learning it with a refused request.
*/
const learnedBatchLimits = new Map();
/**
* An {@link EmbeddingProvider} over any OpenAI-compatible `POST /v1/embeddings` endpoint, with no SDK
* dependency: OpenAI itself, or a gateway/local server speaking the same protocol (TEI, Ollama,
* vLLM, LocalAI…). For a Vercel AI SDK model use `aiSdkEmbedding` from `-ai-sdk` instead.
*
* Batches by {@link OpenAiEmbeddingsOptions.batchSize} — splitting and retrying a batch the server
* refuses as too large, and remembering the size that worked — restores input order from each item's
* `index`, and fails loudly (an {@link HttpModelError}) on a non-2xx answer or a vector count that
* doesn't match the inputs. Empty strings are sent as a single space, since OpenAI rejects them.
* Implements `embedWithUsage` with the provider-reported tokens (summed over the batches) and model, so
* embedding spend reaches the ledger and the quota.
*
* ```ts
* const embedder = openAiEmbeddings({ model: 'text-embedding-3-small', apiKey: process.env.OPENAI_API_KEY });
* const local = openAiEmbeddings({ baseUrl: 'http://tei:8080/v1', model: 'BAAI/bge-m3' });
* ```
*/
export function openAiEmbeddings(options) {
const url = `${(options.baseUrl ?? 'https://api.openai.com/v1').replace(/\/+$/, '')}/embeddings`;
const limitKey = `${url} ${options.model}`;
const configured = Math.max(1, Math.floor(options.batchSize ?? 64));
let batchSize = Math.min(configured, learnedBatchLimits.get(limitKey) ?? configured);
const maxInputChars = options.maxInputChars ?? 24_000;
const timeoutMs = options.timeoutMs ?? 120_000;
const doFetch = options.fetch ?? fetch;
const request = async (input) => {
const response = await doFetch(url, {
method: 'POST',
headers: {
'content-type': 'application/json',
...(options.apiKey ? { authorization: `Bearer ${options.apiKey}` } : {}),
...options.headers,
},
body: JSON.stringify({
model: options.model,
input,
...(options.dimensions !== undefined ? { dimensions: options.dimensions } : {}),
}),
signal: AbortSignal.timeout(timeoutMs),
});
const body = await response.text();
if (!response.ok) {
throw new HttpModelError(response.status, `Embeddings request failed (${response.status}): ${errorMessageOf(body)}`);
}
const json = parseJson(body);
const items = [...(json?.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
if (items.length !== input.length || !items.every((item) => Array.isArray(item.embedding))) {
throw new HttpModelError(502, `Embeddings returned ${items.length} vectors for ${input.length} inputs`);
}
const tokens = json?.usage?.total_tokens ?? json?.usage?.prompt_tokens;
const counted = typeof tokens === 'number' && Number.isFinite(tokens) ? tokens : undefined;
if (counted !== undefined) {
options.onUsage?.(counted);
}
return {
vectors: items.map((item) => item.embedding),
...(counted !== undefined ? { tokens: counted } : {}),
...(typeof json?.model === 'string' ? { modelId: json.model } : {}),
};
};
const merge = (into, from) => {
into.vectors.push(...from.vectors);
if (from.tokens !== undefined) {
into.tokens = (into.tokens ?? 0) + from.tokens;
}
if (into.modelId === undefined && from.modelId !== undefined) {
into.modelId = from.modelId;
}
};
// A batch refused for its size is split in halves (recursively) and retried; the smaller size
// becomes the batch size for every later request to this server and model.
const requestSplitting = async (input) => {
if (input.length > batchSize) {
// The size was learned while this batch was being split: send the rest at the size that works.
const result = { vectors: [] };
for (let start = 0; start < input.length; start += batchSize) {
merge(result, await requestSplitting(input.slice(start, start + batchSize)));
}
return result;
}
try {
return await request(input);
}
catch (error) {
if (!isBatchTooLarge(error) || input.length <= 1) {
throw error;
}
const half = Math.ceil(input.length / 2);
if (half < batchSize) {
batchSize = half;
learnedBatchLimits.set(limitKey, half);
options.onWarn?.(`embeddings server ${url} refused a batch of ${input.length} inputs (${error.message}); retrying in batches of ${half}. Raise the server's batch limit (TEI: --max-client-batch-size) or lower batchSize.`);
}
const result = await requestSplitting(input.slice(0, half));
merge(result, await requestSplitting(input.slice(half)));
return result;
}
};
const provider = {
async embed(texts) {
return (await provider.embedWithUsage(texts)).vectors;
},
async embedWithUsage(texts) {
const result = { vectors: [] };
for (let start = 0; start < texts.length;) {
const input = texts
.slice(start, start + batchSize)
.map((text) => (text.length > maxInputChars ? text.slice(0, maxInputChars) : text) || ' ');
merge(result, await requestSplitting(input));
start += input.length;
}
return {
vectors: result.vectors,
...(result.tokens !== undefined
? { usage: { inputTokens: result.tokens, modelId: result.modelId ?? options.model } }
: {}),
};
},
};
return provider;
}
function parseJson(body) {
try {
return JSON.parse(body);
}
catch {
return null;
}
}
/** The human part of an error body: OpenAI's `error.message`, a bare `message`/`error`, or the text. */
export function errorMessageOf(body) {
const json = parseJson(body);
const message = typeof json?.error === 'string'
? json.error
: (json?.error?.message ?? json?.message ?? undefined);
return typeof message === 'string' ? message : body.slice(0, 300);
}
//# sourceMappingURL=openai-embeddings.js.map