crewai-ts
Version:
TypeScript port of crewAI for agent-based workflows
84 lines (83 loc) • 2.93 kB
JavaScript
/**
* Custom Embedder Implementation
*
* Wrapper for user-provided embedding functions
* with the same optimizations as built-in embedders
*/
import { BaseEmbedder } from './BaseEmbedder.js';
/**
* Custom Embedder Implementation
*
* Allows using any embedding function with the standardized interface
* while benefiting from the optimizations in BaseEmbedder
*/
export class CustomEmbedder extends BaseEmbedder {
_embeddingFunction;
_dimensions;
constructor(options) {
super(options);
if (!options.embeddingFunction) {
throw new Error('Embedding function is required for CustomEmbedder');
}
this._embeddingFunction = options.embeddingFunction;
this._dimensions = options.dimensions;
}
async embed(text) {
if (!text) {
throw new Error('Text is required for embedding');
}
const embedding = await this.executeWithRetry(async () => {
const result = await this.embedText(text);
return result;
});
return this.options.normalize ? this.normalizeVector(embedding) : embedding;
}
async embedBatch(texts) {
if (!texts?.length) {
return [];
}
const embeddings = await this.executeBatchWithRetry(async () => {
const results = await Promise.all(texts.map(text => this.embedText(text)));
return results;
});
return this.options.normalize ? embeddings.map(e => this.normalizeVector(e)) : embeddings;
}
async executeWithRetry(operation, maxRetries, initialBackoff, maxBackoff) {
return await operation();
}
async executeBatchWithRetry(operation, maxRetries, initialBackoff, maxBackoff) {
return await operation();
}
isTransientError(error) {
const message = error.message.toLowerCase();
return (message.includes('timeout') ||
message.includes('network error') ||
message.includes('connection') ||
message.includes('rate limit') ||
message.includes('429') ||
message.includes('500') ||
message.includes('503'));
}
async embedText(text) {
if (!text) {
if (this.options.debug) {
console.warn('Empty text provided for embedding, returning zero vector');
}
return new Float32Array(this._dimensions);
}
const cacheKey = this.generateCacheKey(text);
const cached = this.getCachedEmbedding(cacheKey);
if (cached) {
return cached;
}
try {
const embedding = await this._embeddingFunction(text);
this.cache.set(cacheKey, embedding);
return embedding;
}
catch (error) {
console.error('Custom embedding failed:', error);
return new Float32Array(this._dimensions);
}
}
}