@forge-ml/rag/dist/utils/tokenEstimate.js

A RAG (Retrieval-Augmented Generation) package for Forge ML

export const estimateTokens = (chunks) => {
    return chunks.reduce((total, chunk) => {
        // Estimate tokens using a simple heuristic (4 characters per token)
        const estimatedTokens = Math.ceil(chunk.text.length / 4);
        return total + estimatedTokens;
    }, 0);
};
export const estimateTokensByLength = (length) => {
    return Math.ceil(length / 4);
};