@forge-ml/rag
Version:
A RAG (Retrieval-Augmented Generation) package for Forge ML
16 lines (15 loc) • 602 B
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.estimateTokensByLength = exports.estimateTokens = void 0;
const estimateTokens = (chunks) => {
return chunks.reduce((total, chunk) => {
// Estimate tokens using a simple heuristic (4 characters per token)
const estimatedTokens = Math.ceil(chunk.text.length / 4);
return total + estimatedTokens;
}, 0);
};
exports.estimateTokens = estimateTokens;
const estimateTokensByLength = (length) => {
return Math.ceil(length / 4);
};
exports.estimateTokensByLength = estimateTokensByLength;