@gaiaverse/semantic-turning-point-detector/dist/tokensUtil.js

Detects key semantic turning points in conversations using recursive semantic distance analysis. Ideal for conversation analysis, dialogue segmentation, insight detection, and AI-assisted reasoning tasks.

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.countTokens = countTokens;
exports.generateEmbedding = generateEmbedding;
// src/tokenUtils.ts
const gpt_tokenizer_1 = __importDefault(require("gpt-tokenizer"));
const openai_1 = __importDefault(require("openai"));
const dotenv_1 = __importDefault(require("dotenv"));
dotenv_1.default.config();
/**
 * Count tokens in a given string using OpenAI-compatible tokenization.
 *
 * @param text - The text to tokenize
 * @returns Number of tokens
 */
function countTokens(text) {
    return gpt_tokenizer_1.default.encode(text).length;
}
/**
 * Generates an embedding for a given text using the OpenAI API
 * This provides the vector representation for semantic distance calculation
 */
async function generateEmbedding(text, model) {
    const openai = new openai_1.default({ apiKey: process.env.OPENAI_API_KEY });
    const response = await openai.embeddings.create({
        model: model,
        input: text,
        encoding_format: "float",
    });
    return new Float32Array(response.data?.[0].embedding);
}
//# sourceMappingURL=tokensUtil.js.map