@gaiaverse/semantic-turning-point-detector
Version:
Detects key semantic turning points in conversations using recursive semantic distance analysis. Ideal for conversation analysis, dialogue segmentation, insight detection, and AI-assisted reasoning tasks.
895 lines (891 loc) • 105 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SemanticTurningPointDetector = void 0;
// file: semanticTurningPointDetector.ts
const fs_extra_1 = __importDefault(require("fs-extra"));
const winston_1 = __importDefault(require("winston"));
const ollama_1 = require("ollama");
const dotenv_1 = __importDefault(require("dotenv"));
dotenv_1.default.config();
// setup winston
/*****************************************************************************************
* SEMANTIC TURNING POINT DETECTOR
*
* A TypeScript implementation of the Adaptive Recursive Convergence (ARC) with
* Cascading Re-Dimensional Attention (CRA) framework for conversation analysis.
*
* This detector identifies semantic "Turning Points" in conversations as a concrete
* application of the ARC/CRA theoretical framework for multi-step reasoning
* and dynamic dimensional expansion.
*
* Framework implementation:
* 1. Analyze semantic relationships between messages using embeddings (dimension n)
* 2. Calculate semantic distances that correspond to the contraction mapping
* 3. Apply the complexity function χ to determine dimensional saturation
* 4. Use the transition operator Ψ to determine whether to stay in dimension n or escalate
* 5. Employ meta-messages and recursive analysis for dimensional expansion (n → n+1)
* 6. Merge and prune results to demonstrate formal convergence
*****************************************************************************************/
const async_1 = __importDefault(require("async"));
const openai_1 = require("openai");
const lru_cache_1 = require("lru-cache");
const crypto_1 = __importDefault(require("crypto"));
const tokensUtil_1 = require("./tokensUtil");
const Message_1 = require("./Message");
const stripContent_1 = require("./stripContent");
const prompt_1 = require("./prompt");
const types_1 = require("./types");
// Cache for token counts to avoid recalculating - implements atomic memory concept
const tokenCountCache = new lru_cache_1.LRUCache({
max: 10000,
ttl: 1000 * 60 * 60 * 24,
});
// -----------------------------------------------------------------------------
// Main Detector Class
// -----------------------------------------------------------------------------
class SemanticTurningPointDetector {
config;
/**
* For ease of use in llm requests, openai's client is used as it allows configurable endpoints. Further expoloration might be reasonable in leveraging other libaries, such as ollama, llmstudio, genai, etc, for more direct compatibility with other LLM providers. Though at this time, the OpenAI client is sufficient for requests done by this detector.
*/
openai;
/**
* This provides the array of the initial messages that were passed to the detector. This is noted as such as throughout the process, ARC involves analyzing subsets of the original messages, and the original messages are not modified.
*/
originalMessages = [];
/**
* AN array of changes of state across iterations, used for convergence measurement.
* This is used to track the evolution of turning points across iterations and dimensions.
* This is used when returning the final results, to determine whether the turning points have converged.
*/
convergenceHistory = [];
/**
* Used to help mitigate repeat embedding requests for the same message content. And can be configured to avoid excessive RAM usage via `embeddingCacheRamLimitMB`.
*/
embeddingCache;
endpointType;
ollama = null;
logger;
/**
* Creates a new instance of the semantic turning point detector
*/
constructor(config = {}) {
// Default configuration (from your provided code)
this.config = {
apiKey: config.apiKey || process.env.OPENAI_API_KEY || "",
classificationModel: config.classificationModel || "gpt-4o-mini",
embeddingModel: config.embeddingModel || "text-embedding-3-small",
embeddingEndpoint: config.embeddingEndpoint,
semanticShiftThreshold: config.semanticShiftThreshold || 0.22,
minTokensPerChunk: config.minTokensPerChunk || 250,
maxTokensPerChunk: config.maxTokensPerChunk || 2000,
concurrency: (config.concurrency ?? config?.endpoint) ? 1 : 4,
embeddingConcurrency: config.embeddingConcurrency ?? 5,
logger: config?.logger ?? undefined,
embeddingCacheRamLimitMB: config.embeddingCacheRamLimitMB || 256,
maxRecursionDepth: config.maxRecursionDepth || 3,
onlySignificantTurningPoints: config.onlySignificantTurningPoints ?? true,
significanceThreshold: config.significanceThreshold || 0.5,
minMessagesPerChunk: config.minMessagesPerChunk || 3,
maxTurningPoints: config.maxTurningPoints || 5,
debug: config.debug || false,
turningPointCategories: config?.turningPointCategories &&
config?.turningPointCategories.length > 0
? config.turningPointCategories
: types_1.turningPointCategories,
endpoint: config.endpoint,
temperature: config?.temperature ?? 0.6,
top_p: config?.top_p ?? 0.95,
complexitySaturationThreshold: config.complexitySaturationThreshold || 4.5,
measureConvergence: config.measureConvergence ?? true,
};
this.endpointType = config?.endpoint
? config.endpoint.includes("api.openai.com")
? "unknown"
: "unknown"
: "openai";
if (this.config.logger === undefined) {
fs_extra_1.default.ensureDirSync("results");
this.logger = winston_1.default.createLogger({
level: "info",
format: winston_1.default.format.combine(winston_1.default.format.timestamp(), winston_1.default.format.json()),
transports: [
new winston_1.default.transports.Console({
format: winston_1.default.format.combine(winston_1.default.format.colorize(), winston_1.default.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), winston_1.default.format.printf(({ timestamp, level, message }) => {
return `${timestamp} ${level}: ${message}`;
})),
}),
new winston_1.default.transports.File({
filename: "results/semanticTurningPointDetector.log",
format: winston_1.default.format.json(),
}),
],
});
}
// now validate the turning point categories (that wil simply log warnings), and also after the logging is setup above.
if (config?.turningPointCategories &&
config?.turningPointCategories.length > 0) {
this.validateTurningPointCategories(config.turningPointCategories);
}
// Initialize OpenAI client
this.openai = new openai_1.OpenAI({
apiKey: this.config.apiKey ??
process.env.LLM_API_KEY ??
process.env.OPENAI_API_KEY,
baseURL: this.config.endpoint,
});
/**
* Initialize the embedding cache with the specified RAM limit.
*/
this.embeddingCache = (0, tokensUtil_1.createEmbeddingCache)(this.config.embeddingCacheRamLimitMB);
if (this.config.debug) {
this.logger.info("[TurningPointDetector] Initialized with config:", {
...this.config,
apiKey: "[REDACTED]",
});
this.logger.info(`[TurningPointDetector] Embedding cache initialized with ${this.embeddingCache.max} max entries (${this.config.embeddingCacheRamLimitMB}MB limit)`);
}
}
getModelName() {
return this.config.classificationModel;
}
/**
* Main entry point: Detect turning points in a conversation
* Implements the full ARC/CRA framework
*/
async detectTurningPoints(messages) {
this.logger.info("Starting turning-point detection (ARC/CRA) on", messages.length, "messages");
this.convergenceHistory = [];
const isEndpointOllamaBased = await this.isOllamaEndpoint(this.config.endpoint);
if (isEndpointOllamaBased) {
this.endpointType = "ollama";
const url = new URL(this.config.endpoint);
const host = `${url.protocol}//${url.hostname}${url.port ? `:${url.port}` : ""}`;
this.logger.info(`Detected Ollama endpoint: ${host}. Initializing Ollama client.`);
this.ollama = new ollama_1.Ollama({
host,
});
}
// ── cache original conversation for downstream helpers
const totalTokens = await this.getMessageArrayTokenCount(messages);
this.logger.info(`Total conversation tokens: ${totalTokens}`);
this.originalMessages = messages.map((m) => ({ ...m }));
// ── 1️⃣ full multi-layer detection (dim-0 entry)
const turningPointsFound = await this.multiLayerDetection(messages, 0);
this.logger.info(`Multi-layer detection returned ${turningPointsFound.length} turning points`);
// ── 2️⃣ compute a per-TP confidence score
const confidenceScoresByPoint = new Array(turningPointsFound.length).fill(0);
// helper to collapse per-message embeddings into a single mean vector
const meanEmbedding = (embs) => {
if (embs.length === 0)
return new Float32Array(1536);
const dim = embs[0].embedding.length;
const softMax = (values) => {
const maxVal = Math.max(...values);
const exps = values.map((v) => Math.exp(v - maxVal));
const sumExps = exps.reduce((sum, v) => sum + v, 0);
return exps.map((v) => v / sumExps);
};
const magnitudes = embs.map(({ embedding }) => Math.sqrt(embedding.reduce((sum, v) => sum + v * v, 0)));
const attnWeights = softMax(magnitudes);
const acc = new Float32Array(dim);
for (let idx = 0; idx < embs.length; idx++) {
const { embedding } = embs[idx];
const weight = attnWeights[idx];
for (let i = 0; i < dim; i++) {
acc[i] += embedding[i] * weight;
}
}
return acc;
};
await async_1.default.eachOfLimit(turningPointsFound, this.config.concurrency, async (tp, idxStr) => {
const idx = Number(idxStr);
// slice conversation around this TP
const pre = messages.slice(0, tp.span.startIndex);
const turn = messages.slice(tp.span.startIndex, tp.span.endIndex + 1);
const post = messages.slice(tp.span.endIndex + 1);
if (pre.length === 0 || post.length === 0) {
this.logger.info(`TP ${tp.id} at edges of convo – skipping confidence`);
confidenceScoresByPoint[idx] = 0;
return;
}
// generate *per message* embeddings for each slice
const [preE, turnE, postE] = await Promise.all([
this.generateMessageEmbeddings(pre, 0),
this.generateMessageEmbeddings(turn, 0),
this.generateMessageEmbeddings(post, 0),
]);
// collapse to single vectors
const vPre = meanEmbedding(preE);
const vTurn = meanEmbedding(turnE);
const vPost = meanEmbedding(postE);
// distance (0-1) – higher when meaning shifts
const distPre = this.calculateSemanticDistance(vPre, vTurn);
const distPost = this.calculateSemanticDistance(vTurn, vPost);
// simple confidence: average outward semantic shift
confidenceScoresByPoint[idx] = (distPre + distPost) / 2;
this.logger.info(`TP ${tp.id}: distPre=${distPre.toFixed(3)}, distPost=${distPost.toFixed(3)}, conf=${confidenceScoresByPoint[idx].toFixed(3)}`);
});
// ── 3️⃣ aggregate conversation-level confidence (mean of non-zero scores)
const valid = confidenceScoresByPoint.filter((v) => v > 0);
const aggregateConfidence = valid.length === 0 ? 0 : valid.reduce((s, v) => s + v, 0) / valid.length;
this.logger.info(`Aggregate confidence for conversation: ${aggregateConfidence.toFixed(3)}`);
return {
confidence: aggregateConfidence,
points: turningPointsFound,
};
}
/**
* Multi-layer detection implementing the ARC/CRA dimensional processing
* This is the primary implementation of the transition operator Ψ
*/
async multiLayerDetection(messages, dimension) {
this.logger.info(`Starting dimensional analysis at n=${dimension}`);
// Check recursion depth - hard limit on dimensional expansion
if (dimension >= this.config.maxRecursionDepth) {
this.logger.info(`Maximum dimension (n=${dimension}) reached, processing directly without further expansion`);
// Pass originalMessages context only at dimension 0 if needed by detectTurningPointsInChunk->classifyTurningPoint
return await this.detectTurningPointsInChunk(messages, dimension, 0, this.originalMessages);
}
// For very small conversations (or at deeper levels), use sliding window
let localTurningPoints = [];
// Adjusted condition to handle small message counts more directly
if (messages.length < this.config.minMessagesPerChunk * 2 &&
dimension === 0) {
this.logger.info(`Dimension ${dimension}: Small conversation (${messages.length} msgs), processing directly`);
// Optionally adjust threshold for small conversations
const originalThreshold = this.config.semanticShiftThreshold;
this.config.semanticShiftThreshold = Math.max(0.3, originalThreshold * 1.1); // Slightly higher threshold
localTurningPoints = await this.detectTurningPointsInChunk(messages, dimension, 0, this.originalMessages);
// Restore config
this.config.semanticShiftThreshold = originalThreshold;
}
else {
// Chunk the conversation
const { chunks } = await this.chunkConversation(messages, dimension);
this.logger.info(`Dimension ${dimension}: Split into ${chunks.length} chunks`);
if (chunks.length === 0) {
this.logger.info(`Dimension ${dimension}: No valid chunks created, returning empty.`);
return [];
}
// Process each chunk in parallel to find local turning points
const chunkTurningPoints = new Array(chunks.length);
const durationsSeconds = new Array(chunks.length).fill(-1);
const limit = this.config.concurrency;
await async_1.default.eachOfLimit(chunks, limit, async (chunk, indexStr) => {
const index = Number(indexStr);
const startTime = Date.now();
if (index % 10 === 0 || limit < 10 || this.config.debug) {
this.logger.info(` - Dimension ${dimension}: Processing chunk ${index + 1}/${chunks.length} (${chunk.length} messages)`);
}
// Pass originalMessages context only at dimension 0
chunkTurningPoints[index] = await this.detectTurningPointsInChunk(chunk, dimension, index, this.originalMessages);
const durationSecs = (Date.now() - startTime) / 1000;
durationsSeconds[index] = durationSecs;
if (index % 10 === 0 || limit < 10 || this.config.debug) {
const processedCount = durationsSeconds.filter((d) => d > 0).length;
if (processedCount > 0) {
const averageDuration = durationsSeconds.filter((d) => d > 0).reduce((a, b) => a + b, 0) /
processedCount;
const remainingChunks = durationsSeconds.length - processedCount;
const remainingTime = (averageDuration * remainingChunks).toFixed(1);
const percentageComplete = (processedCount / durationsSeconds.length) * 100;
this.logger.info(` - Chunk ${index + 1} processed in ${durationSecs.toFixed(1)}s. Est. remaining: ${remainingTime}s (${percentageComplete.toFixed(1)}% complete)`);
}
else {
this.logger.info(` - Chunk ${index + 1} processed in ${durationSecs.toFixed(1)}s.`);
}
}
});
// Flatten all turning points from all chunks
localTurningPoints = chunkTurningPoints.flat();
}
this.logger.info(`Dimension ${dimension}: Found ${localTurningPoints.length} raw turning points`);
// If we found zero or one turning point at this level, return it directly (after potential filtering if needed)
if (localTurningPoints.length <= 1) {
// Apply filtering even for single points if configured
return this.config.onlySignificantTurningPoints
? this.filterSignificantTurningPoints(localTurningPoints)
: localTurningPoints;
}
// First merge any similar turning points at this level
const mergedLocalTurningPoints = this.mergeSimilarTurningPoints(localTurningPoints);
this.logger.info(`Dimension ${dimension}: Merged similar TPs to ${mergedLocalTurningPoints.length}`);
// If merging resulted in 0 or 1 TP, return it (after filtering)
if (mergedLocalTurningPoints.length <= 1) {
return this.config.onlySignificantTurningPoints
? this.filterSignificantTurningPoints(mergedLocalTurningPoints)
: mergedLocalTurningPoints;
}
// ------------------- CRITICAL ARC/CRA IMPLEMENTATION -------------------
// Determine whether to expand dimension based on complexity saturation
// Calculate the maximum complexity in this dimension
const maxComplexity = Math.max(0, ...mergedLocalTurningPoints.map((tp) => tp.complexityScore)); // Ensure non-negative
// Implement Transition Operator Ψ
const needsDimensionalEscalation = maxComplexity >= this.config.complexitySaturationThreshold;
this.logger.info(`Dimension ${dimension}: Max complexity = ${maxComplexity.toFixed(2)}, Saturation threshold = ${this.config.complexitySaturationThreshold}`);
this.logger.info(`Dimension ${dimension}: Needs Escalation (Ψ)? ${needsDimensionalEscalation}`);
// Conditions to STOP escalation and finalize at this dimension:
// 1. Max recursion depth reached
// 2. Too few turning points to warrant higher-level analysis
// 3. Complexity hasn't saturated (no need to escalate)
if (dimension >= this.config.maxRecursionDepth - 1 ||
mergedLocalTurningPoints.length <= 2 || // Adjusted slightly, maybe 2 TPs isn't enough to find meta-patterns
!needsDimensionalEscalation) {
this.logger.info(`Dimension ${dimension}: Finalizing at this level. Applying final filtering.`);
// Track convergence for this dimension
if (this.config.measureConvergence) {
this.convergenceHistory.push({
previousTurningPoints: [], // No previous state at the final level of processing
currentTurningPoints: mergedLocalTurningPoints, // TPs before final filtering
dimension,
distanceMeasure: 0, // No comparison needed at final step
hasConverged: true, // Considered converged as processing stops here
didEscalate: false,
});
}
// Filter the merged points before returning
return this.filterSignificantTurningPoints(mergedLocalTurningPoints);
}
// ----- DIMENSIONAL ESCALATION (n → n+1) -----
this.logger.info(`Dimension ${dimension}: Escalating to dimension ${dimension + 1}`);
// Create meta-messages from the merged turning points at this level
// Pass originalMessages for context if needed by createMetaMessagesFromTurningPoints
const metaMessages = this.createMetaMessagesFromTurningPoints(mergedLocalTurningPoints, this.originalMessages);
this.logger.info(`Dimension ${dimension}: Created ${metaMessages.length} meta-messages for dimension ${dimension + 1}`);
if (metaMessages.length < 2) {
this.logger.info(`Dimension ${dimension}: Not enough meta-messages (${metaMessages.length}) to perform higher-level analysis. Finalizing with current TPs.`);
if (this.config.measureConvergence) {
this.convergenceHistory.push({
previousTurningPoints: mergedLocalTurningPoints, // State before attempted escalation
currentTurningPoints: mergedLocalTurningPoints, // State after failed escalation
dimension: dimension + 1, // Represents the attempted next dimension
distanceMeasure: 0, // No change
hasConverged: true, // Converged because escalation failed
didEscalate: false, // Escalation attempted but yielded no processable result
});
}
return this.filterSignificantTurningPoints(mergedLocalTurningPoints);
}
// Recursively process the meta-messages to find higher-dimensional turning points
const higherDimensionTurningPoints = await this.multiLayerDetection(metaMessages, dimension + 1);
this.logger.info(`Dimension ${dimension + 1}: Found ${higherDimensionTurningPoints.length} higher-dimension TPs.`);
// Track convergence and dimension escalation
if (this.config.measureConvergence) {
const convergenceState = {
previousTurningPoints: mergedLocalTurningPoints, // TPs from dim n
currentTurningPoints: higherDimensionTurningPoints, // TPs found in dim n+1
dimension: dimension + 1,
distanceMeasure: this.calculateStateDifference(mergedLocalTurningPoints, higherDimensionTurningPoints),
hasConverged: higherDimensionTurningPoints.length > 0, // Converged if TPs were found at higher level
didEscalate: true,
};
this.convergenceHistory.push(convergenceState);
this.logger.info(`Dimension ${dimension} → ${dimension + 1}: Convergence distance: ${convergenceState.distanceMeasure.toFixed(3)}. Converged: ${convergenceState.hasConverged}`);
}
// Combine turning points from local (n) and higher (n+1) dimensions
// The combine function will handle merging, prioritizing higher-dim, and filtering
return this.combineTurningPoints(mergedLocalTurningPoints, higherDimensionTurningPoints);
}
/**
* Calculate a difference measure between two states (sets of turning points)
* Used for convergence tracking. Considers significance and location.
*/
calculateStateDifference(state1, state2) {
// Handle empty states
if (state1.length === 0 && state2.length === 0)
return 0.0; // No difference
if (state1.length === 0 || state2.length === 0)
return 1.0; // Maximum difference
// 1. Average Significance Difference
const avgSig1 = state1.reduce((sum, tp) => sum + tp.significance, 0) / state1.length;
const avgSig2 = state2.reduce((sum, tp) => sum + tp.significance, 0) / state2.length;
const sigDiff = Math.abs(avgSig1 - avgSig2); // Range [0, 1]
// 2. Structural Difference (using Jaccard index on span ranges)
const spans1 = new Set(state1.map((tp) => `${tp.span.startIndex}-${tp.span.endIndex}`));
const spans2 = new Set(state2.map((tp) => `${tp.span.startIndex}-${tp.span.endIndex}`));
const intersection = new Set([...spans1].filter((span) => spans2.has(span)));
const union = new Set([...spans1, ...spans2]);
const jaccardDistance = union.size > 0 ? 1.0 - intersection.size / union.size : 0.0; // Range [0, 1]
// Combine the measures (e.g., weighted average)
const combinedDistance = sigDiff * 0.5 + jaccardDistance * 0.5;
return Math.min(1.0, Math.max(0.0, combinedDistance)); // Ensure bounds [0, 1]
}
/**
* Apply complexity function χ from the ARC/CRA framework
*/
calculateComplexityScore(significance, semanticShiftMagnitude) {
// Base complexity from significance (maps [0,1] to [1, 5])
let complexity = 1 + significance * 4;
// Adjust based on semantic shift magnitude (distance, scaled 0-1)
// Larger shifts slightly increase complexity, centered around a baseline distance
const baselineDistance = 0.3; // Assumes threshold is around here
complexity += (semanticShiftMagnitude - baselineDistance) * 1.0; // Adjust sensitivity as needed
// Ensure complexity is within the [1, 5] range
return Math.max(1, Math.min(5, complexity));
}
/**
* Detect turning points within a single chunk of the conversation
*/
/**
* Detect turning points within a single chunk of the conversation
* This represents the local refinement process in the current dimension
*/
async detectTurningPointsInChunk(messages, dimension, chunkIndex, // Optional index for logging purposes
originalMessages) {
if (messages.length < 2)
return [];
// Generate embeddings for all messages in the chunk
const embeddings = await this.generateMessageEmbeddings(messages, dimension);
// Find significant semantic shifts between adjacent messages
const turningPoints = [];
const distances = []; // Store distances for logging
const allDistances = []; // Store all distances for logging
for (let i = 0; i < embeddings.length - 1; i++) {
const current = embeddings[i];
const next = embeddings[i + 1];
// Calculate semantic distance between current and next message
const distance = this.calculateSemanticDistance(current.embedding, next.embedding);
let thresholdScaleFactor;
const baseThreshold = this.config.semanticShiftThreshold;
if (baseThreshold > 0.7) {
// For high initial thresholds (like 0.75), scale down more aggressively
thresholdScaleFactor = Math.pow(0.25, dimension); // More aggressive (0.25 instead of 0.4)
}
else if (baseThreshold > 0.5) {
// For medium thresholds
thresholdScaleFactor = Math.pow(0.35, dimension);
}
else {
// For already low thresholds
thresholdScaleFactor = Math.pow(0.5, dimension);
}
const dimensionAdjustedThreshold = baseThreshold * thresholdScaleFactor;
this.logger.debug(`Anlyzing with dimensionAdjustedThreshold: ${dimensionAdjustedThreshold.toFixed(3)}, compared to original threshold: ${baseThreshold.toFixed(3)}`);
if (dimensionAdjustedThreshold <= distance) {
distances.push({
current: current.index,
next: next.index,
distance: distance,
}); // Store distance for logging
}
allDistances.push({
current: current.index,
next: next.index,
distance: distance,
});
}
this.logger.info(`For a total number of points: ${embeddings.length}, there were ${distances.length} distances found as being greater than the threshold of ${this.config.semanticShiftThreshold}. Across this span of messages of length ${messages.length}, the following distances were found:
- The top 3 greatest distances are: ${allDistances
.slice(0, 3)
.sort((a, b) => b.distance - a.distance)
.map((d) => d.distance.toFixed(3))
.join(", ")}
This means there were ${distances.length} potential turning points detected ${dimension === 0 ? "with valid user-assistant turn pairs" : "with valid meta-messages"}`);
if (distances.length === 0) {
this.logger.info(`No significant semantic shifts detected in chunk ${chunkIndex}`);
return [];
}
await async_1.default.eachOfLimit(distances, this.config.concurrency, async (distanceObj, idxStr) => {
const d = Number(idxStr);
const i = distanceObj.current; // Current message index
const current = embeddings[i]; // Current message embedding
const next = embeddings[distanceObj.next]; // Next message embedding
// If the distance exceeds our threshold, we've found a turning point
// Use direct array indices to get the messages
const distance = distanceObj.distance; // Semantic distance between current and next message
const beforeMessage = messages[i];
const afterMessage = messages[i + 1];
if (beforeMessage == undefined || afterMessage == undefined) {
this.logger.info(`detectTurningPointsInChunk: warning beforeMessage or afterMessage is undefined, beforeMessage: ${beforeMessage}, afterMessage: ${afterMessage}`);
return;
}
// Classify the turning point using LLM
const turningPoint = await this.classifyTurningPoint(beforeMessage, afterMessage, distance, dimension, originalMessages, d);
this.logger.info(` ...${chunkIndex ? `[Chunk ${chunkIndex}] ` : ""}Potential turning point detected between messages ${current.id} and ${next.id} (distance: ${distance.toFixed(3)}, complexity: ${turningPoint.complexityScore.toFixed(1)}), signif: ${turningPoint.significance.toFixed(2)} category: ${turningPoint.category}`);
if (turningPoint.significance > 1) {
if (turningPoint.significance > 10) {
turningPoint.significance = turningPoint.significance / 100;
}
else {
turningPoint.significance = turningPoint.significance / 10; // Adjusting for scale
}
}
turningPoints.push(turningPoint);
});
return turningPoints;
}
async clearOllamaCache(modelName) {
// 4. This is the bogey request. It tells the server to unload the model.
// await fetch('http://your-ollama-host:11434/api/generate', {
// method: 'POST',
// headers: { 'Content-Type': 'application/json' },
// body: JSON.stringify({
// model: modelName,
// keep_alive: 0,
// }),
// });
try {
return await this.ollama.generate({
model: modelName,
keep_alive: 0, // Unload the model after use,
prompt: "\\no_think",
options: {
num_predict: 0, // No predictions needed, just unload
},
});
}
catch (error) {
this.logger.warn(`Error clearing Ollama cache for model ${modelName}: ${error?.message || error}`);
}
}
/**
* Use LLM to classify a turning point and generate metadata.
* *** MODIFIED to prioritize message.spanData over regex ***
*/
/**
* Use LLM to classify a turning point and generate metadata.
* This implementation uses a highly modular prompt architecture with
* multiple distinct user messages to ensure clarity. The payload consists of:
* - A system message that sets the core identity and universal constraints.
* - A static context user message containing framework and evaluation criteria.
* - A dynamic data user message that provides conversation context and the specific messages to analyze.
* - A final user instruction message that tells the model what to do with all this information.
*/
async classifyTurningPoint(beforeMessage, afterMessage, distance, dimension, originalMessages, index = 0) {
let span;
if (dimension > 0) {
if (!(beforeMessage instanceof Message_1.MetaMessage) ||
!(afterMessage instanceof Message_1.MetaMessage)) {
throw new Error("Before or after message is not a MetaMessage at higher dimension");
}
const beforeMessageMeta = beforeMessage;
const afterMessageMeta = afterMessage;
// For higher dimensions, extract the starting and ending message from within the meta-message's inner list
span = {
startId: beforeMessageMeta.getMessagesInTurningPointSpanToMessagesArray()[0]
.id,
endId: afterMessageMeta.getMessagesInTurningPointSpanToMessagesArray()[0].id,
startIndex: this.originalMessages.findIndex((candidateM) => candidateM.id ===
beforeMessageMeta.getMessagesInTurningPointSpanToMessagesArray()[0]
.id),
endIndex: this.originalMessages.findIndex((candidateM) => candidateM.id ===
afterMessageMeta.getMessagesInTurningPointSpanToMessagesArray()[0]
.id),
originalSpan: {
startId: beforeMessage.id,
endId: afterMessage.id,
startIndex: index,
endIndex: index + 1,
},
};
}
else {
// For base-level conversations, use the original message IDs and find their indices.
span = {
startId: beforeMessage.id,
endId: afterMessage.id,
startIndex: Message_1.MetaMessage.findIndexOfMessageFromId({
id: beforeMessage.id,
beforeMessage,
afterMessage,
messages: originalMessages,
}),
endIndex: Message_1.MetaMessage.findIndexOfMessageFromId({
id: afterMessage.id,
beforeMessage,
afterMessage,
messages: originalMessages,
}),
};
}
// --- Constructing the Modular Prompt ---
// 1. System Message: Core identity and immutable instructions.
const systemMessage = this.config.customSystemInstruction &&
this.config.customSystemInstruction.length > 0
? this.config.customSystemInstruction
: `You are an expert conversation analyzer specializing in semantic turning point detection.
Your primary goal is to identify significant shifts in conversation flow and meaning.
Analyze semantic differences in the provided conversation context and provide a structured JSON output as described.`;
// 2. Static Context User Message: Framework and evaluation criteria.
const frameworkContextMessage = `<analysis_framework>
Turning points are significant shifts in conversation that indicate changes in subject, emotion, or decision-making.
A semantic distance of ${distance.toFixed(3)} has been detected between the messages.
You are analyzing dimension ${dimension} where ${dimension > 0 ? "each message represents a group of related turning points" : "messages are direct conversation exchanges"}.
Classification categories include:
- Topic, Insight, Emotion, Meta-Reflection, Decision,
- Question, Problem, Action, Clarification, Objection, Other.
Significance (0.0 to 1.0) reflects the impact of the turning point on the overall conversation.
</analysis_framework>
<output_format>
Your JSON response must include:
- label: (string, max 50 chars) a brief description,
- category: (string) one of the categories mentioned,
- keywords: (array of strings, max 4),
- quotes: (array of strings, max 3),
- emotionalTone: (string),
- sentiment: (one of "positive", "negative", "neutral"),
- significance: (number, 0.0 to 1.0),
- best_id: (string) the representative message ID.
</output_format>`;
// 3. Dynamic Data User Message: Conversation context and messages to analyze.
const contextualInfo = this.prepareContextualInfoMeta(beforeMessage, afterMessage, span, originalMessages, dimension, 2, dimension > 0);
const dynamicDataMessage = `<conversation_context>
${contextualInfo}
</conversation_context>
<messages_to_analyze>
BEFORE MESSAGE:
- Role: ${beforeMessage.author}
- Content: ${(0, stripContent_1.returnFormattedMessageContent)(this.config, beforeMessage, dimension)}
AFTER MESSAGE:
- Role: ${afterMessage.author}
- Content: ${(0, stripContent_1.returnFormattedMessageContent)(this.config, afterMessage, dimension)}
</messages_to_analyze>`;
// 4. Final Task Instruction User Message: Direct instruction to the LLM.
const finalInstructionMessage = this.config.customUserInstruction &&
this.config.customUserInstruction.length > 0
? this.config.customUserInstruction
: `Using the criteria provided in <analysis_framework> and the detailed context in <conversation_context> along with the specific messages in <messages_to_analyze>,
analyze whether the provided messages represent a turning point in the conversation.
Determine the category, significance, and other attributes as specified in <output_format>.
Return your answer as valid JSON.`;
// Assemble all messages as a multi-message payload
const messagesPayload = [
{ role: "system", content: systemMessage },
{ role: "user", content: frameworkContextMessage },
{ role: "user", content: dynamicDataMessage },
{ role: "user", content: finalInstructionMessage },
];
let classification = {};
try {
// Call the LLM using the assembled messages
let classificationResponseStringContent = null;
if (this.endpointType !== "ollama") {
const response = await this.openai.chat.completions.create({
model: this.config.classificationModel,
messages: messagesPayload,
temperature: this.config.temperature,
response_format: (0, prompt_1.formResponseFormatSchema)(dimension, this.config),
top_p: this.config.top_p,
});
classificationResponseStringContent =
response.choices[0]?.message?.content || "{}";
}
else {
const response = await this.ollama.chat({
model: this.config.classificationModel,
messages: messagesPayload.map((msg) => ({
role: msg.role,
content: String(msg.content),
})),
stream: false,
format: (0, prompt_1.formResponseFormatSchema)(dimension, this.config).json_schema
.schema,
options: {
temperature: this.config.temperature,
top_p: this.config.top_p,
top_k: 20,
num_ctx: this.config.maxTokensPerChunk,
},
});
// now try to json parse, if failure do the ame fallback
classificationResponseStringContent = response?.message?.content ?? "";
}
if (classificationResponseStringContent) {
classification = this.parseClassificationResponse(classificationResponseStringContent, span);
}
else {
// Fallback if no response content
classification = {
label: "No Response - Unclassified",
category: "Other",
keywords: [],
emotionalTone: "neutral",
sentiment: "neutral",
significance: 0.0, // Lower significance for no response
quotes: [],
best_id: span.startId,
};
}
// Validate and sanitize the LLM output.
const validatedClassification = {
label: typeof classification.label === "string"
? classification.label.substring(0, 50)
: "Unknown Turning Point",
category: typeof classification.category === "string"
? classification.category
: "Other",
keywords: Array.isArray(classification.keywords)
? classification.keywords.map(String).slice(0, 4)
: [],
emotionalTone: typeof classification.emotionalTone === "string"
? classification.emotionalTone
: "neutral",
sentiment: ["positive", "negative", "neutral"].includes(classification.sentiment)
? classification.sentiment
: "neutral",
significance: typeof classification.significance === "number"
? Math.max(0, Math.min(1, classification.significance))
: 0.5,
quotes: Array.isArray(classification.quotes)
? classification.quotes.map(String).slice(0, 3)
: [],
best_id: typeof classification.best_id === "string"
? classification.best_id
: span.startId,
};
// Calculate complexity score using the significance and the raw distance.
const complexityScore = this.calculateComplexityScore(validatedClassification.significance, distance);
// Construct and return the final TurningPoint object.
return {
id: `tp-${dimension}-${span.startIndex}-${span.endIndex}`,
label: validatedClassification.label,
category: validatedClassification.category,
span: span,
semanticShiftMagnitude: distance,
keywords: validatedClassification.keywords,
quotes: validatedClassification.quotes,
emotionalTone: validatedClassification.emotionalTone,
sentiment: validatedClassification.sentiment,
detectionLevel: dimension,
significance: validatedClassification.significance,
complexityScore: complexityScore,
};
}
catch (err) {
this.logger.info(`Error during LLM call for turning point classification: ${err.message}`);
if (this.config.throwOnError) {
throw err;
}
else {
return {
id: `tp-err-${dimension}-${span.startId}`,
label: "LLM Error - Unclassified",
category: "Other",
span: span,
semanticShiftMagnitude: distance,
keywords: [],
quotes: [],
emotionalTone: "neutral",
sentiment: "neutral",
detectionLevel: dimension,
significance: 0.1,
complexityScore: 1.0,
};
}
}
}
/**
* Updated to utilize new classes of Message and MetaMessage for better structure and clarity
* @param turningPoints
* @param originalMessages
* @returns
*/
createMetaMessagesFromTurningPoints(turningPoints, originalMessages) {
if (turningPoints.length === 0)
return [];
// Group turning points by category (first-level abstraction)
const groupedByCategory = {};
turningPoints.forEach((tp) => {
const category = tp.category;
if (!groupedByCategory[category]) {
groupedByCategory[category] = [];
}
groupedByCategory[category].push(tp);
});
this.logger.info(`Grouped categories:\n` + JSON.stringify(groupedByCategory, null, 2));
// Create meta-messages (one per category to find higher-level patterns)
const metaMessages = [];
// First create category messages - represents dimension n to n+1 transformation
Object.entries(groupedByCategory).forEach(([category, points], index) => {
// Use the factory method from MetaMessage class to create a properly typed meta-message
const metaMessage = Message_1.MetaMessage.createCategoryMetaMessage(category, points, index, originalMessages);
metaMessages.push(metaMessage);
});
// Create timeline/section meta-messages
const sortedPoints = [...turningPoints].sort((a, b) => a.span.startIndex - b.span.startIndex);
const sectionCount = Math.min(4, Math.ceil(sortedPoints.length / 2));
const pointsPerSection = Math.ceil(sortedPoints.length / sectionCount);
// Create chronological section meta-messages
for (let i = 0; i < sectionCount; i++) {
const sectionPoints = sortedPoints.slice(i * pointsPerSection, Math.min((i + 1) * pointsPerSection, sortedPoints.length));
if (sectionPoints.length === 0)
continue;
// Create a section meta-message using the factory method
const sectionMetaMessage = Message_1.MetaMessage.createSectionMetaMessage(sectionPoints, i, this.originalMessages);
console.info("created sectionMetageMessage");
metaMessages.push(sectionMetaMessage);
}
this.logger.info(`Created ${metaMessages.length} meta-messages for dimensional expansion: ${metaMessages
.map((m) => m.id)
.join(", ")}`);
return metaMessages;
}
// --- Remaining methods are kept identical to your second provided version ---
/**
* Filter turning points to keep only significant ones
* (Using original logic from the second code block)
*/
filterSignificantTurningPoints(turningPoints) {
if (!this.config.onlySignificantTurningPoints ||
turningPoints.length === 0) {
// Ensure sorted return even if not filtering
return turningPoints.sort((a, b) => a.span.startIndex - b.span.startIndex);
}
this.logger.info(`Filtering ${turningPoints.length} TPs based on significance >= ${this.config.significanceThreshold} and maxPoints = ${this.config.maxTurningPoints}`);
// Sort by significance, complexity, magnitude
const sorted = [...turningPoints].sort((a, b) => {
if (b.significance !== a.significance)
return b.significance - a.significance;
if (b.complexityScore !== a.complexityScore)
return b.complexityScore - a.complexityScore;
return b.semanticShiftMagnitude - a.semanticShiftMagnitude;
});
const result = [];
const coveredIndices = new Set(); // Use indices for overlap check
const maxPoints = this.config.maxTurningPoints;
for (const tp of sorted) {
// Check significance threshold first
if (tp.significance < this.config.significanceThreshold) {
// Only consider points below threshold if we haven't found enough significant ones yet
if (result.length >= Math.ceil(maxPoints / 2)) {
// Heuristic: if we have half the max points, stop adding insignificant ones
continue;
}
}
// Check for significant overlap with already selected points
let overlapRatio = 0;
let isOverlapping = false;
const tpSpanSize = tp.span.endIndex - tp.span.startIndex + 1;
if (tpSpanSize > 0) {
let overlapCount = 0;
for (let i = tp.span.startIndex; i <= tp.span.endIndex; i++) {
if (coveredIndices.has(i)) {
overlapCount++;
}
}
overlapRatio = overlapCount / tpSpanSize;
}
// Define significant overlap threshold (e.g., 40% from original code)
const overlapThreshold = 0.4;
isOverlapping = overlapRatio > overlapThreshold;
if (!isOverlapping && result.length < maxPoints) {
result.push(tp);
// Mark indices covered by this TP
for (let i = tp.span.startIndex; i <= tp.span.endIndex; i++) {
coveredIndices.add(i);
}
}
else if (isOverlapping) {
this.logger.info(` TP ${tp.id} (Sig: ${tp.significance.toFixed(2)}) overlaps significantly (${(overlapRatio * 100).toFixed(0)}%) with existing TPs. Skipping.`);
}
else if (result.length >= maxPoints) {
this.logger.info(` Reached max turning points (${maxPoints}). Skipping TP ${tp.id}.`);
}
}
// Ensure at least one TP is returned if any were found initially
if (result.length === 0 && sorted.length > 0) {
this.logger.info("No TPs met significance/overlap criteria, returning the single most sig