create-ai-chat-context-experimental
Version:
Phase 2: TypeScript rewrite - AI Chat Context & Memory System with conversation extraction and AICF format support (powered by aicf-core v2.1.0).
621 lines • 26.1 kB
JavaScript
;
/**
* This file is part of create-ai-chat-context-experimental.
* Licensed under the GNU Affero General Public License v3.0 or later (AGPL-or-later).
* See LICENSE file for details.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.SessionConsolidationAgent = void 0;
/**
* Session Consolidation Agent
* Phase 6.5: Convert individual conversation files into session-based template format
* Phase 8: Enhanced with aicf-core integration - October 2025
*
* Takes 10,260 individual .aicf files and consolidates them into ~20 session files
* using the clean, readable template format from templates/aicf/conversations.aicf
*
* Benefits:
* - 98% storage reduction (92MB → 2MB)
* - 97% token reduction (18.5M → 0.5M tokens)
* - AI-readable format (scannable, no JSON blobs)
* - Automatic deduplication
* - Enterprise-grade writes (thread-safe, validated, PII redaction)
*/
const fs_1 = require("fs");
const path_1 = require("path");
const result_js_1 = require("../types/result.js");
/**
* Consolidates individual conversation files into session-based template format
* Now uses aicf-core for enterprise-grade file operations
*/
class SessionConsolidationAgent {
inputDir;
outputDir;
constructor(cwd = process.cwd()) {
this.inputDir = (0, path_1.join)(cwd, '.aicf', 'recent');
this.outputDir = (0, path_1.join)(cwd, '.aicf', 'sessions');
}
/**
* Main consolidation method
* Reads all conversation files, groups into sessions, deduplicates, and writes template format
*/
async consolidate() {
try {
// Ensure output directory exists
if (!(0, fs_1.existsSync)(this.outputDir)) {
(0, fs_1.mkdirSync)(this.outputDir, { recursive: true });
}
// Read all conversation files
const files = this.findConversationFiles();
if (files.length === 0) {
return (0, result_js_1.Ok)({
totalFiles: 0,
totalConversations: 0,
sessionsCreated: 0,
uniqueConversations: 0,
duplicatesRemoved: 0,
storageReduction: '0%',
tokenReduction: '0%',
});
}
// Parse conversations from files
const conversations = this.parseConversations(files);
// Group into sessions by date
const sessions = this.groupIntoSessions(conversations);
// Deduplicate each session
const deduplicatedSessions = sessions.map((session) => this.deduplicateSession(session));
// Write session files
let sessionsWritten = 0;
for (const session of deduplicatedSessions) {
const writeResult = await this.writeSessionFile(session);
if (writeResult.ok) {
sessionsWritten++;
}
}
// Calculate stats
const totalConversations = conversations.length;
const uniqueConversations = deduplicatedSessions.reduce((sum, s) => sum + s.conversations.length, 0);
const duplicatesRemoved = totalConversations - uniqueConversations;
return (0, result_js_1.Ok)({
totalFiles: files.length,
totalConversations,
sessionsCreated: sessionsWritten,
uniqueConversations,
duplicatesRemoved,
storageReduction: this.calculateStorageReduction(files.length, sessionsWritten),
tokenReduction: this.calculateTokenReduction(totalConversations, uniqueConversations),
});
}
catch (error) {
return (0, result_js_1.Err)(error instanceof Error ? error : new Error(`Session consolidation failed: ${String(error)}`));
}
}
/**
* Find all conversation files in input directory
*/
findConversationFiles() {
if (!(0, fs_1.existsSync)(this.inputDir)) {
return [];
}
const files = (0, fs_1.readdirSync)(this.inputDir);
return files
.filter((file) => file.endsWith('.aicf'))
.filter((file) => file.match(/^\d{4}-\d{2}-\d{2}_/)) // Match date prefix
.map((file) => (0, path_1.join)(this.inputDir, file));
}
/**
* Parse conversations from files
* Option B: Each file may produce multiple ConversationEssentials (one per day)
*/
parseConversations(files) {
const conversations = [];
for (const file of files) {
try {
const content = (0, fs_1.readFileSync)(file, 'utf-8');
const essentialsArray = this.extractEssentials(content, file);
if (essentialsArray && essentialsArray.length > 0) {
conversations.push(...essentialsArray);
}
}
catch (error) {
// Skip files that can't be parsed
console.warn(`Failed to parse ${file}:`, error);
}
}
return conversations;
}
/**
* Unescape AICF format special characters
* Replaces \\n with newlines and \\| with pipes
*/
unescapeAICF(text) {
return text.replace(/\\n/g, '\n').replace(/\\\|/g, '|');
}
/**
* Extract essential information from conversation file
* NEW FORMAT: Reads extracted analysis results directly from AICF file
* Option B: Returns array of ConversationEssentials (one per day for multi-day conversations)
*/
extractEssentials(content, _filePath) {
try {
const lines = content.split('\n');
// Parse AICF format - new extracted format
let conversationId = '';
let timestamp = '';
const userIntents = [];
const aiActions = [];
const decisions = [];
// Extract actual message timestamps from @CONVERSATION section
// Group by date for multi-day conversations
const messagesByDate = new Map();
let inConversationSection = false;
for (const line of lines) {
// Track @CONVERSATION section
if (line.startsWith('@CONVERSATION')) {
inConversationSection = true;
continue;
}
else if (line.startsWith('@')) {
inConversationSection = false;
}
// Extract timestamps and group messages by date
if (inConversationSection && line.includes('|')) {
const parts = line.split('|');
// Format: timestamp|role|content
if (parts.length >= 3 && parts[0]) {
const ts = parts[0].trim();
// Validate ISO timestamp format
if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(ts)) {
const date = ts.split('T')[0]; // Extract YYYY-MM-DD
if (date) {
if (!messagesByDate.has(date)) {
messagesByDate.set(date, []);
}
messagesByDate.get(date)?.push(ts);
}
}
}
}
if (line.startsWith('conversationId|')) {
conversationId = line.split('|')[1] || '';
}
else if (line.startsWith('timestamp|')) {
timestamp = line.split('|')[1] || '';
}
else if (line.startsWith('userIntents|')) {
// Format: userIntents|timestamp|intent|confidence;timestamp|intent|confidence;...
const intentData = line.substring('userIntents|'.length);
if (intentData) {
// Split by semicolon to get individual intents
const intentEntries = intentData.split(';');
for (const entry of intentEntries) {
const parts = entry.split('|');
if (parts.length >= 2 && parts[1]) {
const intent = this.unescapeAICF(parts[1]); // timestamp|intent|confidence
if (intent && intent.length > 5) {
userIntents.push(intent);
}
}
}
}
}
else if (line.startsWith('aiActions|')) {
// Format: aiActions|timestamp|type|details;timestamp|type|details;...
const actionData = line.substring('aiActions|'.length);
if (actionData) {
const actionEntries = actionData.split(';');
for (const entry of actionEntries) {
const parts = entry.split('|');
if (parts.length >= 3 && parts[2]) {
const details = this.unescapeAICF(parts[2]); // timestamp|type|details
if (details && details.length > 5) {
aiActions.push(details);
}
}
}
}
}
else if (line.startsWith('decisions|')) {
// Format: decisions|timestamp|decision|impact;timestamp|decision|impact;...
const decisionData = line.substring('decisions|'.length);
if (decisionData) {
const decisionEntries = decisionData.split(';');
for (const entry of decisionEntries) {
const parts = entry.split('|');
if (parts.length >= 2 && parts[1]) {
const decision = this.unescapeAICF(parts[1]); // timestamp|decision|impact
if (decision && decision.length > 5) {
decisions.push(decision);
}
}
}
}
}
}
if (!conversationId)
return [];
// If no extracted data, skip this conversation
if (userIntents.length === 0 && aiActions.length === 0) {
return [];
}
// Combine all content for title and summary
const allContent = [...userIntents, ...aiActions].join('\n\n');
// Extract title from first user intent or AI action
const title = this.extractTitle(allContent);
// Extract summary from all content
const summary = this.extractSummary(allContent);
// AI model is 'augment' by default (can be enhanced later)
const aiModel = 'augment';
// Status is always COMPLETED for now
const status = 'COMPLETED';
// Generate content hash for deduplication
const contentHash = this.hashContent(allContent);
// Option B: Create one ConversationEssentials per date
// This allows multi-day conversations to appear in multiple session files
const results = [];
if (messagesByDate.size > 0) {
// Multi-day conversation: create one entry per date
for (const [date, timestamps] of messagesByDate.entries()) {
const earliestTimestamp = timestamps.sort()[0] || `${date}T00:00:00Z`;
results.push({
id: conversationId,
timestamp: earliestTimestamp,
title: `${title} (${date})`, // Add date suffix for multi-day conversations
summary,
aiModel,
decisions,
actions: aiActions,
status,
contentHash: `${contentHash}-${date}`, // Unique hash per date
});
}
}
else {
// No message timestamps found, fall back to conversation timestamp
results.push({
id: conversationId,
timestamp: timestamp || new Date().toISOString(),
title,
summary,
aiModel,
decisions,
actions: aiActions,
status,
contentHash,
});
}
return results;
}
catch {
return [];
}
}
/**
* Extract title from response text (first meaningful sentence)
* Improved: Skip common filler phrases, prioritize action statements
*/
extractTitle(text) {
if (!text)
return 'Untitled conversation';
// Remove markdown formatting and code blocks
let cleaned = text.replace(/```[\s\S]*?```/g, ''); // Remove code blocks
cleaned = cleaned.replace(/[*_`#]/g, '').trim();
// Skip common filler phrases
const fillerPhrases = [
/^(ok|okay|yes|no|right|sure|alright|got it|i see|understood)[.,!?\s]/i,
/^(let me|now|first|next|then)[.,\s]/i,
/^(the issue is|the problem is|i need to|i'll|i will)[.,\s]/i,
];
// Get sentences
const sentences = cleaned.split(/[.!?\n]+/).filter((s) => s.trim().length > 15);
// Find first meaningful sentence (skip filler)
for (const sentence of sentences) {
const trimmed = sentence.trim();
// Skip if it's a filler phrase
if (fillerPhrases.some((pattern) => pattern.test(trimmed))) {
continue;
}
// Skip if it's too short or just a greeting
if (trimmed.length < 15) {
continue;
}
// This is a good title
return trimmed.length > 80 ? trimmed.substring(0, 77) + '...' : trimmed;
}
// Fallback: use first sentence even if it's filler
if (sentences.length > 0) {
const firstSentence = sentences[0];
if (firstSentence) {
const title = firstSentence.trim();
return title.length > 80 ? title.substring(0, 77) + '...' : title;
}
}
return 'Untitled conversation';
}
/**
* Extract summary from response text
* Improved: Focus on outcomes, decisions, and concrete work
*/
extractSummary(text) {
if (!text)
return 'No summary available';
// Remove markdown, code blocks, and excessive whitespace
let cleaned = text.replace(/```[\s\S]*?```/g, ''); // Remove code blocks
cleaned = cleaned.replace(/[*_`#]/g, '').trim();
cleaned = cleaned.replace(/\s+/g, ' '); // Normalize whitespace
// Look for summary indicators
const summaryPatterns = [
// Explicit summaries
/(?:summary|tldr|in short|to summarize)[\s:]+([^\n]{20,200})/i,
// Results/outcomes
/(?:result|outcome|achieved|completed|finished)[\s:]+([^\n]{20,200})/i,
// What was done
/(?:implemented|created|built|fixed|updated)[\s]+([^\n]{20,200})/i,
];
// Try to find explicit summary
for (const pattern of summaryPatterns) {
const match = cleaned.match(pattern);
if (match && match[1]) {
const summary = match[1].trim();
return summary.length > 200 ? summary.substring(0, 197) + '...' : summary;
}
}
// Fallback: Get first 2-3 meaningful sentences
const sentences = cleaned.split(/[.!?\n]+/).filter((s) => s.trim().length > 15);
// Skip filler sentences
const meaningfulSentences = sentences.filter((s) => {
const lower = s.toLowerCase();
return !(lower.startsWith('ok') ||
lower.startsWith('yes') ||
lower.startsWith('no') ||
lower.startsWith('right') ||
lower.startsWith('let me'));
});
const summary = meaningfulSentences.slice(0, 2).join('. ');
if (summary.length > 0) {
return summary.length > 200 ? summary.substring(0, 197) + '...' : summary;
}
return 'No summary available';
}
/**
* Generate content hash for deduplication
*/
hashContent(content) {
let hash = 0;
for (let i = 0; i < content.length; i++) {
const char = content.charCodeAt(i);
hash = (hash << 5) - hash + char;
hash = hash & hash;
}
return Math.abs(hash).toString(36);
}
/**
* Group conversations into sessions by date
*/
groupIntoSessions(conversations) {
const sessionMap = new Map();
for (const conv of conversations) {
const dateParts = conv.timestamp.split('T');
const date = dateParts[0]; // Extract YYYY-MM-DD
if (!date)
continue;
if (!sessionMap.has(date)) {
sessionMap.set(date, []);
}
const dateConvs = sessionMap.get(date);
if (dateConvs) {
dateConvs.push(conv);
}
}
const sessions = [];
for (const [date, convs] of sessionMap.entries()) {
// Sort by timestamp
convs.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
sessions.push({
date,
conversations: convs,
metadata: {
totalConversations: convs.length,
uniqueConversations: convs.length, // Will be updated after deduplication
duration: this.calculateDuration(convs),
focus: this.determineFocus(convs),
},
});
}
return sessions.sort((a, b) => a.date.localeCompare(b.date));
}
/**
* Calculate session duration
*/
calculateDuration(conversations) {
if (conversations.length === 0)
return '0 hours';
const firstConv = conversations[0];
const lastConv = conversations[conversations.length - 1];
if (!firstConv || !lastConv)
return '0 hours';
const first = new Date(firstConv.timestamp);
const last = new Date(lastConv.timestamp);
const hours = (last.getTime() - first.getTime()) / (1000 * 60 * 60);
return hours < 1 ? `${Math.round(hours * 60)} minutes` : `${hours.toFixed(1)} hours`;
}
/**
* Determine session focus from conversation titles
*/
determineFocus(conversations) {
// Extract common keywords from titles
const keywords = conversations
.map((c) => c.title.toLowerCase())
.join(' ')
.split(/\s+/)
.filter((w) => w.length > 4);
// Count frequency
const freq = new Map();
for (const word of keywords) {
freq.set(word, (freq.get(word) || 0) + 1);
}
// Get most common word
let maxWord = 'Development';
let maxCount = 0;
for (const [word, count] of freq.entries()) {
if (count > maxCount) {
maxWord = word;
maxCount = count;
}
}
return maxWord.charAt(0).toUpperCase() + maxWord.slice(1);
}
/**
* Deduplicate conversations within a session
*/
deduplicateSession(session) {
const seen = new Set();
const unique = [];
for (const conv of session.conversations) {
if (!seen.has(conv.contentHash)) {
seen.add(conv.contentHash);
unique.push(conv);
}
}
return {
...session,
conversations: unique,
metadata: {
...session.metadata,
uniqueConversations: unique.length,
},
};
}
/**
* Write session to template format file
* Uses direct fs.writeFileSync since session files contain multi-line structured data
* that should not be escaped (unlike single-line AICF entries)
*/
async writeSessionFile(session) {
try {
// Ensure output directory exists
if (!(0, fs_1.existsSync)(this.outputDir)) {
(0, fs_1.mkdirSync)(this.outputDir, { recursive: true });
}
const filePath = (0, path_1.join)(this.outputDir, `${session.date}-session.aicf`);
const content = this.generateTemplateFormat(session);
// Write directly - session files are multi-line structured data
(0, fs_1.writeFileSync)(filePath, content, 'utf-8');
return (0, result_js_1.Ok)(undefined);
}
catch (error) {
return (0, result_js_1.Err)(error instanceof Error ? error : new Error(`Failed to write session file: ${String(error)}`));
}
}
/**
* Generate AICF format content for LLM memory
* Format: @CONVERSATION sections with structured fields
* This is the format that GPT, Claude, Augment, and Warp all agreed on
*/
generateTemplateFormat(session) {
const lines = [];
// Session header
lines.push(`# Session: ${session.date}`);
lines.push(`# Total conversations: ${session.metadata.totalConversations}`);
lines.push(`# Unique conversations: ${session.metadata.uniqueConversations}`);
lines.push(`# Focus: ${session.metadata.focus}`);
lines.push('');
// Each conversation gets its own @CONVERSATION block
for (const conv of session.conversations) {
if (!conv)
continue;
// @CONVERSATION header
lines.push(`@CONVERSATION:${conv.id}`);
lines.push(`timestamp_start=${conv.timestamp}`);
lines.push(`timestamp_end=${conv.timestamp}`); // We don't track end time yet
lines.push(`messages=${conv.actions.length + conv.decisions.length}`);
lines.push(`tokens=0`); // We don't track tokens yet
lines.push(`title=${this.escapeField(conv.title)}`);
lines.push(`summary=${this.escapeField(conv.summary)}`);
lines.push(`ai_model=${conv.aiModel}`);
lines.push(`status=${conv.status}`);
lines.push('');
// @STATE section
lines.push('@STATE');
lines.push('working_on=development');
lines.push('blockers=none');
lines.push('next_action=continue');
lines.push('');
// @FLOW section
lines.push('@FLOW');
if (conv.actions.length > 0) {
// Create flow from actions
const flowSteps = conv.actions.slice(0, 5).map((action) => {
return action
.substring(0, 30)
.replace(/[^a-z0-9_]/gi, '_')
.toLowerCase();
});
lines.push(flowSteps.join('|'));
}
else {
lines.push('user_query|ai_response|session_complete');
}
lines.push('');
// @INSIGHTS section
lines.push('@INSIGHTS');
// FIX #3: Deduplicate insights before writing
const uniqueInsights = this.deduplicateArray(conv.decisions.slice(0, 3));
if (uniqueInsights.length > 0) {
for (const insight of uniqueInsights) {
lines.push(`${this.escapeField(insight)}|GENERAL|MEDIUM|MEDIUM`);
}
}
else {
lines.push('No significant insights extracted');
}
lines.push('');
// @DECISIONS section
lines.push('@DECISIONS');
// FIX #3: Deduplicate decisions before writing
const uniqueDecisions = this.deduplicateArray(conv.decisions);
if (uniqueDecisions.length > 0) {
for (const decision of uniqueDecisions) {
lines.push(`${this.escapeField(decision)}|extracted_from_conversation|IMPACT:MEDIUM|CONF:MEDIUM`);
}
}
else {
lines.push('No explicit decisions extracted');
}
lines.push('');
}
return lines.join('\n') + '\n';
}
/**
* Deduplicate array of strings
* Used to remove duplicate decisions and insights before writing to session files
*/
deduplicateArray(items) {
return Array.from(new Set(items));
}
/**
* Escape field for pipe-delimited format
*/
escapeField(field) {
return field.replace(/\|/g, '\\|').replace(/\n/g, ' ');
}
/**
* Calculate storage reduction percentage
*/
calculateStorageReduction(inputFiles, outputFiles) {
const reduction = ((inputFiles - outputFiles) / inputFiles) * 100;
return `${reduction.toFixed(1)}%`;
}
/**
* Calculate token reduction percentage
*/
calculateTokenReduction(totalConvs, uniqueConvs) {
// Assume 1800 tokens per old format, 50 tokens per new format
const oldTokens = totalConvs * 1800;
const newTokens = uniqueConvs * 50;
const reduction = ((oldTokens - newTokens) / oldTokens) * 100;
return `${reduction.toFixed(1)}%`;
}
}
exports.SessionConsolidationAgent = SessionConsolidationAgent;
//# sourceMappingURL=SessionConsolidationAgent.js.map