aicf-core
Version:
Universal AI Context Format (AICF) - Enterprise-grade AI memory infrastructure with 95.5% compression and zero semantic loss
386 lines • 14.4 kB
JavaScript
/**
* SPDX-License-Identifier: AGPL-3.0-or-later
* Copyright (c) 2025 Dennis van Leeuwen
*
* AICF Stream Reader - Memory-efficient streaming access to AICF files
*
* Replaces fs.readFileSync() with streaming to handle large files (1GB+)
* with constant memory usage regardless of file size.
*/
import { createReadStream, existsSync, statSync } from "node:fs";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { createInterface } from "node:readline";
/**
* AICF Stream Reader
*/
export class AICFStreamReader {
aicfDir;
indexCache = null;
lastIndexRead = 0;
MAX_LINE_LENGTH = 1024 * 1024; // 1MB per line max
constructor(aicfDir = ".aicf") {
this.aicfDir = aicfDir;
}
/**
* Stream read a file line by line with callback
* Memory-efficient: O(1) memory usage regardless of file size
*/
async streamFile(filePath, lineCallback, options = {}) {
const { onProgress = null, onError = null, maxLines = Infinity } = options;
return new Promise((resolve, reject) => {
if (!existsSync(filePath)) {
const error = new Error(`File not found: ${filePath}`);
if (onError)
onError(error);
return reject(error);
}
const fileStream = createReadStream(filePath, {
encoding: "utf8",
highWaterMark: 64 * 1024, // 64KB chunks
});
const rl = createInterface({
input: fileStream,
crlfDelay: Infinity,
});
let lineCount = 0;
let processedCount = 0;
const stats = statSync(filePath);
let bytesRead = 0;
rl.on("line", (line) => {
lineCount++;
bytesRead += Buffer.byteLength(line, "utf8");
// Security: Prevent extremely long lines (potential DoS)
if (line.length > this.MAX_LINE_LENGTH) {
console.warn(`⚠️ Line ${lineCount} exceeds max length, truncating`);
line = line.substring(0, this.MAX_LINE_LENGTH);
}
// Process line
try {
const shouldContinue = lineCallback(line, lineCount);
processedCount++;
// Progress callback
if (onProgress && lineCount % 1000 === 0) {
onProgress({
lineCount,
processedCount,
bytesRead,
totalBytes: stats.size,
progress: ((bytesRead / stats.size) * 100).toFixed(2),
});
}
// Stop if callback returns false or max lines reached
if (shouldContinue === false || lineCount >= maxLines) {
rl.close();
fileStream.destroy();
}
}
catch (error) {
console.error(`Error processing line ${lineCount}:`, error instanceof Error ? error.message : String(error));
if (onError && error instanceof Error) {
onError(error, lineCount);
}
}
});
rl.on("close", () => {
resolve({
lineCount,
processedCount,
bytesRead,
});
});
rl.on("error", (error) => {
if (onError)
onError(error);
reject(error);
});
fileStream.on("error", (error) => {
if (onError)
onError(error);
reject(error);
});
});
}
/**
* Get index with streaming (for large index files)
*/
async getIndex() {
const indexPath = join(this.aicfDir, "index.aicf");
if (!existsSync(indexPath)) {
throw new Error(`Index file not found: ${indexPath}`);
}
const stats = statSync(indexPath);
// Use cache if available and not modified
if (this.indexCache && stats.mtimeMs <= this.lastIndexRead) {
return this.indexCache;
}
// For small files (<1MB), use synchronous read (faster)
if (stats.size < 1024 * 1024) {
const content = readFileSync(indexPath, "utf8");
const lines = content.split("\n").filter(Boolean);
const index = {};
let currentSection = null;
lines.forEach((line) => {
const parts = line.split("|", 2);
const data = parts[1];
if (!data)
return;
if (data.startsWith("@")) {
currentSection = data.substring(1);
if (currentSection) {
index[currentSection] = {};
}
}
else if (currentSection && data.includes("=")) {
const [key, value] = data.split("=", 2);
const section = index[currentSection];
if (key && value !== undefined && section) {
section[key] = value;
}
}
});
this.indexCache = index;
this.lastIndexRead = stats.mtimeMs;
return index;
}
// For large files, use streaming
const index = {};
let currentSection = null;
await this.streamFile(indexPath, (line) => {
if (!line.trim())
return true;
const parts = line.split("|", 2);
const data = parts[1];
if (!data)
return true;
if (data.startsWith("@")) {
currentSection = data.substring(1);
if (currentSection) {
index[currentSection] = {};
}
}
else if (currentSection && data.includes("=")) {
const [key, value] = data.split("=", 2);
const section = index[currentSection];
if (key && value !== undefined && section) {
section[key] = value;
}
}
return true;
});
this.indexCache = index;
this.lastIndexRead = stats.mtimeMs;
return index;
}
/**
* Get last N conversations with streaming
*/
async getLastConversations(count = 5) {
const conversationsPath = join(this.aicfDir, "conversations.aicf");
if (!existsSync(conversationsPath))
return [];
const conversations = [];
let currentConv = null;
const allLines = [];
await this.streamFile(conversationsPath, (line) => {
if (line.trim()) {
allLines.push(line);
}
return true;
});
// Parse from end to get most recent first
for (let i = allLines.length - 1; i >= 0 && conversations.length < count; i--) {
const parts = allLines[i]?.split("|", 2);
if (!parts)
continue;
const lineNum = parts[0];
const data = parts[1];
if (!data)
continue;
if (data.startsWith("@CONVERSATION:")) {
if (currentConv) {
conversations.unshift(currentConv);
}
currentConv = {
id: data.substring(14),
line: parseInt(lineNum || "0"),
metadata: {},
};
}
else if (currentConv && data.includes("=")) {
const [key, value] = data.split("=", 2);
if (key && value !== undefined) {
currentConv.metadata[key] = value;
}
}
}
if (currentConv && conversations.length < count) {
conversations.unshift(currentConv);
}
return conversations;
}
/**
* Get decisions by date range with streaming
*/
async getDecisionsByDate(startDate, endDate = new Date()) {
const decisionsPath = join(this.aicfDir, "decisions.aicf");
if (!existsSync(decisionsPath))
return [];
const decisions = [];
let currentDecision = null;
await this.streamFile(decisionsPath, (line) => {
if (!line.trim())
return true;
const parts = line.split("|", 2);
const lineNum = parts[0];
const data = parts[1];
if (!data)
return true;
if (data.startsWith("@DECISION:")) {
if (currentDecision) {
decisions.push(currentDecision);
}
currentDecision = {
id: data.substring(10),
line: parseInt(lineNum || "0"),
metadata: {},
};
}
else if (currentDecision && data.includes("=")) {
const [key, value] = data.split("=", 2);
if (key && value !== undefined) {
currentDecision.metadata[key] = value;
}
}
return true;
});
if (currentDecision) {
decisions.push(currentDecision);
}
// Filter by date range
return decisions.filter((decision) => {
const timestamp = decision.metadata["timestamp"];
if (!timestamp)
return false;
const decisionDate = new Date(timestamp);
return decisionDate >= startDate && decisionDate <= endDate;
});
}
/**
* Get insights with streaming
*/
async getInsights(options = {}) {
const { limit = 100, category = null, priority = null } = options;
const insightsPath = join(this.aicfDir, "insights.aicf");
if (!existsSync(insightsPath))
return [];
const insights = [];
let currentInsight = null;
await this.streamFile(insightsPath, (line) => {
if (!line.trim())
return true;
const parts = line.split("|", 2);
const lineNum = parts[0];
const data = parts[1];
if (!data)
return true;
if (data.startsWith("@INSIGHT:")) {
if (currentInsight) {
const matchesCategory = !category || (currentInsight.category ?? "") === category;
const matchesPriority = !priority || (currentInsight.priority ?? "") === priority;
if (matchesCategory && matchesPriority) {
insights.push(currentInsight);
}
}
if (insights.length >= limit) {
return false;
}
currentInsight = {
id: data.substring(9),
line: parseInt(lineNum || "0"),
metadata: {},
};
}
else if (currentInsight && data.includes("=")) {
const [key, value] = data.split("=", 2);
if (!key || value === undefined)
return true;
if (key === "text") {
currentInsight.text = value;
}
else if (key === "category") {
currentInsight.category = value;
}
else if (key === "priority") {
currentInsight.priority = value;
}
else if (key === "confidence") {
currentInsight.confidence = value;
}
else if (key === "timestamp") {
currentInsight.timestamp = value;
}
else {
currentInsight.metadata[key] = value;
}
}
return true;
});
if (currentInsight !== null && insights.length < limit) {
const insight = currentInsight;
const matchesCategory = !category || (insight.category ?? "") === category;
const matchesPriority = !priority || (insight.priority ?? "") === priority;
if (matchesCategory && matchesPriority) {
insights.push(insight);
}
}
return insights;
}
/**
* Search across files with streaming (memory-efficient)
*/
async search(term, fileTypes = [
"conversations",
"decisions",
"work-state",
"technical-context",
], options = {}) {
const results = [];
const { maxResults = 100, onProgress = null } = options;
for (const fileType of fileTypes) {
if (results.length >= maxResults)
break;
const filePath = join(this.aicfDir, `${fileType}.aicf`);
if (!existsSync(filePath))
continue;
const contextLines = [];
await this.streamFile(filePath, (line, lineNum) => {
contextLines.push(line);
if (contextLines.length > 3) {
contextLines.shift();
}
if (line.toLowerCase().includes(term.toLowerCase())) {
const parts = line.split("|", 2);
const num = parts[0];
const data = parts[1];
results.push({
file: fileType,
line: parseInt(num || "0") || lineNum,
content: data || line,
context: contextLines.join("\n"),
});
if (onProgress) {
onProgress({ file: fileType, matches: results.length });
}
if (results.length >= maxResults) {
return false;
}
}
return true;
});
}
return results;
}
}
//# sourceMappingURL=aicf-stream-reader.js.map