crewai-ts
Version:
TypeScript port of crewAI for agent-based workflows
455 lines (454 loc) • 15.8 kB
JavaScript
/**
* DeduplicatedContentStorage implementation
* Provides memory-efficient storage with content deduplication
* to minimize redundant memory usage for identical or similar content
*/
import { createHash } from 'crypto';
/**
* Simple bloom filter implementation for fast negative lookups
* A bloom filter allows quickly determining if an item is definitely NOT in a set
* with no false negatives (but possible false positives)
*/
// Use a more TypeScript-friendly implementation with proper initialization
class BloomFilter {
// Define properties with explicit initialization
bits;
hashFunctions;
size;
constructor(size = 10000, hashFunctions = 7) {
this.size = size;
this.bits = new Uint8Array(Math.ceil(size / 8));
this.hashFunctions = hashFunctions;
// Ensure bits is always initialized
if (!this.bits) {
this.bits = new Uint8Array(Math.ceil(this.size / 8));
}
}
add(item) {
// Ensure bits array is initialized before use
if (!this.bits || this.bits.length === 0) {
this.bits = new Uint8Array(Math.ceil(this.size / 8));
}
const positions = this.getPositions(item);
// Use type assertion to satisfy TypeScript
const bits = this.bits;
for (const pos of positions) {
const byteIndex = Math.floor(pos / 8);
// Add bounds check to ensure byteIndex is valid
if (byteIndex >= 0 && byteIndex < bits.length) {
const bitIndex = pos % 8;
bits[byteIndex] |= (1 << bitIndex);
}
}
}
contains(item) {
// Ensure bits array is initialized before use
if (!this.bits || this.bits.length === 0) {
return false; // Empty filter never contains anything
}
const positions = this.getPositions(item);
// Use type assertion to satisfy TypeScript
const bitsArray = this.bits;
for (const pos of positions) {
const byteIndex = Math.floor(pos / 8);
// Add bounds check to ensure byteIndex is valid
if (byteIndex < 0 || byteIndex >= bitsArray.length) {
return false; // Position out of bounds, so item can't be in set
}
const bitIndex = pos % 8;
if (!(bitsArray[byteIndex] & (1 << bitIndex))) {
return false; // Definitely not in the set
}
}
return true; // Might be in the set
}
clear() {
const bits = this.bits;
if (bits) {
bits.fill(0);
}
}
// Generate hash positions for an item
getPositions(item) {
const positions = [];
const h1 = this.hash1(item);
const h2 = this.hash2(item);
for (let i = 0; i < this.hashFunctions; i++) {
const pos = (h1 + i * h2) % this.size;
positions.push(pos);
}
return positions;
}
// Simple hash functions for the bloom filter
hash1(item) {
let hash = 0;
for (let i = 0; i < item.length; i++) {
hash = ((hash << 5) - hash) + item.charCodeAt(i);
hash |= 0; // Convert to 32-bit integer
}
return Math.abs(hash % this.size);
}
hash2(item) {
let hash = 0;
for (let i = 0; i < item.length; i++) {
hash = ((hash << 7) - hash) + item.charCodeAt(i);
hash |= 0; // Convert to 32-bit integer
}
return Math.abs(hash % this.size) || 1; // Ensure non-zero
}
}
/**
* DeduplicatedContentStorage class
* Provides memory-efficient storage through content deduplication
*/
export class DeduplicatedContentStorage {
// Content store maps content hashes to actual content
contentStore = new Map();
// Metadata for each stored content
contentMetadata = new Map();
// Reference mappings (ID -> content hash)
references = new Map();
// Bloom filter for fast negative lookups
bloomFilter = null;
// Partial content chunks for chunk-level deduplication
contentChunks = new Map();
// Configuration
hashAlgorithm;
trackStats;
chunkSize;
compressContent;
// Statistics for monitoring performance
stats = {
totalItems: 0,
uniqueContents: 0,
totalSizeBytes: 0,
dedupSavingsBytes: 0,
dedupRatio: 0,
retrievals: 0,
stores: 0,
chunkStats: {
totalChunks: 0,
uniqueChunks: 0,
chunkSavingsBytes: 0
}
};
constructor(options = {}) {
this.hashAlgorithm = options.hashAlgorithm ?? 'sha256';
this.trackStats = options.trackStats ?? true;
this.chunkSize = options.chunkSize ?? 0;
this.compressContent = options.compressContent ?? false;
// Initialize bloom filter if enabled
if (options.useBloomFilter ?? true) {
this.bloomFilter = new BloomFilter();
}
}
/**
* Store content with deduplication
* Returns a reference ID for the stored content
*/
store(content, id, metadata) {
// Generate ID if not provided
const contentId = id ?? this.generateId();
// Calculate content hash
const hash = this.hashContent(content);
// Check if content already exists using bloom filter for fast negative lookups
if (this.bloomFilter && !this.bloomFilter.contains(hash)) {
// Definitely not in the store, add to bloom filter
this.bloomFilter.add(hash);
}
// Store content if it doesn't exist yet
if (!this.contentStore.has(hash)) {
// Process content for storage (chunk or compress if enabled)
const processedContent = this.processContentForStorage(content);
// Store the content
this.contentStore.set(hash, processedContent);
// Create metadata
this.contentMetadata.set(hash, {
hash,
size: processedContent.length,
originalSize: content.length,
referenceCount: 0,
createdAt: Date.now(),
lastAccessedAt: Date.now(),
compressionRatio: this.compressContent ? processedContent.length / content.length : undefined
});
// Update statistics
if (this.trackStats) {
this.stats.uniqueContents++;
}
}
// Update the reference mapping
this.references.set(contentId, hash);
// Increment reference count
const metadata_ = this.contentMetadata.get(hash);
if (metadata_) {
metadata_.referenceCount++;
metadata_.lastAccessedAt = Date.now();
this.contentMetadata.set(hash, metadata_);
}
// Update statistics
if (this.trackStats) {
this.stats.totalItems++;
this.stats.totalSizeBytes += content.length;
this.stats.dedupSavingsBytes = this.calculateDedupSavings();
this.stats.dedupRatio = this.stats.dedupSavingsBytes / this.stats.totalSizeBytes;
this.stats.stores++;
}
return contentId;
}
/**
* Retrieve content by reference ID
*/
retrieve(id) {
// Get the content hash from the reference
const hash = this.references.get(id);
if (!hash) {
return null;
}
// Get the content using the hash
const content = this.contentStore.get(hash);
if (!content) {
return null;
}
// Update access time
const metadata = this.contentMetadata.get(hash);
if (metadata) {
metadata.lastAccessedAt = Date.now();
this.contentMetadata.set(hash, metadata);
}
// Update statistics
if (this.trackStats) {
this.stats.retrievals++;
}
// Process content for retrieval (unchunk or decompress)
return this.processContentForRetrieval(content);
}
/**
* Get content metadata
*/
getMetadata(id) {
const hash = this.references.get(id);
if (!hash) {
return null;
}
return this.contentMetadata.get(hash) || null;
}
/**
* Remove content reference
* Content is only deleted when no more references exist
*/
remove(id) {
// Get the content hash from the reference
const hash = this.references.get(id);
if (!hash) {
return false;
}
// Remove the reference
this.references.delete(id);
// Get metadata to update reference count
const metadata = this.contentMetadata.get(hash);
if (metadata) {
metadata.referenceCount--;
// If no more references, remove the content
if (metadata.referenceCount <= 0) {
this.contentStore.delete(hash);
this.contentMetadata.delete(hash);
// Update statistics
if (this.trackStats) {
this.stats.uniqueContents--;
}
}
else {
// Update metadata
this.contentMetadata.set(hash, metadata);
}
}
// Update statistics
if (this.trackStats) {
this.stats.totalItems--;
this.stats.dedupSavingsBytes = this.calculateDedupSavings();
this.stats.dedupRatio = this.stats.dedupSavingsBytes / (this.stats.totalSizeBytes || 1);
}
return true;
}
/**
* Check if a reference exists
*/
has(id) {
return this.references.has(id);
}
/**
* Clear all content and references
*/
clear() {
this.contentStore.clear();
this.contentMetadata.clear();
this.references.clear();
this.contentChunks.clear();
if (this.bloomFilter) {
this.bloomFilter.clear();
}
// Reset statistics
if (this.trackStats) {
this.stats = {
totalItems: 0,
uniqueContents: 0,
totalSizeBytes: 0,
dedupSavingsBytes: 0,
dedupRatio: 0,
retrievals: 0,
stores: 0,
chunkStats: {
totalChunks: 0,
uniqueChunks: 0,
chunkSavingsBytes: 0
}
};
}
}
/**
* Get storage statistics
*/
getStats() {
return { ...this.stats };
}
/**
* Calculate the memory savings due to deduplication
*/
calculateDedupSavings() {
// Sum all original content sizes
let totalOriginalSize = 0;
let actualStorageSize = 0;
for (const [hash, metadata] of this.contentMetadata.entries()) {
totalOriginalSize += metadata.originalSize * metadata.referenceCount;
actualStorageSize += metadata.size; // Only count once since content is shared
}
return totalOriginalSize - actualStorageSize;
}
/**
* Process content for storage, including chunking or compression
*/
processContentForStorage(content) {
if (this.chunkSize > 0) {
// Implement chunk-level deduplication
return this.chunkifyContent(content);
}
if (this.compressContent) {
// In a real implementation, we would compress content here
// For demonstration, we'll just return the original content
// since actual compression would require additional libraries
return content;
}
return content;
}
/**
* Process content for retrieval, including unchunking or decompression
*/
processContentForRetrieval(content) {
if (this.chunkSize > 0) {
// Rebuild content from chunks
return this.unchunkifyContent(content);
}
if (this.compressContent) {
// In a real implementation, we would decompress content here
return content;
}
return content;
}
/**
* Chunkify content for chunk-level deduplication
*/
chunkifyContent(content) {
if (this.chunkSize <= 0) {
return content;
}
// Split content into chunks of chunkSize
const chunks = [];
for (let i = 0; i < content.length; i += this.chunkSize) {
const chunkContent = content.substring(i, i + this.chunkSize);
const chunkHash = this.hashContent(chunkContent);
// Store chunk if it doesn't exist
if (!this.contentChunks.has(chunkHash)) {
this.contentChunks.set(chunkHash, chunkContent);
if (this.trackStats) {
this.stats.chunkStats.uniqueChunks++;
}
}
// Add chunk hash to the list
chunks.push(chunkHash);
if (this.trackStats) {
this.stats.chunkStats.totalChunks++;
}
}
// Return the list of chunk hashes as a string
return chunks.join('|');
}
/**
* Rebuild content from chunks
*/
unchunkifyContent(chunkedContent) {
if (this.chunkSize <= 0 || !chunkedContent.includes('|')) {
return chunkedContent;
}
// Split the chunked content into hash references
const chunkHashes = chunkedContent.split('|');
// Rebuild the content from chunks
const contentParts = [];
for (const chunkHash of chunkHashes) {
// Safe access to contentChunks with null check
const contentChunks = this.contentChunks;
if (contentChunks) {
const chunkContent = contentChunks.get(chunkHash);
if (chunkContent) {
contentParts.push(chunkContent);
}
else {
// Missing chunk - this shouldn't happen with proper reference counting
contentParts.push(`[Missing chunk: ${chunkHash}]`);
}
}
else {
contentParts.push(`[Missing chunk map: ${chunkHash}]`);
}
}
return contentParts.join('');
}
/**
* Hash content using configured algorithm
*/
hashContent(content) {
if (this.hashAlgorithm === 'simple') {
// Simple non-cryptographic hash for performance
return this.simpleHash(content);
}
// Use Node.js crypto module for cryptographic hashes
return createHash(this.hashAlgorithm).update(content).digest('hex');
}
/**
* Simple non-cryptographic hash function
* This is faster but has more collisions than cryptographic hashes
*/
simpleHash(content) {
let h1 = 0xdeadbeef;
let h2 = 0x41c6ce57;
for (let i = 0; i < content.length; i++) {
const ch = content.charCodeAt(i);
h1 = Math.imul(h1 ^ ch, 2654435761);
h2 = Math.imul(h2 ^ ch, 1597334677);
}
h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
// Combine h1 and h2 into a 16-character hex string
return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
}
/**
* Generate a unique ID for content references
*/
generateId() {
return createHash('md5')
.update(`${Date.now()}-${Math.random()}`)
.digest('hex');
}
}