crewai-ts
Version:
TypeScript port of crewAI for agent-based workflows
366 lines (365 loc) • 12.9 kB
JavaScript
/**
* SQLiteStorageAdapter
*
* A SQLite-based implementation of the StorageAdapter interface
* Optimized for efficient local persistence with high performance
*/
import { BaseStorageAdapter } from './StorageAdapter.js';
/**
* SQLiteStorageAdapter implements the StorageAdapter interface
* using SQLite for efficient and durable storage
*
* Note: This implementation requires the 'better-sqlite3' package
* npm install better-sqlite3
*/
export class SQLiteStorageAdapter extends BaseStorageAdapter {
db; // SQLite database instance
statements = {}; // Prepared statements cache
cache = new Map();
cacheSize;
initialized = false;
addCacheMutex = false; // Simple mutex to prevent concurrent operations
constructor(options) {
super(options);
this.cacheSize = options.cacheSize || 1000;
try {
// Dynamically import better-sqlite3
// This allows the adapter to be used in environments where SQLite isn't needed
const initDb = () => {
try {
// We use dynamic import for better-sqlite3 to avoid requiring it as a dependency
// when other storage adapters are used
const Database = require('better-sqlite3');
this.db = new Database(options.dbPath, {
verbose: options.debug ? console.log : undefined
});
// Enable WAL mode for better performance if requested
if (options.enableWAL) {
this.db.pragma('journal_mode = WAL');
}
// Enable query cache for better performance if requested
if (options.enableQueryCache) {
this.db.pragma('cache_size = 2000');
}
// Setup database
this.setupDatabase(options.schemaVersion || 1);
this.initialized = true;
}
catch (error) {
if (options.debug) {
console.error('Failed to initialize SQLite database:', error);
}
throw new Error(`SQLite initialization failed: ${error instanceof Error ? error.message : String(error)}`);
}
};
// Initialize immediately but don't block constructor
initDb();
}
catch (error) {
if (options.debug) {
console.error('Error in SQLiteStorageAdapter constructor:', error);
}
}
}
/**
* Set up the database schema
*/
setupDatabase(schemaVersion) {
// Enable foreign keys
this.db.pragma('foreign_keys = ON');
// Create a settings table to track schema version
this.db.exec(`
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)
`);
// Create the main storage table
this.db.exec(`
CREATE TABLE IF NOT EXISTS storage (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
timestamp INTEGER NOT NULL,
namespace TEXT NOT NULL
)
`);
// Create an index on namespace for faster queries
this.db.exec(`
CREATE INDEX IF NOT EXISTS idx_storage_namespace
ON storage(namespace)
`);
// Check schema version and perform migrations if needed
const getVersionStmt = this.db.prepare('SELECT value FROM settings WHERE key = ?');
const currentVersionRow = getVersionStmt.get('schema_version');
const currentVersion = currentVersionRow ? parseInt(currentVersionRow.value, 10) : 0;
if (currentVersion < schemaVersion) {
// Perform migrations based on version differences
this.migrateSchema(currentVersion, schemaVersion);
// Update schema version
const setVersionStmt = this.db.prepare('INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)');
setVersionStmt.run('schema_version', schemaVersion.toString());
}
// Prepare common statements for better performance
this.prepareStatements();
}
/**
* Prepare common SQL statements
*/
prepareStatements() {
this.statements.insert = this.db.prepare('INSERT OR REPLACE INTO storage (key, value, timestamp, namespace) VALUES (?, ?, ?, ?)');
this.statements.select = this.db.prepare('SELECT value, timestamp FROM storage WHERE key = ?');
this.statements.delete = this.db.prepare('DELETE FROM storage WHERE key = ?');
this.statements.keys = this.db.prepare('SELECT key, timestamp FROM storage WHERE namespace = ?');
this.statements.clear = this.db.prepare('DELETE FROM storage WHERE namespace = ?');
// Create a transaction for batch inserts with proper typing
this.statements.batchInsert = this.db.transaction((items) => {
for (const item of items) {
this.statements.insert.run(item.key, item.value, item.timestamp, item.namespace);
}
});
}
/**
* Migrate database schema
*/
migrateSchema(fromVersion, toVersion) {
// Example migration logic
if (fromVersion < 1 && toVersion >= 1) {
// Migration to version 1
// (Already handled by initial setup)
}
if (fromVersion < 2 && toVersion >= 2) {
// Example: Add a new index for timestamp-based queries
this.db.exec(`
CREATE INDEX IF NOT EXISTS idx_storage_timestamp
ON storage(timestamp)
`);
}
}
/**
* Ensure database is initialized
*/
ensureInitialized() {
if (!this.initialized) {
throw new Error('SQLite database not initialized');
}
}
/**
* Add an item to the cache with LRU eviction
*/
addToCache(key, value, timestamp) {
// Simple mutex to prevent concurrent cache operations
if (this.addCacheMutex)
return;
try {
this.addCacheMutex = true;
// If cache is full, remove least recently used item
if (this.cache.size >= this.cacheSize) {
// Get the oldest item (first inserted)
const oldestKey = this.cache.keys().next().value;
if (oldestKey) {
this.cache.delete(oldestKey);
}
}
// Add to cache
this.cache.set(key, { value, timestamp });
}
finally {
this.addCacheMutex = false;
}
}
/**
* Save a value with transaction support
*/
async save(key, value) {
this.ensureInitialized();
const namespacedKey = this.getNamespacedKey(key);
const timestamp = Date.now();
const serializedValue = this.serialize(value);
try {
// Save to database
this.statements.insert.run(namespacedKey, serializedValue, timestamp, this.options.namespace);
// Update cache
this.addToCache(namespacedKey, value, timestamp);
}
catch (error) {
if (this.options.debug) {
console.error(`Error saving key ${key}:`, error);
}
throw error;
}
}
/**
* Load a value with caching
*/
async load(key) {
this.ensureInitialized();
const namespacedKey = this.getNamespacedKey(key);
// Check cache first
const cachedItem = this.cache.get(namespacedKey);
if (cachedItem) {
// Check for expiration
if (this.isExpired(cachedItem.timestamp)) {
// Remove expired item
this.cache.delete(namespacedKey);
await this.delete(key);
return null;
}
return cachedItem.value;
}
try {
// Load from database
const row = this.statements.select.get(namespacedKey);
if (!row)
return null;
// Check for expiration
if (this.isExpired(row.timestamp)) {
await this.delete(key);
return null;
}
// Deserialize value
const value = this.deserialize(row.value);
// Update cache
this.addToCache(namespacedKey, value, row.timestamp);
return value;
}
catch (error) {
if (this.options.debug) {
console.error(`Error loading key ${key}:`, error);
}
return null;
}
}
/**
* Delete a value with proper cleanup
*/
async delete(key) {
this.ensureInitialized();
const namespacedKey = this.getNamespacedKey(key);
try {
// Delete from database
const result = this.statements.delete.run(namespacedKey);
// Remove from cache
this.cache.delete(namespacedKey);
return result.changes > 0;
}
catch (error) {
if (this.options.debug) {
console.error(`Error deleting key ${key}:`, error);
}
return false;
}
}
/**
* Clear all values for the current namespace
*/
async clear() {
this.ensureInitialized();
try {
// Clear from database
this.statements.clear.run(this.options.namespace);
// Clear cache for this namespace
const namespacedPrefix = `${this.options.namespace}:`;
for (const key of this.cache.keys()) {
if (key.startsWith(namespacedPrefix)) {
this.cache.delete(key);
}
}
}
catch (error) {
if (this.options.debug) {
console.error('Error clearing storage:', error);
}
throw error;
}
}
/**
* Get all keys with namespace and expiration handling
*/
async keys() {
this.ensureInitialized();
const prefix = `${this.options.namespace}:`;
try {
// Get all keys for this namespace
const rows = this.statements.keys.all(this.options.namespace);
const result = [];
const now = Date.now();
for (const row of rows) {
// Skip expired keys
if (this.isExpired(row.timestamp)) {
// Delete expired key
this.statements.delete.run(row.key);
continue;
}
// Remove namespace prefix
if (row.key.startsWith(prefix)) {
result.push(row.key.substring(prefix.length));
}
}
return result;
}
catch (error) {
if (this.options.debug) {
console.error('Error getting keys:', error);
}
return [];
}
}
/**
* Close the database connection
*/
async close() {
if (this.initialized && this.db) {
try {
this.db.close();
this.initialized = false;
}
catch (error) {
if (this.options.debug) {
console.error('Error closing database:', error);
}
}
}
}
/**
* Run vacuum to optimize database
*/
async optimize() {
this.ensureInitialized();
try {
// Run vacuum to reclaim space and optimize
this.db.exec('VACUUM');
// Run analyze to update statistics
this.db.exec('ANALYZE');
}
catch (error) {
if (this.options.debug) {
console.error('Error optimizing database:', error);
}
}
}
/**
* Get storage statistics
*/
getStats() {
this.ensureInitialized();
try {
// Count items
const countStmt = this.db.prepare('SELECT COUNT(*) as count FROM storage WHERE namespace = ?');
const countResult = countStmt.get(this.options.namespace);
// Get database size
const sizeStmt = this.db.prepare('SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()');
const sizeResult = sizeStmt.get();
return {
itemCount: countResult.count,
cacheSize: this.cache.size,
dbSize: sizeResult.size
};
}
catch (error) {
if (this.options.debug) {
console.error('Error getting stats:', error);
}
return { itemCount: 0, cacheSize: 0, dbSize: 0 };
}
}
}