@gensx/storage
Version:
Cloud storage, blobs, sqlite, and vector database providers/hooks for GenSX.
403 lines (399 loc) • 14.2 kB
JavaScript
/**
* Check out the docs at https://www.gensx.com/docs
* Find us on Github https://github.com/gensx-inc/gensx
* Find us on Discord https://discord.gg/F5BSU8Kc
*/
import * as fs from 'node:fs/promises';
import { createRequire } from 'node:module';
import * as path from 'node:path';
import { fromBase64UrlSafe, toBase64UrlSafe } from '../utils/base64.js';
import { DatabaseError, DatabaseNotFoundError, DatabasePermissionDeniedError, DatabaseSyntaxError, DatabaseConstraintError, DatabaseInternalError } from './types.js';
/* eslint-disable @typescript-eslint/only-throw-error */
const LIBSQL_CLIENT = "@libsql/client";
const requireLibsql = createRequire(import.meta.url);
function getLibsql() {
try {
return requireLibsql(LIBSQL_CLIENT);
}
catch {
throw new Error('@libsql/client is required to use local databases in @gensx/storage but is not installed. Install it as a dev dependency by running: npm install -d @libsql/client\n\nAlternatively, you can use cloud databases instead by setting the storage kind to "cloud" in your configuration.');
}
}
/**
* Helper to convert between filesystem/libSQL errors and DatabaseErrors
*/
function handleError(err, operation) {
if (err instanceof DatabaseError) {
throw err;
}
if (err instanceof Error) {
const nodeErr = err;
if (nodeErr.code === "ENOENT") {
throw new DatabaseNotFoundError(`Database not found: ${String(err.message)}`, err);
}
else if (nodeErr.code === "EACCES") {
throw new DatabasePermissionDeniedError(`Permission denied for operation ${operation}: ${String(err.message)}`, err);
}
// Handle libSQL specific errors
const message = err.message.toLowerCase();
if (message.includes("syntax error")) {
throw new DatabaseSyntaxError(`Syntax error in ${operation}: ${err.message}`, err);
}
if (message.includes("constraint failed") ||
message.includes("unique constraint") ||
message.includes("foreign key constraint")) {
throw new DatabaseConstraintError(`Constraint violation in ${operation}: ${err.message}`, err);
}
}
// Default error case
throw new DatabaseInternalError(`Error during ${operation}: ${String(err)}`, err);
}
/**
* Convert libSQL ResultSet to our DatabaseResult format
*/
function mapResult(result) {
return {
columns: result.columns,
rows: result.rows.map((row) => Object.values(row)),
rowsAffected: result.rowsAffected,
lastInsertId: result.lastInsertRowid
? Number(result.lastInsertRowid)
: undefined,
};
}
/**
* Implementation of Database interface for filesystem storage
*/
class FileSystemDatabase {
client;
dbPath;
dbName;
constructor(rootPath, dbName) {
this.dbName = dbName;
this.dbPath = path.join(rootPath, `${dbName}.db`);
const { createClient } = getLibsql();
this.client = createClient({ url: `file:${this.dbPath}` });
}
async execute(sql, params) {
try {
const result = await this.client.execute({
sql,
args: params,
});
return mapResult(result);
}
catch (err) {
throw handleError(err, "execute");
}
}
async batch(statements) {
try {
const results = [];
// Create a transaction with explicit write mode
const transactionPromise = this.client.transaction("write");
const transaction = await transactionPromise;
try {
// Execute each statement within the transaction
for (const statement of statements) {
const result = await transaction.execute({
sql: statement.sql,
args: statement.params,
});
results.push(mapResult(result));
}
// Commit the transaction if all statements succeeded
await transaction.commit();
return { results };
}
catch (err) {
// Transaction will be rolled back in the finally block
throw err;
}
finally {
// Always close the transaction
transaction.close();
}
}
catch (err) {
throw handleError(err, "batch");
}
}
async executeMultiple(sql) {
try {
// Split the SQL by semicolons, ignoring those in quotes or comments
const statements = sql
.split(";")
.map((s) => s.trim())
.filter((s) => s.length > 0)
.map((s) => ({ sql: `${s};` }));
const results = [];
// Execute each statement without transaction
for (const statement of statements) {
try {
const result = await this.client.execute({ sql: statement.sql });
results.push(mapResult(result));
}
catch (_) {
// If one statement fails, we still try to execute the others
results.push({
columns: [],
rows: [],
rowsAffected: 0,
lastInsertId: undefined,
});
}
}
return { results };
}
catch (err) {
throw handleError(err, "executeMultiple");
}
}
async migrate(sql) {
try {
// Disable foreign keys, run migrations, then re-enable foreign keys
const results = [];
// Disable foreign keys
const disableResult = await this.client.execute({
sql: "PRAGMA foreign_keys = OFF;",
});
results.push(mapResult(disableResult));
// Split and execute migration statements
const migrationStatements = sql
.split(";")
.map((s) => s.trim())
.filter((s) => s.length > 0)
.map((s) => ({ sql: `${s};` }));
// Execute migrations
for (const statement of migrationStatements) {
try {
const result = await this.client.execute({ sql: statement.sql });
results.push(mapResult(result));
}
catch (err) {
// Re-enable foreign keys before rethrowing
await this.client.execute({ sql: "PRAGMA foreign_keys = ON;" });
throw err;
}
}
// Re-enable foreign keys
const enableResult = await this.client.execute({
sql: "PRAGMA foreign_keys = ON;",
});
results.push(mapResult(enableResult));
return { results };
}
catch (err) {
throw handleError(err, "migrate");
}
}
async getInfo() {
try {
// Get file stats
let stats;
try {
stats = await fs.stat(this.dbPath);
}
catch (err) {
if (err.code === "ENOENT") {
// Database file doesn't exist yet, return minimal info
return {
name: this.dbName,
size: 0,
lastModified: new Date(),
tables: [],
};
}
throw err;
}
// Get table information
const tablesResult = await this.client.execute({
sql: "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';",
});
const tables = [];
for (const row of tablesResult.rows) {
const tableName = row.name;
// Get column information for this table
const columnsResult = await this.client.execute({
sql: `PRAGMA table_info(${tableName});`,
});
const columns = columnsResult.rows.map((col) => ({
name: col.name,
type: col.type,
notNull: Boolean(col.notnull),
defaultValue: col.dflt_value,
primaryKey: Boolean(col.pk),
}));
tables.push({
name: tableName,
columns,
});
}
return {
name: this.dbName,
size: stats.size,
lastModified: stats.mtime,
tables,
};
}
catch (err) {
throw handleError(err, "getInfo");
}
}
close() {
try {
this.client.close();
}
catch (err) {
throw handleError(err, "close");
}
}
}
/**
* Implementation of DatabaseStorage interface for filesystem storage
*/
class FileSystemDatabaseStorage {
rootPath;
databases = new Map();
constructor(rootPath) {
this.rootPath = rootPath;
// Ensure rootPath exists on instantiation
void this.ensureRootDir();
}
/**
* Ensure the root directory exists
*/
async ensureRootDir() {
try {
await fs.mkdir(this.rootPath, { recursive: true });
}
catch (err) {
throw handleError(err, "ensureRootDir");
}
}
getDatabase(name) {
let db = this.databases.get(name);
if (!db) {
db = new FileSystemDatabase(this.rootPath, name);
this.databases.set(name, db);
}
return db;
}
async listDatabases(options) {
try {
const files = await fs.readdir(this.rootPath);
// Filter for .db files and remove extension
let dbFiles = files
.filter((file) => file.endsWith(".db"))
.map((file) => file.slice(0, -3)); // Remove .db extension
// Sort files for consistent pagination
dbFiles.sort();
// If cursor is provided, start after that file
if (options?.cursor) {
const lastFile = fromBase64UrlSafe(options.cursor);
const startIndex = dbFiles.findIndex((file) => file > lastFile);
if (startIndex !== -1) {
dbFiles = dbFiles.slice(startIndex);
}
else {
dbFiles = [];
}
}
// Apply limit if provided
let nextCursor;
if (options?.limit && options.limit < dbFiles.length) {
const limitedFiles = dbFiles.slice(0, options.limit);
// Set cursor to last file for next page
nextCursor = toBase64UrlSafe(limitedFiles[limitedFiles.length - 1]);
dbFiles = limitedFiles;
}
// Get creation dates for each database file
const databases = await Promise.all(dbFiles.map(async (file) => {
const filePath = path.join(this.rootPath, `${file}.db`);
const stats = await fs.stat(filePath);
return {
name: file,
createdAt: stats.birthtime,
};
}));
return {
databases,
...(nextCursor && { nextCursor }),
};
}
catch (err) {
if (err.code === "ENOENT") {
return {
databases: [],
};
}
throw handleError(err, "listDatabases");
}
}
async ensureDatabase(name) {
try {
const dbPath = path.join(this.rootPath, `${name}.db`);
// First check if database file exists
let exists = false;
try {
await fs.access(dbPath);
exists = true;
}
catch (err) {
if (err.code !== "ENOENT") {
throw err;
}
}
if (exists) {
// If it exists, make sure it's in our cache
if (!this.databases.has(name)) {
this.getDatabase(name);
}
return { exists: true, created: false };
}
// Ensure the directory exists
await this.ensureRootDir();
// Create an empty database by getting a database instance
// which will initialize the file
const db = this.getDatabase(name);
// Execute a simple query to ensure the file is created
await db.execute("SELECT 1");
return { exists: false, created: true };
}
catch (err) {
throw handleError(err, "ensureDatabase");
}
}
async deleteDatabase(name) {
try {
const dbPath = path.join(this.rootPath, `${name}.db`);
// Close any open connection to this database
if (this.databases.has(name)) {
const db = this.databases.get(name);
db.close();
this.databases.delete(name);
}
// Check if file exists
try {
await fs.access(dbPath);
}
catch (err) {
if (err.code === "ENOENT") {
return { deleted: false };
}
throw err;
}
// Delete the file
await fs.unlink(dbPath);
return { deleted: true };
}
catch (err) {
throw handleError(err, "deleteDatabase");
}
}
hasEnsuredDatabase(name) {
return this.databases.has(name);
}
}
export { FileSystemDatabase, FileSystemDatabaseStorage };
//# sourceMappingURL=filesystem.js.map