@sailboat-computer/data-storage
Version:
Shared data storage library for sailboat computer v3
599 lines • 22.6 kB
JavaScript
"use strict";
/**
* Storage manager implementation
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.createStorageManager = exports.StorageManagerImpl = void 0;
const uuid_1 = require("uuid");
const types_1 = require("../types");
const errors_1 = require("../utils/errors");
const redis_1 = require("./providers/redis");
const influxdb_1 = require("./providers/influxdb");
const postgresql_1 = require("./providers/postgresql");
/**
* Default storage manager options
*/
const DEFAULT_OPTIONS = {
retryAttempts: 3,
retryDelayMs: 100,
enableMetrics: true,
metricsInterval: 60000, // 1 minute
logger: {
level: 'info',
console: true
}
};
/**
* Storage manager implementation
*/
class StorageManagerImpl {
/**
* Create a new storage manager
*
* @param config - Storage configuration
* @param options - Storage manager options
*/
constructor(config, options = DEFAULT_OPTIONS) {
this.config = config;
this.options = options;
this.providers = {};
this.initialized = false;
this.metrics = {
operationsPerSecond: 0,
averageLatency: 0,
errorRate: 0,
operationCount: 0,
errorCount: 0,
totalLatency: 0
};
}
/**
* Initialize the storage manager
*/
async initialize() {
if (this.initialized) {
return;
}
try {
// Initialize providers
if (this.config.providers.hot) {
this.providers[types_1.StorageTier.HOT] = (0, redis_1.createRedisProvider)(this.config.providers.hot.config);
}
if (this.config.providers.warm) {
this.providers[types_1.StorageTier.WARM] = (0, influxdb_1.createInfluxDBProvider)(this.config.providers.warm.config);
}
if (this.config.providers.cold) {
this.providers[types_1.StorageTier.COLD] = (0, postgresql_1.createPostgreSQLProvider)(this.config.providers.cold.config);
}
// Initialize each provider
for (const [tier, provider] of Object.entries(this.providers)) {
try {
await provider.initialize();
console.log(`Initialized ${tier} storage provider`);
}
catch (error) {
console.error(`Failed to initialize ${tier} storage provider:`, error);
throw new errors_1.StorageError(errors_1.StorageErrorCode.CONNECTION_FAILED, `Failed to initialize ${tier} storage provider`, { tier, error });
}
}
// Start metrics collection if enabled
if (this.options.enableMetrics) {
this.startMetricsCollection();
}
this.initialized = true;
console.log('Storage manager initialized');
}
catch (error) {
console.error('Failed to initialize storage manager:', error);
throw new errors_1.StorageError(errors_1.StorageErrorCode.CONNECTION_FAILED, 'Failed to initialize storage manager', { error });
}
}
/**
* Store data
*
* @param data - Data to store
* @param category - Data category
* @param options - Storage options
* @returns Data ID
*/
async store(data, category, options = {}) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
// Determine storage tier
const tier = options.tier || types_1.StorageTier.HOT;
const provider = this.getProvider(tier);
// Prepare metadata
const metadata = {
id: (0, uuid_1.v4)(),
category,
timestamp: options.timestamp ? options.timestamp.toISOString() : new Date().toISOString(),
tags: options.tags || {},
tier
};
// Store data with retry
const id = await (0, errors_1.withRetry)(() => provider.store(data, metadata), this.options.retryAttempts, this.options.retryDelayMs);
this.updateMetrics(Date.now() - startTime);
return id;
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.STORE_FAILED, 'Failed to store data', { category, error });
}
}
/**
* Retrieve data
*
* @param query - Data query
* @param tier - Storage tier to query (optional)
* @returns Stored data
*/
async retrieve(query, tier) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
if (tier) {
// Query specific tier
const provider = this.getProvider(tier);
const results = await (0, errors_1.withRetry)(() => provider.retrieve(query), this.options.retryAttempts, this.options.retryDelayMs);
this.updateMetrics(Date.now() - startTime);
return results;
}
else {
// Query all tiers, starting with hot and moving to colder tiers
const tiers = [types_1.StorageTier.HOT, types_1.StorageTier.WARM, types_1.StorageTier.COLD];
let results = [];
for (const t of tiers) {
if (this.providers[t]) {
try {
const tierResults = await this.providers[t].retrieve(query);
results = [...results, ...tierResults];
// If we found results and the query has a limit, check if we've reached it
if (results.length > 0 && query.limit && results.length >= query.limit) {
results = results.slice(0, query.limit);
break;
}
}
catch (err) {
console.warn(`Error retrieving from ${t} tier:`, err);
// Continue to next tier on error
}
}
}
this.updateMetrics(Date.now() - startTime);
return results;
}
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.RETRIEVE_FAILED, 'Failed to retrieve data', { query, tier, error });
}
}
/**
* Update stored data
*
* @param data - Data to update
*/
async update(data) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
const tier = data.metadata.tier || types_1.StorageTier.HOT;
const provider = this.getProvider(tier);
await (0, errors_1.withRetry)(() => provider.update(data), this.options.retryAttempts, this.options.retryDelayMs);
this.updateMetrics(Date.now() - startTime);
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.UPDATE_FAILED, 'Failed to update data', { id: data.metadata.id, error });
}
}
/**
* Delete data
*
* @param id - Data ID
*/
async delete(id) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
// Try to delete from all tiers
const tiers = [types_1.StorageTier.HOT, types_1.StorageTier.WARM, types_1.StorageTier.COLD];
let deleted = false;
for (const tier of tiers) {
if (this.providers[tier]) {
try {
await this.providers[tier].delete(id);
deleted = true;
}
catch (err) {
// Continue to next tier on error
console.warn(`Error deleting from ${tier} tier:`, err);
}
}
}
if (!deleted) {
throw new Error('Data not found in any tier');
}
this.updateMetrics(Date.now() - startTime);
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.DELETE_FAILED, 'Failed to delete data', { id, error });
}
}
/**
* Store multiple data items
*
* @param items - Data items to store
* @returns Data IDs
*/
async storeBatch(items) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
// Group items by tier
const tierGroups = {
'storage_hot': [],
'storage_warm': [],
'storage_cold': []
};
// Prepare items for each tier
for (const item of items) {
const tier = item.options?.tier || types_1.StorageTier.HOT;
tierGroups[tier].push({
data: item.data,
metadata: {
id: (0, uuid_1.v4)(),
category: item.category,
timestamp: item.options?.timestamp ? item.options.timestamp.toISOString() : new Date().toISOString(),
tags: item.options?.tags || {},
tier
}
});
}
// Store items for each tier
const results = [];
for (const [tier, tierItems] of Object.entries(tierGroups)) {
if (tierItems.length > 0 && this.providers[tier]) {
const provider = this.providers[tier];
const tierResults = await (0, errors_1.withRetry)(() => provider.storeBatch(tierItems), this.options.retryAttempts, this.options.retryDelayMs);
results.push(...tierResults);
}
}
this.updateMetrics(Date.now() - startTime);
return results;
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.STORE_FAILED, 'Failed to store batch data', { itemCount: items.length, error });
}
}
/**
* Retrieve multiple data sets
*
* @param queries - Data queries
* @param tier - Storage tier to query (optional)
* @returns Stored data sets
*/
async retrieveBatch(queries, tier) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
const results = [];
for (const query of queries) {
const queryResults = await this.retrieve(query, tier);
results.push(queryResults);
}
this.updateMetrics(Date.now() - startTime);
return results;
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.RETRIEVE_FAILED, 'Failed to retrieve batch data', { queryCount: queries.length, error });
}
}
/**
* Clean up old data
*
* @param category - Data category
* @param tier - Storage tier
* @param retentionDays - Retention period in days
* @returns Cleanup result
*/
async cleanup(category, tier, retentionDays) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
const provider = this.getProvider(tier);
const result = await (0, errors_1.withRetry)(() => provider.cleanup(category, retentionDays), this.options.retryAttempts, this.options.retryDelayMs);
this.updateMetrics(Date.now() - startTime);
return result;
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.CLEANUP_FAILED, 'Failed to clean up data', { category, tier, retentionDays, error });
}
}
/**
* Migrate data between tiers
*
* @param id - Data ID
* @param fromTier - Source tier
* @param toTier - Destination tier
*/
async migrate(id, fromTier, toTier) {
this.ensureInitialized();
const startTime = Date.now();
let error = null;
try {
// Get source provider
const sourceProvider = this.getProvider(fromTier);
// Get destination provider
const destProvider = this.getProvider(toTier);
// Retrieve data from source
const data = await sourceProvider.retrieve({ id });
if (data.length === 0) {
throw new Error(`Data with ID ${id} not found in ${fromTier} tier`);
}
// Store data in destination
const item = data[0];
// Update tier in metadata
const updatedItem = {
...item,
metadata: {
...item.metadata,
tier: toTier
}
};
// Store in destination
await destProvider.store(updatedItem.data, updatedItem.metadata);
// Delete from source
await sourceProvider.delete(id);
this.updateMetrics(Date.now() - startTime);
}
catch (err) {
error = err instanceof Error ? err : new Error(String(err));
this.updateMetrics(Date.now() - startTime, true);
throw new errors_1.StorageError(errors_1.StorageErrorCode.MIGRATION_FAILED, 'Failed to migrate data', { id, fromTier, toTier, error });
}
}
/**
* Shutdown the storage manager
*/
async shutdown() {
if (!this.initialized) {
return;
}
try {
// Stop metrics collection
if (this.metricsInterval) {
clearInterval(this.metricsInterval);
this.metricsInterval = undefined;
}
// Close all providers
for (const [tier, provider] of Object.entries(this.providers)) {
try {
await provider.shutdown();
console.log(`Closed ${tier} storage provider`);
}
catch (error) {
console.error(`Error closing ${tier} storage provider:`, error);
}
}
this.initialized = false;
console.log('Storage manager closed');
}
catch (error) {
console.error('Error closing storage manager:', error);
throw new errors_1.StorageError(errors_1.StorageErrorCode.SYSTEM_ERROR, 'Failed to close storage manager', { error });
}
}
/**
* Get storage status
*
* @returns Storage status
*/
async getStatus() {
this.ensureInitialized();
const status = {
connected: true,
healthy: true,
lastOperation: new Date(),
metrics: {
operationsPerSecond: this.metrics.operationsPerSecond,
averageLatency: this.metrics.averageLatency,
errorRate: this.metrics.errorRate,
storageUsed: 0,
storageAvailable: 0
},
queryPerformance: {
hot: {
averageQueryTime: 0,
queriesPerSecond: 0
},
warm: {
averageQueryTime: 0,
queriesPerSecond: 0
},
cold: {
averageQueryTime: 0,
queriesPerSecond: 0
},
pendingItems: 0,
averageBatchSize: 0,
totalBatches: 0,
failedBatches: 0,
itemsRemoved: 0,
bytesFreed: 0,
duration: 0,
hotStorageUsage: 0,
warmStorageUsage: 0,
coldStorageUsage: 0,
totalStorageUsage: 0,
hotStorageCapacity: 0,
warmStorageCapacity: 0,
coldStorageCapacity: 0,
totalStorageCapacity: 0,
operationsPerSecond: this.metrics.operationsPerSecond,
averageLatency: this.metrics.averageLatency,
errorRate: this.metrics.errorRate,
queryPerformance: {
hot: {
averageQueryTime: 0,
queriesPerSecond: 0
},
warm: {
averageQueryTime: 0,
queriesPerSecond: 0
},
cold: {
averageQueryTime: 0,
queriesPerSecond: 0
},
overall: types_1.HealthStatus.HEALTHY,
providers: {
hot: types_1.HealthStatus.HEALTHY,
warm: types_1.HealthStatus.HEALTHY,
cold: types_1.HealthStatus.HEALTHY
},
issues: {
severity: types_1.AlertSeverity.INFO,
message: 'Storage manager status',
component: 'storage-manager',
timestamp: new Date()
},
circuitBreaker: {
state: {},
failures: {},
lastFailure: {}
},
retry: {
retryCount: {},
successCount: {},
failureCount: {},
averageDelay: {}
},
fallback: {
fallbackCount: {},
successCount: {},
failureCount: {}
},
timeout: {
timeoutCount: {},
averageDuration: {}
},
bulkhead: {
concurrentOperations: {},
queueSize: {},
rejectionCount: {}
},
rateLimit: {
limitedCount: {},
bypassCount: {},
waitTimeMs: {},
avgTokensPerSecond: {},
currentTokens: {}
}
}
}
};
// Check provider status
for (const [tier, provider] of Object.entries(this.providers)) {
try {
const providerStatus = await provider.getStatus();
if (!providerStatus.connected || !providerStatus.healthy) {
status.healthy = false;
status.error = `${tier} provider is unhealthy: ${providerStatus.error}`;
}
}
catch (error) {
status.healthy = false;
status.error = `Failed to get ${tier} provider status: ${error}`;
}
}
return status;
}
/**
* Get provider for tier
*
* @param tier - Storage tier
* @returns Storage provider
* @throws StorageError if provider not found
*/
getProvider(tier) {
const provider = this.providers[tier];
if (!provider) {
throw new errors_1.StorageError(errors_1.StorageErrorCode.INVALID_CONFIG, `No provider configured for ${tier} tier`, { tier });
}
return provider;
}
/**
* Ensure storage manager is initialized
*
* @throws StorageError if not initialized
*/
ensureInitialized() {
if (!this.initialized) {
throw new errors_1.StorageError(errors_1.StorageErrorCode.SYSTEM_ERROR, 'Storage manager not initialized', {});
}
}
/**
* Update metrics
*
* @param latency - Operation latency in milliseconds
* @param isError - Whether operation resulted in error
*/
updateMetrics(latency, isError = false) {
this.metrics.operationCount++;
this.metrics.totalLatency += latency;
if (isError) {
this.metrics.errorCount++;
}
this.metrics.averageLatency = this.metrics.totalLatency / this.metrics.operationCount;
this.metrics.errorRate = this.metrics.errorCount / this.metrics.operationCount;
}
/**
* Start metrics collection
*/
startMetricsCollection() {
this.metricsInterval = setInterval(() => {
// Calculate operations per second
if (this.options.metricsInterval) {
this.metrics.operationsPerSecond = this.metrics.operationCount / (this.options.metricsInterval / 1000);
}
// Reset counters
this.metrics.operationCount = 0;
this.metrics.errorCount = 0;
this.metrics.totalLatency = 0;
}, this.options.metricsInterval);
}
}
exports.StorageManagerImpl = StorageManagerImpl;
/**
* Create a new storage manager
*
* @param config - Storage configuration
* @param options - Storage manager options
* @returns Storage manager
*/
function createStorageManager(config, options) {
// Config validation can be added here if needed
return new StorageManagerImpl(config, options);
}
exports.createStorageManager = createStorageManager;
//# sourceMappingURL=manager.js.map