digitaltwin-core
Version:
Minimalist framework to collect and handle data in a Digital Twin project
625 lines • 23.1 kB
JavaScript
import express from 'ultimate-express';
import multer from 'multer';
import { initializeComponents, initializeAssetsManagers } from './initializer.js';
import { exposeEndpoints } from './endpoints.js';
import { scheduleComponents } from './scheduler.js';
import { LogLevel } from '../utils/logger.js';
import { QueueManager } from './queue_manager.js';
/**
* Digital Twin Engine - Core orchestrator for collectors, harvesters, and handlers
*
* The engine manages the lifecycle of all components, sets up queues for processing,
* exposes HTTP endpoints, and handles the overall coordination of the digital twin system.
*
* @class DigitalTwinEngine
* @example
* ```TypeScript
* import { DigitalTwinEngine } from './digital_twin_engine.js'
* import { StorageServiceFactory } from '../storage/storage_factory.js'
* import { KnexDatabaseAdapter } from '../database/adapters/knex_database_adapter.js'
*
* const storage = StorageServiceFactory.create()
* const database = new KnexDatabaseAdapter({ client: 'sqlite3', connection: ':memory:' }, storage)
*
* const engine = new DigitalTwinEngine({
* storage,
* database,
* collectors: [myCollector],
* server: { port: 3000 }
* })
*
* await engine.start()
* ```
*/
export class DigitalTwinEngine {
#collectors;
#harvesters;
#handlers;
#assetsManagers;
#storage;
#database;
#app;
#router;
#options;
#queueManager;
#server;
#workers = [];
/** Get all active components (collectors and harvesters) */
get #activeComponents() {
return [...this.#collectors, ...this.#harvesters];
}
/** Get all components (collectors + harvesters + handlers + assetsManagers) */
get #allComponents() {
return [...this.#collectors, ...this.#harvesters, ...this.#handlers, ...this.#assetsManagers];
}
/** Check if multi-queue mode is enabled */
get #isMultiQueueEnabled() {
return this.#options.queues?.multiQueue ?? true;
}
/**
* Creates a new Digital Twin Engine instance
*
* @param {EngineOptions} options - Configuration options for the engine
* @throws {Error} If required options (storage, database) are missing
*
* @example
* ```TypeScript
* const engine = new DigitalTwinEngine({
* storage: myStorageService,
* database: myDatabaseAdapter,
* collectors: [collector1, collector2],
* server: { port: 4000, host: 'localhost' }
* })
* ```
*/
constructor(options) {
this.#options = this.#applyDefaults(options);
this.#collectors = this.#options.collectors;
this.#harvesters = this.#options.harvesters;
this.#handlers = this.#options.handlers;
this.#assetsManagers = this.#options.assetsManagers;
this.#storage = this.#options.storage;
this.#database = this.#options.database;
this.#app = express();
this.#router = express.Router();
this.#queueManager = this.#createQueueManager();
}
#applyDefaults(options) {
return {
collectors: [],
harvesters: [],
handlers: [],
assetsManagers: [],
server: {
port: 3000,
host: '0.0.0.0',
...options.server
},
queues: {
multiQueue: true,
workers: {
collectors: 1,
harvesters: 1,
...options.queues?.workers
},
...options.queues
},
logging: {
level: LogLevel.INFO,
format: 'text',
...options.logging
},
dryRun: false,
...options
};
}
#createQueueManager() {
// Only create queue manager if we have collectors or harvesters
if (this.#collectors.length === 0 && this.#harvesters.length === 0) {
return null;
}
return new QueueManager({
redis: this.#options.redis,
collectorWorkers: this.#options.queues?.workers?.collectors,
harvesterWorkers: this.#options.queues?.workers?.harvesters,
queueOptions: this.#options.queues?.options
});
}
/**
* Setup monitoring endpoints for queue statistics
* @private
*/
#setupMonitoringEndpoints() {
this.#router.get('/api/queues/stats', async (req, res) => {
if (this.#queueManager) {
const stats = await this.#queueManager.getQueueStats();
res.json(stats);
}
else {
res.json({
collectors: { status: 'No collectors configured' },
harvesters: { status: 'No harvesters configured' }
});
}
});
}
/**
* Starts the Digital Twin Engine
*
* This method:
* 1. Initializes all registered components (collectors, harvesters, handlers)
* 2. Set up HTTP endpoints for component access
* 3. Configures and starts background job queues
* 4. Starts the HTTP server
* 5. Exposes queue monitoring endpoints
*
* @async
* @returns {Promise<void>}
*
* @example
* ```TypeScript
* await engine.start()
* console.log('Engine is running!')
* ```
*/
async start() {
const isDryRun = this.#options.dryRun ?? false;
if (isDryRun) {
console.log('🧪 Starting in DRY RUN mode - no data will be persisted');
// In dry run, just validate everything without creating tables
const validationResult = await this.validateConfiguration();
if (!validationResult.valid) {
throw new Error(`Validation failed:\n${validationResult.engineErrors.join('\n')}`);
}
console.log('✅ Dry run completed successfully - all components are valid');
return;
}
// Normal startup - initialize components and create tables if needed
await initializeComponents(this.#activeComponents, this.#database, this.#storage);
// Initialize assets managers and create their tables if needed
await initializeAssetsManagers(this.#assetsManagers, this.#database, this.#storage);
// Initialize handlers (inject dependencies if needed)
for (const handler of this.#handlers) {
if ('setDependencies' in handler && typeof handler.setDependencies === 'function') {
handler.setDependencies(this.#database, this.#storage);
}
// If it's a GlobalAssetsHandler, inject the AssetsManager instances
if ('setAssetsManagers' in handler && typeof handler.setAssetsManagers === 'function') {
handler.setAssetsManagers(this.#assetsManagers);
}
}
await exposeEndpoints(this.#router, this.#allComponents);
// Setup component scheduling with queue manager (only if we have active components)
if (this.#activeComponents.length > 0 && this.#queueManager) {
this.#workers = await scheduleComponents(this.#activeComponents, this.#queueManager, this.#isMultiQueueEnabled);
}
this.#setupMonitoringEndpoints();
// Configure Express middlewares for body parsing
this.#app.use(express.json({ limit: '50mb' }));
this.#app.use(express.urlencoded({ extended: true, limit: '50mb' }));
// Add multipart/form-data support for file uploads
const upload = multer({
storage: multer.memoryStorage(),
limits: { fileSize: 50 * 1024 * 1024 } // 50MB limit
});
this.#app.use(upload.single('file'));
this.#app.use(this.#router);
const { port, host = '0.0.0.0' } = this.#options.server;
// Wait for server to be ready
await new Promise(resolve => {
this.#server = this.#app.listen(port, host, () => {
if (process.env.NODE_ENV !== 'test') {
console.log(`Digital Twin Engine started on ${host}:${port}`);
console.log(`Multi-queue mode: ${this.#isMultiQueueEnabled}`);
}
resolve();
});
});
}
/**
* Get the server port
*
* @returns {number | undefined} The server port or undefined if not started
*
* @example
* ```TypeScript
* const port = engine.getPort()
* console.log(`Server running on port ${port}`)
* ```
*/
getPort() {
if (!this.#server)
return undefined;
try {
const address = this.#server.address();
if (typeof address === 'object' && address !== null && 'port' in address) {
return address.port;
}
}
catch {
// If address() fails, return the configured port
return this.#options.server?.port;
}
return this.#options.server?.port;
}
/**
* Stops the Digital Twin Engine gracefully
*
* This method:
* 1. Closes HTTP server
* 2. Stops background workers
* 3. Closes all queue connections
* 4. Closes database connections
* 5. Clean up resources
*
* @async
* @returns {Promise<void>}
*
* @example
* ```TypeScript
* await engine.stop()
* console.log('Engine stopped gracefully')
* ```
*/
async stop() {
const errors = [];
// 1. Close HTTP server first
if (this.#server) {
try {
await new Promise((resolve, reject) => {
this.#server.close(err => {
if (err)
reject(err);
else
resolve();
});
});
}
catch (error) {
errors.push(new Error(`Server close error: ${error instanceof Error ? error.message : String(error)}`));
}
}
// 2. Close all workers with extended timeout and force close
if (this.#workers.length > 0) {
await Promise.all(this.#workers.map(async (worker) => {
try {
await Promise.race([
worker.close(),
new Promise((_, reject) => setTimeout(() => reject(new Error('Worker close timeout')), 5000))
]);
}
catch {
// Force close if timeout or error
try {
await worker.disconnect();
}
catch (disconnectError) {
errors.push(new Error(`Worker force close error: ${disconnectError instanceof Error ? disconnectError.message : String(disconnectError)}`));
}
}
}));
}
// 3. Close queue connections (only if we have a queue manager)
if (this.#queueManager) {
try {
await this.#queueManager.close();
}
catch (error) {
errors.push(new Error(`Queue manager close error: ${error instanceof Error ? error.message : String(error)}`));
}
}
// 4. Close database connections
try {
await this.#database.close();
}
catch (error) {
errors.push(new Error(`Database close error: ${error instanceof Error ? error.message : String(error)}`));
}
if (process.env.NODE_ENV !== 'test') {
if (errors.length > 0) {
console.warn('Digital Twin Engine stopped with warnings:', errors.map(e => e.message).join(', '));
}
else {
console.log('Digital Twin Engine stopped gracefully');
}
}
}
/**
* Validate the engine configuration and all components
*
* This method checks that all components are properly configured and can be initialized
* without actually creating tables or starting the server.
*
* @returns {Promise<ValidationResult>} Comprehensive validation results
*
* @example
* ```typescript
* const result = await engine.validateConfiguration()
* if (!result.valid) {
* console.error('Validation errors:', result.engineErrors)
* }
* ```
*/
async validateConfiguration() {
const componentResults = [];
const engineErrors = [];
// Validate collectors
for (const collector of this.#collectors) {
componentResults.push(await this.#validateComponent(collector, 'collector'));
}
// Validate harvesters
for (const harvester of this.#harvesters) {
componentResults.push(await this.#validateComponent(harvester, 'harvester'));
}
// Validate handlers
for (const handler of this.#handlers) {
componentResults.push(await this.#validateComponent(handler, 'handler'));
}
// Validate assets managers
for (const assetsManager of this.#assetsManagers) {
componentResults.push(await this.#validateComponent(assetsManager, 'assets_manager'));
}
// Validate engine-level configuration
try {
if (!this.#storage) {
engineErrors.push('Storage service is required');
}
if (!this.#database) {
engineErrors.push('Database adapter is required');
}
// Test storage connection
if (this.#storage && typeof this.#storage.save === 'function') {
// Storage validation passed
}
else {
engineErrors.push('Storage service does not implement required methods');
}
// Test database connection
if (this.#database && typeof this.#database.save === 'function') {
// Database validation passed
}
else {
engineErrors.push('Database adapter does not implement required methods');
}
}
catch (error) {
engineErrors.push(`Engine configuration error: ${error instanceof Error ? error.message : String(error)}`);
}
// Calculate summary
const validComponents = componentResults.filter(c => c.valid).length;
const totalWarnings = componentResults.reduce((acc, c) => acc + c.warnings.length, 0);
const result = {
valid: componentResults.every(c => c.valid) && engineErrors.length === 0,
components: componentResults,
engineErrors,
summary: {
total: componentResults.length,
valid: validComponents,
invalid: componentResults.length - validComponents,
warnings: totalWarnings
}
};
return result;
}
/**
* Test all components by running their core methods without persistence
*
* @returns {Promise<ComponentValidationResult[]>} Test results for each component
*
* @example
* ```typescript
* const results = await engine.testComponents()
* results.forEach(result => {
* console.log(`${result.name}: ${result.valid ? '✅' : '❌'}`)
* })
* ```
*/
async testComponents() {
const results = [];
// Test collectors
for (const collector of this.#collectors) {
const result = await this.#testCollector(collector);
results.push(result);
}
// Test harvesters
for (const harvester of this.#harvesters) {
const result = await this.#testHarvester(harvester);
results.push(result);
}
// Test handlers
for (const handler of this.#handlers) {
const result = await this.#testHandler(handler);
results.push(result);
}
// Test assets managers
for (const assetsManager of this.#assetsManagers) {
const result = await this.#testAssetsManager(assetsManager);
results.push(result);
}
return results;
}
/**
* Validate a single component
*/
async #validateComponent(component, type) {
const errors = [];
const warnings = [];
try {
// Check if component has required methods
if (typeof component.getConfiguration !== 'function') {
errors.push('Component must implement getConfiguration() method');
}
const config = component.getConfiguration();
// Validate configuration
if (!config.name) {
errors.push('Component configuration must have a name');
}
if (!config.description) {
warnings.push('Component configuration should have a description');
}
// Type-specific validation
if (type === 'collector' || type === 'harvester') {
const activeComponent = component;
if (typeof activeComponent.setDependencies !== 'function') {
errors.push('Active components must implement setDependencies() method');
}
}
if (type === 'collector') {
const collector = component;
if (typeof collector.collect !== 'function') {
errors.push('Collector must implement collect() method');
}
if (typeof collector.getSchedule !== 'function') {
errors.push('Collector must implement getSchedule() method');
}
}
if (type === 'harvester') {
const harvester = component;
if (typeof harvester.harvest !== 'function') {
errors.push('Harvester must implement harvest() method');
}
}
if (type === 'assets_manager') {
const assetsManager = component;
if (typeof assetsManager.uploadAsset !== 'function') {
errors.push('AssetsManager must implement uploadAsset() method');
}
if (typeof assetsManager.getAllAssets !== 'function') {
errors.push('AssetsManager must implement getAllAssets() method');
}
}
}
catch (error) {
errors.push(`Validation error: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: component.getConfiguration?.()?.name || 'unknown',
type,
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test a collector by running its collect method
*/
async #testCollector(collector) {
const errors = [];
const warnings = [];
const config = collector.getConfiguration();
try {
// Test the collect method
const result = await collector.collect();
if (!Buffer.isBuffer(result)) {
errors.push('collect() method must return a Buffer');
}
if (result.length === 0) {
warnings.push('collect() method returned empty buffer');
}
}
catch (error) {
errors.push(`collect() method failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'collector',
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test a harvester (more complex as it needs mock data)
*/
async #testHarvester(harvester) {
const errors = [];
const warnings = [];
const config = harvester.getConfiguration();
try {
// Create mock data for testing
const mockData = {
id: 1,
name: 'test',
date: new Date(),
contentType: 'application/json',
url: 'test://url',
data: async () => Buffer.from('{"test": true}')
};
// Test the harvest method
const result = await harvester.harvest(mockData, {});
if (!Buffer.isBuffer(result)) {
errors.push('harvest() method must return a Buffer');
}
}
catch (error) {
errors.push(`harvest() method failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'harvester',
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test a handler
*/
async #testHandler(handler) {
const errors = [];
const warnings = [];
const config = handler.getConfiguration();
try {
// Handlers are mostly validated through their endpoint configuration
if (typeof handler.getEndpoints === 'function') {
const endpoints = handler.getEndpoints();
if (!Array.isArray(endpoints)) {
errors.push('getEndpoints() must return an array');
}
}
}
catch (error) {
errors.push(`Handler test failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'handler',
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test an assets manager
*/
async #testAssetsManager(assetsManager) {
const errors = [];
const warnings = [];
const config = assetsManager.getConfiguration();
try {
// Test configuration
if (!config.contentType) {
errors.push('AssetsManager configuration must have a contentType');
}
// In dry run mode, we can't test actual upload/download without dependencies
// Just validate that the methods exist and are callable
if (typeof assetsManager.getEndpoints === 'function') {
const endpoints = assetsManager.getEndpoints();
if (!Array.isArray(endpoints)) {
errors.push('getEndpoints() must return an array');
}
}
}
catch (error) {
errors.push(`AssetsManager test failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'assets_manager',
valid: errors.length === 0,
errors,
warnings
};
}
}
//# sourceMappingURL=digital_twin_engine.js.map