digitaltwin-core
Version:
Minimalist framework to collect and handle data in a Digital Twin project
790 lines • 30.8 kB
JavaScript
import express from 'ultimate-express';
import multer from 'multer';
import fs from 'fs/promises';
import cors from 'cors';
import { initializeComponents, initializeAssetsManagers } from './initializer.js';
import { UserService } from '../auth/user_service.js';
import { exposeEndpoints } from './endpoints.js';
import { scheduleComponents } from './scheduler.js';
import { LogLevel } from '../utils/logger.js';
import { QueueManager } from './queue_manager.js';
import { UploadProcessor } from './upload_processor.js';
import { isAsyncUploadable } from '../components/async_upload.js';
/**
* Digital Twin Engine - Core orchestrator for collectors, harvesters, and handlers
*
* The engine manages the lifecycle of all components, sets up queues for processing,
* exposes HTTP endpoints, and handles the overall coordination of the digital twin system.
*
* @class DigitalTwinEngine
* @example
* ```TypeScript
* import { DigitalTwinEngine } from './digital_twin_engine.js'
* import { StorageServiceFactory } from '../storage/storage_factory.js'
* import { KnexDatabaseAdapter } from '../database/adapters/knex_database_adapter.js'
*
* const storage = StorageServiceFactory.create()
* const database = new KnexDatabaseAdapter({ client: 'sqlite3', connection: ':memory:' }, storage)
*
* const engine = new DigitalTwinEngine({
* storage,
* database,
* collectors: [myCollector],
* server: { port: 3000 }
* })
*
* await engine.start()
* ```
*/
export class DigitalTwinEngine {
#collectors;
#harvesters;
#handlers;
#assetsManagers;
#customTableManagers;
#storage;
#database;
#app;
#router;
#options;
#queueManager;
#uploadProcessor;
#server;
#workers = [];
/** Get all active components (collectors and harvesters) */
get #activeComponents() {
return [...this.#collectors, ...this.#harvesters];
}
/** Get all components (collectors + harvesters + handlers + assetsManagers + customTableManagers) */
get #allComponents() {
return [
...this.#collectors,
...this.#harvesters,
...this.#handlers,
...this.#assetsManagers,
...this.#customTableManagers
];
}
/** Check if multi-queue mode is enabled */
get #isMultiQueueEnabled() {
return this.#options.queues?.multiQueue ?? true;
}
/**
* Creates a new Digital Twin Engine instance
*
* @param {EngineOptions} options - Configuration options for the engine
* @throws {Error} If required options (storage, database) are missing
*
* @example
* ```TypeScript
* const engine = new DigitalTwinEngine({
* storage: myStorageService,
* database: myDatabaseAdapter,
* collectors: [collector1, collector2],
* server: { port: 4000, host: 'localhost' }
* })
* ```
*/
constructor(options) {
this.#options = this.#applyDefaults(options);
this.#collectors = this.#options.collectors ?? [];
this.#harvesters = this.#options.harvesters ?? [];
this.#handlers = this.#options.handlers ?? [];
this.#assetsManagers = this.#options.assetsManagers ?? [];
this.#customTableManagers = this.#options.customTableManagers ?? [];
this.#storage = this.#options.storage;
this.#database = this.#options.database;
this.#app = express();
this.#router = express.Router();
this.#queueManager = this.#createQueueManager();
this.#uploadProcessor = this.#createUploadProcessor();
}
#createUploadProcessor() {
// Only create upload processor if we have a queue manager (which means Redis is available)
if (!this.#queueManager) {
return null;
}
return new UploadProcessor(this.#storage, this.#database);
}
#applyDefaults(options) {
return {
collectors: [],
harvesters: [],
handlers: [],
assetsManagers: [],
customTableManagers: [],
server: {
port: 3000,
host: '0.0.0.0',
...options.server
},
queues: {
multiQueue: true,
workers: {
collectors: 1,
harvesters: 1,
...options.queues?.workers
},
...options.queues
},
logging: {
level: LogLevel.INFO,
format: 'text',
...options.logging
},
dryRun: false,
...options
};
}
#createQueueManager() {
// Create queue manager if we have collectors, harvesters, OR assets managers that may need async uploads
const hasActiveComponents = this.#collectors.length > 0 || this.#harvesters.length > 0;
const hasAssetsManagers = this.#assetsManagers.length > 0;
if (!hasActiveComponents && !hasAssetsManagers) {
return null;
}
return new QueueManager({
redis: this.#options.redis,
collectorWorkers: this.#options.queues?.workers?.collectors,
harvesterWorkers: this.#options.queues?.workers?.harvesters,
queueOptions: this.#options.queues?.options
});
}
/**
* Initialize store managers and create their database tables
* @private
*/
async #initializeCustomTableManagers() {
for (const customTableManager of this.#customTableManagers) {
// Inject dependencies
customTableManager.setDependencies(this.#database);
// Initialize the table with custom columns
await customTableManager.initializeTable();
}
}
/**
* Ensure temporary upload directory exists
* @private
*/
async #ensureTempUploadDir() {
const tempDir = process.env.TEMP_UPLOAD_DIR || '/tmp/digitaltwin-uploads';
try {
await fs.mkdir(tempDir, { recursive: true });
}
catch (error) {
throw new Error(`Failed to create temp upload directory ${tempDir}: ${error}`);
}
}
/**
* Setup monitoring endpoints for queue statistics and health checks
* @private
*/
#setupMonitoringEndpoints() {
// Health check endpoint
this.#router.get('/api/health', async (req, res) => {
const health = {
status: 'ok',
timestamp: new Date().toISOString(),
uptime: process.uptime(),
components: {
collectors: this.#collectors.length,
harvesters: this.#harvesters.length,
handlers: this.#handlers.length,
assetsManagers: this.#assetsManagers.length,
customTableManagers: this.#customTableManagers.length
}
};
res.json(health);
});
// Queue statistics endpoint
this.#router.get('/api/queues/stats', async (req, res) => {
if (this.#queueManager) {
const stats = await this.#queueManager.getQueueStats();
res.json(stats);
}
else {
res.json({
collectors: { status: 'No collectors configured' },
harvesters: { status: 'No harvesters configured' }
});
}
});
}
/**
* Starts the Digital Twin Engine
*
* This method:
* 1. Initializes all registered components (collectors, harvesters, handlers)
* 2. Set up HTTP endpoints for component access
* 3. Configures and starts background job queues
* 4. Starts the HTTP server
* 5. Exposes queue monitoring endpoints
*
* @async
* @returns {Promise<void>}
*
* @example
* ```TypeScript
* await engine.start()
* console.log('Engine is running!')
* ```
*/
async start() {
const isDryRun = this.#options.dryRun ?? false;
if (isDryRun) {
// In dry run, just validate everything without creating tables
const validationResult = await this.validateConfiguration();
if (!validationResult.valid) {
throw new Error(`Validation failed:\n${validationResult.engineErrors.join('\n')}`);
}
return;
}
// Normal startup - initialize user management tables first
const userService = new UserService(this.#database);
await userService.initializeTables();
// Get autoMigration setting (default: true)
const autoMigration = this.#options.autoMigration ?? true;
// Initialize components and create tables if needed
await initializeComponents(this.#activeComponents, this.#database, this.#storage, autoMigration);
// Initialize assets managers and create their tables if needed
await initializeAssetsManagers(this.#assetsManagers, this.#database, this.#storage, autoMigration);
// Initialize store managers and create their tables if needed
await this.#initializeCustomTableManagers();
// Initialize handlers (inject dependencies if needed)
for (const handler of this.#handlers) {
if ('setDependencies' in handler && typeof handler.setDependencies === 'function') {
handler.setDependencies(this.#database, this.#storage);
}
// If it's a GlobalAssetsHandler, inject the AssetsManager instances
if ('setAssetsManagers' in handler && typeof handler.setAssetsManagers === 'function') {
handler.setAssetsManagers(this.#assetsManagers);
}
}
// Inject upload queue to components that support async uploads
if (this.#queueManager) {
const allManagers = [...this.#assetsManagers];
for (const manager of allManagers) {
if (isAsyncUploadable(manager)) {
manager.setUploadQueue(this.#queueManager.uploadQueue);
}
}
}
// Start upload processor worker (for async file processing)
// Uses same Redis config as QueueManager (defaults to localhost:6379 if not specified)
if (this.#uploadProcessor) {
const redisConfig = this.#options.redis || {
host: 'localhost',
port: 6379,
maxRetriesPerRequest: null
};
this.#uploadProcessor.start(redisConfig);
}
await exposeEndpoints(this.#router, this.#allComponents);
// Setup component scheduling with queue manager (only if we have active components)
if (this.#activeComponents.length > 0 && this.#queueManager) {
this.#workers = await scheduleComponents(this.#activeComponents, this.#queueManager, this.#isMultiQueueEnabled);
}
this.#setupMonitoringEndpoints();
// Ensure temporary upload directory exists
await this.#ensureTempUploadDir();
// Enable CORS for cross-origin requests from frontend applications
this.#app.use(cors({
origin: process.env.CORS_ORIGIN || true, // Allow all origins by default, configure in production
methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
allowedHeaders: ['Content-Type', 'Authorization'],
credentials: true // Allow cookies/credentials
}));
// Configure Express middlewares for body parsing - no limits for large files
this.#app.use(express.json({ limit: '10gb' }));
this.#app.use(express.urlencoded({ extended: true, limit: '10gb' }));
// Add multipart/form-data support for file uploads with disk storage for large files
const upload = multer({
storage: multer.diskStorage({
destination: (req, file, cb) => {
// Use temporary directory, will be cleaned up after processing
const tempDir = process.env.TEMP_UPLOAD_DIR || '/tmp/digitaltwin-uploads';
cb(null, tempDir);
},
filename: (req, file, cb) => {
// Generate unique filename to avoid conflicts
const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
cb(null, file.fieldname + '-' + uniqueSuffix + '-' + file.originalname);
}
}),
limits: {
// Remove file size limit to allow large files (10GB+)
files: 1, // Only one file per request for safety
parts: 10, // Limit form parts
headerPairs: 2000 // Limit header pairs
}
});
this.#app.use(upload.single('file'));
this.#app.use(this.#router);
const { port, host = '0.0.0.0' } = this.#options.server ?? {
port: 3000,
host: '0.0.0.0'
};
// Wait for server to be ready
await new Promise(resolve => {
this.#server = this.#app.listen(port, host, () => {
resolve();
});
});
// Set server timeouts for large file uploads (10 minutes)
if (this.#server) {
this.#server.timeout = 600000; // 10 minutes for request processing
this.#server.keepAliveTimeout = 620000; // Slightly longer than timeout
this.#server.headersTimeout = 621000; // Slightly longer than keepAliveTimeout
}
}
/**
* Get the server port
*
* @returns {number | undefined} The server port or undefined if not started
*
* @example
* ```TypeScript
* const port = engine.getPort()
* console.log(`Server running on port ${port}`)
* ```
*/
getPort() {
if (!this.#server)
return undefined;
try {
const address = this.#server.address();
if (typeof address === 'object' && address !== null && 'port' in address) {
return address.port;
}
}
catch {
// If address() fails, return the configured port
return this.#options.server?.port;
}
return this.#options.server?.port;
}
/**
* Stops the Digital Twin Engine gracefully
*
* This method:
* 1. Closes HTTP server
* 2. Stops background workers
* 3. Closes all queue connections
* 4. Closes database connections
* 5. Clean up resources
*
* @async
* @returns {Promise<void>}
*
* @example
* ```TypeScript
* await engine.stop()
* console.log('Engine stopped gracefully')
* ```
*/
async stop() {
const errors = [];
// 1. Close HTTP server first
if (this.#server) {
const server = this.#server;
try {
await new Promise((resolve, reject) => {
server.close(err => {
if (err)
reject(err);
else
resolve();
});
});
}
catch (error) {
errors.push(new Error(`Server close error: ${error instanceof Error ? error.message : String(error)}`));
}
}
// 2. Close all workers with extended timeout and force close
if (this.#workers.length > 0) {
await Promise.all(this.#workers.map(async (worker) => {
try {
await Promise.race([
worker.close(),
new Promise((_, reject) => setTimeout(() => reject(new Error('Worker close timeout')), 5000))
]);
}
catch {
// Force close if timeout or error
try {
await worker.disconnect();
}
catch (disconnectError) {
errors.push(new Error(`Worker force close error: ${disconnectError instanceof Error ? disconnectError.message : String(disconnectError)}`));
}
}
}));
}
// 3. Stop upload processor worker
if (this.#uploadProcessor) {
try {
await this.#uploadProcessor.stop();
}
catch (error) {
errors.push(new Error(`Upload processor close error: ${error instanceof Error ? error.message : String(error)}`));
}
}
// 4. Close queue connections (only if we have a queue manager)
if (this.#queueManager) {
try {
await this.#queueManager.close();
}
catch (error) {
errors.push(new Error(`Queue manager close error: ${error instanceof Error ? error.message : String(error)}`));
}
}
// 5. Close database connections
try {
await this.#database.close();
}
catch (error) {
errors.push(new Error(`Database close error: ${error instanceof Error ? error.message : String(error)}`));
}
if (errors.length > 0 && process.env.NODE_ENV !== 'test') {
console.warn('[DigitalTwin] Stopped with warnings:', errors.map(e => e.message).join(', '));
}
}
/**
* Validate the engine configuration and all components
*
* This method checks that all components are properly configured and can be initialized
* without actually creating tables or starting the server.
*
* @returns {Promise<ValidationResult>} Comprehensive validation results
*
* @example
* ```typescript
* const result = await engine.validateConfiguration()
* if (!result.valid) {
* console.error('Validation errors:', result.engineErrors)
* }
* ```
*/
async validateConfiguration() {
const componentResults = [];
const engineErrors = [];
// Validate collectors
for (const collector of this.#collectors) {
componentResults.push(await this.#validateComponent(collector, 'collector'));
}
// Validate harvesters
for (const harvester of this.#harvesters) {
componentResults.push(await this.#validateComponent(harvester, 'harvester'));
}
// Validate handlers
for (const handler of this.#handlers) {
componentResults.push(await this.#validateComponent(handler, 'handler'));
}
// Validate assets managers
for (const assetsManager of this.#assetsManagers) {
componentResults.push(await this.#validateComponent(assetsManager, 'assets_manager'));
}
// Validate store managers
for (const customTableManager of this.#customTableManagers) {
componentResults.push(await this.#validateComponent(customTableManager, 'custom_table_manager'));
}
// Validate engine-level configuration
try {
if (!this.#storage) {
engineErrors.push('Storage service is required');
}
if (!this.#database) {
engineErrors.push('Database adapter is required');
}
// Test storage connection
if (this.#storage && typeof this.#storage.save === 'function') {
// Storage validation passed
}
else {
engineErrors.push('Storage service does not implement required methods');
}
// Test database connection
if (this.#database && typeof this.#database.save === 'function') {
// Database validation passed
}
else {
engineErrors.push('Database adapter does not implement required methods');
}
}
catch (error) {
engineErrors.push(`Engine configuration error: ${error instanceof Error ? error.message : String(error)}`);
}
// Calculate summary
const validComponents = componentResults.filter(c => c.valid).length;
const totalWarnings = componentResults.reduce((acc, c) => acc + c.warnings.length, 0);
const result = {
valid: componentResults.every(c => c.valid) && engineErrors.length === 0,
components: componentResults,
engineErrors,
summary: {
total: componentResults.length,
valid: validComponents,
invalid: componentResults.length - validComponents,
warnings: totalWarnings
}
};
return result;
}
/**
* Test all components by running their core methods without persistence
*
* @returns {Promise<ComponentValidationResult[]>} Test results for each component
*
* @example
* ```typescript
* const results = await engine.testComponents()
* results.forEach(result => {
* console.log(`${result.name}: ${result.valid ? '✅' : '❌'}`)
* })
* ```
*/
async testComponents() {
const results = [];
// Test collectors
for (const collector of this.#collectors) {
const result = await this.#testCollector(collector);
results.push(result);
}
// Test harvesters
for (const harvester of this.#harvesters) {
const result = await this.#testHarvester(harvester);
results.push(result);
}
// Test handlers
for (const handler of this.#handlers) {
const result = await this.#testHandler(handler);
results.push(result);
}
// Test assets managers
for (const assetsManager of this.#assetsManagers) {
const result = await this.#testAssetsManager(assetsManager);
results.push(result);
}
return results;
}
/**
* Validate a single component
*/
async #validateComponent(component, type) {
const errors = [];
const warnings = [];
try {
// Check if component has required methods
if (typeof component.getConfiguration !== 'function') {
errors.push('Component must implement getConfiguration() method');
}
const config = component.getConfiguration();
// Validate configuration
if (!config.name) {
errors.push('Component configuration must have a name');
}
if (!config.description) {
warnings.push('Component configuration should have a description');
}
// Type-specific validation
if (type === 'collector' || type === 'harvester') {
const activeComponent = component;
if (typeof activeComponent.setDependencies !== 'function') {
errors.push('Active components must implement setDependencies() method');
}
}
if (type === 'collector') {
const collector = component;
if (typeof collector.collect !== 'function') {
errors.push('Collector must implement collect() method');
}
if (typeof collector.getSchedule !== 'function') {
errors.push('Collector must implement getSchedule() method');
}
}
if (type === 'harvester') {
const harvester = component;
if (typeof harvester.harvest !== 'function') {
errors.push('Harvester must implement harvest() method');
}
}
if (type === 'assets_manager') {
const assetsManager = component;
if (typeof assetsManager.uploadAsset !== 'function') {
errors.push('AssetsManager must implement uploadAsset() method');
}
if (typeof assetsManager.getAllAssets !== 'function') {
errors.push('AssetsManager must implement getAllAssets() method');
}
}
if (type === 'custom_table_manager') {
const customTableManager = component;
if (typeof customTableManager.setDependencies !== 'function') {
errors.push('CustomTableManager must implement setDependencies() method');
}
if (typeof customTableManager.initializeTable !== 'function') {
errors.push('CustomTableManager must implement initializeTable() method');
}
// Validate store configuration
const config = customTableManager.getConfiguration();
if (typeof config !== 'object' || config === null) {
errors.push('CustomTableManager must return a valid configuration object');
}
else {
if (!config.columns || typeof config.columns !== 'object') {
errors.push('CustomTableManager configuration must define columns');
}
else {
// Validate columns definition
const columnCount = Object.keys(config.columns).length;
if (columnCount === 0) {
warnings.push('CustomTableManager has no custom columns defined');
}
// Validate column names and types
for (const [columnName, columnType] of Object.entries(config.columns)) {
if (!columnName || typeof columnName !== 'string') {
errors.push('Column names must be non-empty strings');
}
if (!columnType || typeof columnType !== 'string') {
errors.push(`Column '${columnName}' must have a valid SQL type`);
}
}
}
}
}
}
catch (error) {
errors.push(`Validation error: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: component.getConfiguration?.()?.name || 'unknown',
type,
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test a collector by running its collect method
*/
async #testCollector(collector) {
const errors = [];
const warnings = [];
const config = collector.getConfiguration();
try {
// Test the collect method
const result = await collector.collect();
if (!Buffer.isBuffer(result)) {
errors.push('collect() method must return a Buffer');
}
if (result.length === 0) {
warnings.push('collect() method returned empty buffer');
}
}
catch (error) {
errors.push(`collect() method failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'collector',
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test a harvester (more complex as it needs mock data)
*/
async #testHarvester(harvester) {
const errors = [];
const warnings = [];
const config = harvester.getConfiguration();
try {
// Create mock data for testing
const mockData = {
id: 1,
name: 'test',
date: new Date(),
contentType: 'application/json',
url: 'test://url',
data: async () => Buffer.from('{"test": true}')
};
// Test the harvest method
const result = await harvester.harvest(mockData, {});
if (!Buffer.isBuffer(result)) {
errors.push('harvest() method must return a Buffer');
}
}
catch (error) {
errors.push(`harvest() method failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'harvester',
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test a handler
*/
async #testHandler(handler) {
const errors = [];
const warnings = [];
const config = handler.getConfiguration();
try {
// Handlers are mostly validated through their endpoint configuration
if (typeof handler.getEndpoints === 'function') {
const endpoints = handler.getEndpoints();
if (!Array.isArray(endpoints)) {
errors.push('getEndpoints() must return an array');
}
}
}
catch (error) {
errors.push(`Handler test failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'handler',
valid: errors.length === 0,
errors,
warnings
};
}
/**
* Test an assets manager
*/
async #testAssetsManager(assetsManager) {
const errors = [];
const warnings = [];
const config = assetsManager.getConfiguration();
try {
// Test configuration
if (!config.contentType) {
errors.push('AssetsManager configuration must have a contentType');
}
// In dry run mode, we can't test actual upload/download without dependencies
// Just validate that the methods exist and are callable
if (typeof assetsManager.getEndpoints === 'function') {
const endpoints = assetsManager.getEndpoints();
if (!Array.isArray(endpoints)) {
errors.push('getEndpoints() must return an array');
}
}
}
catch (error) {
errors.push(`AssetsManager test failed: ${error instanceof Error ? error.message : String(error)}`);
}
return {
name: config.name,
type: 'assets_manager',
valid: errors.length === 0,
errors,
warnings
};
}
}
//# sourceMappingURL=digital_twin_engine.js.map