digitaltwin-core
Version:
Minimalist framework to collect and handle data in a Digital Twin project
257 lines • 9.29 kB
JavaScript
/**
* Utility class for handling source range parsing
*/
class SourceRangeParser {
static { this.ZERO_DATE = new Date('1970-01-01T00:00:00Z'); }
static parseSourceRange(latestDate, sourceRange) {
if (!sourceRange) {
return { startDate: latestDate, limit: 1 };
}
// If it's a number or numeric string (limit mode)
if (typeof sourceRange === 'number' || /^\d+$/.test(sourceRange)) {
return { startDate: latestDate, limit: Number(sourceRange) };
}
const sourceRangeStr = sourceRange.toString();
let value;
let unit;
// Parse time-based ranges
if (sourceRangeStr.includes('d')) {
value = parseInt(sourceRangeStr.replace('d', ''));
unit = 'days';
}
else if (sourceRangeStr.includes('h')) {
value = parseInt(sourceRangeStr.replace('h', ''));
unit = 'hours';
}
else if (sourceRangeStr.includes('m')) {
value = parseInt(sourceRangeStr.replace('m', ''));
unit = 'minutes';
}
else if (sourceRangeStr.includes('s')) {
value = parseInt(sourceRangeStr.replace('s', ''));
unit = 'seconds';
}
else {
throw new Error(`Invalid source range format: ${sourceRange}`);
}
// For time-based ranges, start from latestDate and go forward
const startDate = latestDate;
const endDate = this.addTime(startDate, value, unit);
return { startDate, endDate };
}
static addTime(date, value, unit) {
const result = new Date(date);
switch (unit) {
case 'days':
result.setDate(result.getDate() + value);
break;
case 'hours':
result.setHours(result.getHours() + value);
break;
case 'minutes':
result.setMinutes(result.getMinutes() + value);
break;
case 'seconds':
result.setSeconds(result.getSeconds() + value);
break;
}
return result;
}
static get zeroDate() {
return new Date(this.ZERO_DATE);
}
}
/**
* Abstract Harvester class for processing collected data
*/
export class Harvester {
setDependencies(db, storage) {
this.db = db;
this.storage = storage;
}
/**
* Final configuration with defaults applied
* This is what the engine/scheduler use
*/
getConfiguration() {
if (this._configCache) {
return this._configCache;
}
const userConfig = this.getUserConfiguration();
// Apply defaults first, then user config
const defaults = {
triggerMode: 'on-source',
source_range: 1,
multiple_results: false,
source_range_min: false,
debounceMs: 1000,
dependencies: [],
dependenciesLimit: []
};
this._configCache = {
...defaults,
...userConfig
};
return this._configCache;
}
/**
* Override to make schedule optional based on trigger mode
*/
getSchedule() {
const config = this.getConfiguration();
if (config.triggerMode === 'on-source') {
return '';
}
// Default to every minute instead of every second
return '0 * * * * *';
}
/**
* Main execution method - NOUVELLE VERSION SIMPLIFIÉE
*/
async run() {
const config = this.getConfiguration();
if (!config.source) {
throw new Error(`Harvester ${config.name} must specify a source component`);
}
// Get the latest harvested date
const latestHarvestedRecord = await this.db.getLatestByName(config.name);
// Calculate the starting point for harvesting
let latestDate;
if (!latestHarvestedRecord) {
// First run - get first source record and start from one second before
const firstSourceRecord = await this.db.getFirstByName(config.source);
if (!firstSourceRecord) {
return false;
}
latestDate = new Date(firstSourceRecord.date.getTime() - 1000);
}
else {
latestDate = latestHarvestedRecord.date;
}
// Parse source range
const { startDate, endDate, limit } = SourceRangeParser.parseSourceRange(latestDate, config.source_range);
// Get source data based on range
const sourceData = await this.getSourceData(config.source, startDate, endDate, limit);
if (!sourceData || sourceData.length === 0) {
return false;
}
// Check if we have enough data (strict mode)
if (limit && config.source_range_min && sourceData.length < limit) {
return false;
}
// Calculate storage date
const storageDate = endDate || sourceData[sourceData.length - 1].date;
// Prepare source data for harvesting
const sourceForHarvesting = limit === 1 && !endDate ? sourceData[0] : sourceData;
// Get dependencies data
const dependenciesData = await this.getDependenciesData(config.dependencies || [], config.dependenciesLimit || [], storageDate);
// Execute harvesting
const result = await this.harvest(sourceForHarvesting, dependenciesData);
// Store results
await this.storeResults(config, result, sourceData, storageDate);
return true;
}
/**
* Get source data within the specified range
*/
async getSourceData(sourceName, startDate, endDate, limit) {
let sourceData;
if (endDate) {
// Time-based range: get records between startDate and endDate
sourceData = await this.db.getByDateRange(sourceName, startDate, endDate, limit);
}
else if (limit) {
// Count-based: get records after startDate with limit
sourceData = await this.db.getAfterDate(sourceName, startDate, limit);
}
else {
// Default: get latest record after startDate
sourceData = await this.db.getAfterDate(sourceName, startDate, 1);
}
return sourceData;
}
/**
* Get data from dependent components
*/
async getDependenciesData(dependencies, dependenciesLimit, storageDate) {
const dependenciesData = {};
for (let i = 0; i < dependencies.length; i++) {
const dependency = dependencies[i];
const limit = dependenciesLimit[i] || 1;
if (limit === 1) {
// Get single latest record before storage date
const dependencyRecord = await this.db.getLatestBefore(dependency, storageDate);
dependenciesData[dependency] = dependencyRecord || null;
}
else {
// Get multiple latest records before storage date
const dependencyRecords = await this.db.getLatestRecordsBefore(dependency, storageDate, limit);
dependenciesData[dependency] = dependencyRecords.length > 0 ? dependencyRecords : null;
}
}
return dependenciesData;
}
/**
* Store harvesting results
*/
async storeResults(config, result, sourceData, storageDate) {
if (config.multiple_results && Array.isArray(result) && Array.isArray(sourceData)) {
// Store each result with its corresponding source date
for (let i = 0; i < result.length; i++) {
const item = result[i];
const source = sourceData[i];
const url = await this.storage.save(item, config.name);
await this.db.save({
name: config.name,
type: config.contentType,
url,
date: source.date
});
}
}
else {
// Store single result
const buffer = Array.isArray(result) ? result[0] : result;
const url = await this.storage.save(buffer, config.name);
await this.db.save({
name: config.name,
type: config.contentType,
url,
date: storageDate
});
}
}
/**
* HTTP endpoints
*/
getEndpoints() {
return [
{
method: 'get',
path: `/${this.getConfiguration().endpoint}`,
handler: this.retrieve.bind(this),
responseType: this.getConfiguration().contentType
}
];
}
/**
* Retrieve latest harvested data
*/
async retrieve() {
const config = this.getConfiguration();
const record = await this.db.getLatestByName(config.name);
if (!record) {
return {
status: 404,
content: 'No data available'
};
}
const blob = await record.data();
return {
status: 200,
content: blob,
headers: { 'Content-Type': record.contentType }
};
}
}
//# sourceMappingURL=harvester.js.map