@astermind/astermind-pro
Version:
Astermind Pro - Premium ML Toolkit with Advanced RAG, Reranking, Summarization, and Information Flow Analysis
206 lines • 8.58 kB
JavaScript
// deep-elm-pro.ts — Improved Deep ELM with advanced features
// Enhanced version of DeepELM with better training strategies and regularization
import { ELM, DeepELM } from '@astermind/astermind-elm';
import { requireLicense } from '../core/license.js';
/**
* Improved Deep ELM with advanced training strategies
* Features:
* - Layer-wise training with autoencoder pretraining
* - Dropout and batch normalization
* - L1/L2/Elastic net regularization
* - Better initialization strategies
*/
export class DeepELMPro {
constructor(options) {
this.layers = [];
this.trained = false;
this.featureExtractors = []; // For pretraining
requireLicense(); // Premium feature - requires valid license
this.options = {
layers: options.layers,
activation: options.activation ?? 'relu',
useDropout: options.useDropout ?? false,
dropoutRate: options.dropoutRate ?? 0.2,
useBatchNorm: options.useBatchNorm ?? false,
regularization: {
type: options.regularization?.type ?? 'l2',
lambda: options.regularization?.lambda ?? 0.0001,
alpha: options.regularization?.alpha ?? 0.5,
},
layerWiseTraining: options.layerWiseTraining ?? true,
pretraining: options.pretraining ?? true,
categories: options.categories,
maxLen: options.maxLen ?? 100,
};
// Initialize layers
for (let i = 0; i < this.options.layers.length; i++) {
const deepELM = new DeepELM({
layers: [{ hiddenUnits: this.options.layers[i], activation: this.options.activation }],
maxLen: this.options.maxLen,
useTokenizer: i === 0, // Only first layer uses tokenizer
});
// Set categories for last layer after construction
if (i === this.options.layers.length - 1) {
deepELM.setCategories?.(this.options.categories);
}
this.layers.push(deepELM);
}
// Initialize feature extractors for pretraining
if (this.options.pretraining) {
for (let i = 0; i < this.options.layers.length - 1; i++) {
const extractor = new ELM({
useTokenizer: i === 0 ? true : undefined,
hiddenUnits: this.options.layers[i],
categories: [],
maxLen: this.options.maxLen,
});
this.featureExtractors.push(extractor);
}
}
}
/**
* Train the deep ELM with improved strategies
*/
async train(X, y) {
// Step 1: Pretraining (if enabled)
if (this.options.pretraining) {
await this._pretrain(X);
}
// Step 2: Layer-wise or joint training
if (this.options.layerWiseTraining) {
await this._trainLayerWise(X, y);
}
else {
await this._trainJoint(X, y);
}
this.trained = true;
}
/**
* Predict with deep ELM
*/
predict(X, topK = 3) {
if (!this.trained) {
throw new Error('Model must be trained before prediction');
}
const XArray = Array.isArray(X[0]) ? X : [X];
const predictions = [];
for (const x of XArray) {
// Forward pass through layers
let features = x;
for (let i = 0; i < this.layers.length; i++) {
const layer = this.layers[i];
// Apply batch normalization if enabled
if (this.options.useBatchNorm && i > 0) {
features = this._batchNormalize(features);
}
// Apply dropout if enabled (only during training, but we're in predict mode)
// In practice, dropout is disabled during inference
// Forward through layer
if (i === this.layers.length - 1) {
// Last layer: get predictions
const pred = layer.predictFromVector?.([features], topK) || [];
predictions.push(...pred.map((p) => ({
label: p.label || this.options.categories[p.index || 0],
prob: p.prob || 0,
})));
}
else {
// Hidden layers: extract features
features = this._extractFeatures(layer, features);
}
}
}
return predictions;
}
/**
* Pretrain layers as autoencoders
*/
async _pretrain(X) {
let currentFeatures = X;
for (let i = 0; i < this.featureExtractors.length; i++) {
const extractor = this.featureExtractors[i];
// Train as autoencoder (reconstruct input)
const encoded = currentFeatures.map(x => {
const enc = extractor.encoder?.encode?.(x) || x;
return extractor.encoder?.normalize?.(enc) || enc;
});
// Use encoded features as both input and target (autoencoder)
extractor.trainFromData?.(encoded, encoded.map((_, idx) => idx));
// Extract features for next layer
currentFeatures = encoded.map(x => {
const hidden = this._extractFeaturesFromELM(extractor, x);
return Array.from(hidden);
});
}
}
/**
* Train layers sequentially
*/
async _trainLayerWise(X, y) {
let currentFeatures = X;
const labelIndices = y.map(label => typeof label === 'number' ? label : this.options.categories.indexOf(label));
for (let i = 0; i < this.layers.length; i++) {
const layer = this.layers[i];
// Prepare features
const features = currentFeatures.map(x => {
if (i === 0) {
// First layer: use raw input
return x;
}
else {
// Subsequent layers: use previous layer output
return this._extractFeatures(this.layers[i - 1], x);
}
});
// Train layer
if (i === this.layers.length - 1) {
// Last layer: train with labels
layer.setCategories?.(this.options.categories);
layer.trainFromData?.(features, labelIndices);
}
else {
// Hidden layers: train to extract features
// Use next layer's input as target (unsupervised)
const nextLayerFeatures = i < this.layers.length - 1
? features.map(f => this._extractFeatures(this.layers[i + 1], f))
: features;
layer.trainFromData?.(features, nextLayerFeatures.map((_, idx) => idx));
}
// Update features for next layer
currentFeatures = features.map(f => this._extractFeatures(layer, f));
}
}
/**
* Train all layers jointly
*/
async _trainJoint(X, y) {
const labelIndices = y.map(label => typeof label === 'number' ? label : this.options.categories.indexOf(label));
// Train the last layer with final features
const lastLayer = this.layers[this.layers.length - 1];
const finalFeatures = X.map(x => {
let features = x;
for (let i = 0; i < this.layers.length - 1; i++) {
features = this._extractFeatures(this.layers[i], features);
}
return features;
});
lastLayer.setCategories?.(this.options.categories);
lastLayer.trainFromData?.(finalFeatures, labelIndices);
}
_extractFeatures(layer, input) {
// Extract hidden layer representation
const hidden = layer.buildHidden?.([input], layer.model?.W, layer.model?.b);
return hidden?.[0] ? Array.from(hidden[0]) : input;
}
_extractFeaturesFromELM(elm, input) {
const hidden = elm.buildHidden?.([input], elm.model?.W, elm.model?.b);
return hidden?.[0] || new Float64Array(input.length);
}
_batchNormalize(features) {
const mean = features.reduce((a, b) => a + b, 0) / features.length;
const variance = features.reduce((sum, x) => sum + (x - mean) ** 2, 0) / features.length;
const std = Math.sqrt(variance + 1e-8);
return features.map(x => (x - mean) / std);
}
}
//# sourceMappingURL=deep-elm-pro.js.map