playwright-advanced-ml-healer
Version:
Advanced AI-powered self-healing selectors for Playwright with 19+ healing types, neural networks, machine learning models, and Global DOM Learning ML Model
1,212 lines (1,210 loc) • 177 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AdvancedMLHealing = void 0;
class AdvancedMLHealing {
constructor() {
this.healingHistory = [];
this.typeMismatchLearningHistory = [];
this.neuralNetwork = {
patterns: {
'id-exact': 0.95,
'id-partial': 0.85,
'class-exact': 0.80,
'class-partial': 0.75,
'text-exact': 0.90,
'text-partial': 0.80,
'semantic-match': 0.88,
'context-match': 0.82,
'fuzzy-match': 0.78,
'anagram-match': 0.85,
'abbreviation-match': 0.87,
'nlp-match': 0.89
},
weights: {
'id-weight': 0.3,
'class-weight': 0.2,
'text-weight': 0.25,
'semantic-weight': 0.15,
'context-weight': 0.1
},
biases: {
'base-bias': 0.1,
'confidence-bias': 0.05
},
learningRate: 0.01,
activationFunction: (x) => 1 / (1 + Math.exp(-x)),
layers: [
{
neurons: 10,
weights: [],
biases: [],
activation: 'sigmoid'
}
],
trainingData: [],
backpropagation: {
momentum: 0.9,
batchSize: 32,
epochs: 100
},
deepLearning: {
hiddenLayers: [
{ neurons: 64, activation: 'relu', dropout: 0.2 },
{ neurons: 32, activation: 'relu', dropout: 0.2 },
{ neurons: 16, activation: 'sigmoid', dropout: 0.1 }
],
optimizer: 'adam',
lossFunction: 'binary_crossentropy',
regularization: { l1: 0.01, l2: 0.01 }
},
convolutional: {
filters: [
{ size: 3, channels: 16, stride: 1 },
{ size: 3, channels: 32, stride: 1 },
{ size: 2, channels: 64, stride: 2 }
],
pooling: 'max',
flatten: true
},
recurrent: {
type: 'lstm',
units: 128,
returnSequences: false,
bidirectional: true
}
};
this.advancedCachingSystem = {
memoryCache: new Map(),
get(key) {
const cached = this.memoryCache.get(key);
if (cached && Date.now() - cached.timestamp < cached.ttl) {
cached.accessCount++;
return cached.result;
}
return null;
},
set(key, result, ttl = 300000) {
this.memoryCache.set(key, {
result,
timestamp: Date.now(),
accessCount: 1,
ttl
});
},
clear() {
this.memoryCache.clear();
},
getStats() {
const entries = Array.from(this.memoryCache.values());
const totalAccess = entries.reduce((sum, entry) => sum + (entry?.accessCount || 0), 0);
return {
size: this.memoryCache.size,
hitRate: entries.length > 0 ? totalAccess / entries.length : 0,
avgAccessCount: entries.length > 0 ? totalAccess / entries.length : 0
};
}
};
this.analyticsSystem = {
metrics: {
totalRequests: 0,
successfulHeals: 0,
failedHeals: 0,
averageResponseTime: 0,
averageConfidence: 0,
strategySuccessRates: new Map(),
patternTypeDistribution: new Map(),
elementTypeDistribution: new Map()
},
recordRequest(selector, context, result, responseTime) {
this.metrics.totalRequests++;
if (result) {
this.metrics.successfulHeals++;
this.metrics.averageConfidence =
(this.metrics.averageConfidence * (this.metrics.successfulHeals - 1) + result.confidence) /
this.metrics.successfulHeals;
}
else {
this.metrics.failedHeals++;
}
this.metrics.averageResponseTime =
(this.metrics.averageResponseTime * (this.metrics.totalRequests - 1) + responseTime) /
this.metrics.totalRequests;
// Record strategy success
if (result?.analytics?.patternType) {
const patternType = result.analytics.patternType;
const current = this.metrics.strategySuccessRates.get(patternType) || { success: 0, total: 0 };
current.total++;
if (result.confidence > 0.7)
current.success++;
this.metrics.strategySuccessRates.set(patternType, current);
}
// Record element type distribution
if (result?.features?.tagName) {
const tagName = result.features.tagName;
const current = this.metrics.elementTypeDistribution.get(tagName) || 0;
this.metrics.elementTypeDistribution.set(tagName, current + 1);
}
},
getMetrics() {
return {
...this.metrics,
successRate: this.metrics.totalRequests > 0 ?
(this.metrics.successfulHeals / this.metrics.totalRequests) * 100 : 0,
strategySuccessRates: Object.fromEntries(this.metrics.strategySuccessRates),
elementTypeDistribution: Object.fromEntries(this.metrics.elementTypeDistribution)
};
},
generateReport() {
const metrics = this.getMetrics();
return `
Advanced ML Healing Analytics Report
===================================
Total Requests: ${metrics.totalRequests}
Success Rate: ${metrics.successRate.toFixed(2)}%
Average Response Time: ${metrics.averageResponseTime.toFixed(2)}ms
Average Confidence: ${(metrics.averageConfidence * 100).toFixed(2)}%
Strategy Success Rates:
${Object.entries(metrics.strategySuccessRates).map(([strategy, data]) => {
const typedData = data;
return ` ${strategy}: ${typedData.total > 0 ? ((typedData.success / typedData.total) * 100).toFixed(2) : 0}% (${typedData.success}/${typedData.total})`;
}).join('\n')}
Element Type Distribution:
${Object.entries(metrics.elementTypeDistribution).map(([type, count]) => ` ${type}: ${count}`).join('\n')}
`;
}
};
this.adaptiveLearningSystem = {
successPatterns: [],
failurePatterns: [],
performanceMetrics: {
averageResponseTime: 0,
successRate: 0,
confidenceThreshold: 0.7,
learningRate: 0.01
},
optimizationStrategies: [
{
name: 'neural-network-optimization',
description: 'Optimize neural network weights based on success patterns',
successRate: 0.85,
implementation: 'backpropagation'
},
{
name: 'fuzzy-logic-enhancement',
description: 'Enhance fuzzy matching algorithms',
successRate: 0.78,
implementation: 'levenshtein-optimization'
},
{
name: 'context-aware-improvement',
description: 'Improve context-aware feature extraction',
successRate: 0.82,
implementation: 'semantic-enhancement'
}
]
};
this.machineLearningModels = {
supportVectorMachine: {
kernel: 'rbf',
C: 1.0,
gamma: 'scale',
degree: 3,
coef0: 0.0,
trained: false,
supportVectors: [],
dualCoefficients: [],
intercept: 0
},
randomForest: {
nEstimators: 100,
maxDepth: 10,
minSamplesSplit: 2,
minSamplesLeaf: 1,
maxFeatures: 'sqrt',
bootstrap: true,
trained: false,
trees: []
},
gradientBoosting: {
nEstimators: 100,
learningRate: 0.1,
maxDepth: 3,
subsample: 1.0,
trained: false,
estimators: []
},
naiveBayes: {
type: 'multinomial',
trained: false,
classCounts: {},
featureCounts: {},
classPriors: {}
},
kNearestNeighbors: {
nNeighbors: 5,
weights: 'uniform',
algorithm: 'auto',
leafSize: 30,
trained: false,
trainingData: []
}
};
this.naturalLanguageProcessing = {
tokenization: {
method: 'word',
vocabulary: new Set(),
maxLength: 100,
padding: 'post',
truncation: 'post'
},
embeddings: {
type: 'word2vec',
dimensions: 300,
vocabulary: {},
unknownToken: '<UNK>'
},
languageModel: {
type: 'transformer',
layers: 6,
hiddenSize: 512,
attentionHeads: 8,
dropout: 0.1,
trained: false
},
semanticAnalysis: {
similarityMetrics: ['cosine', 'euclidean'],
clustering: {
method: 'kmeans',
nClusters: 10,
minSamples: 5
},
topicModeling: {
method: 'lda',
nTopics: 20,
maxIterations: 100
}
}
};
this.computerVisionFeatures = {
imageProcessing: {
filters: [
{ type: 'gaussian', kernelSize: 3, sigma: 1.0 },
{ type: 'sobel', kernelSize: 3, sigma: 0 },
{ type: 'canny', kernelSize: 3, sigma: 1.0 }
],
transformations: [
{ type: 'resize', parameters: { width: 224, height: 224 } },
{ type: 'rotate', parameters: { angle: 0 } }
],
colorSpaces: ['rgb', 'hsv', 'grayscale']
},
featureExtraction: {
methods: ['sift', 'surf', 'orb'],
descriptors: [
{ type: 'hog', parameters: { cellSize: 8, blockSize: 16 } },
{ type: 'lbp', parameters: { radius: 1, neighbors: 8 } }
],
keypoints: []
},
objectDetection: {
model: 'yolo',
confidence: 0.5,
nmsThreshold: 0.4,
anchors: [[10, 13], [16, 30], [33, 23]],
classes: ['button', 'input', 'link', 'image', 'text']
},
opticalCharacterRecognition: {
engine: 'tesseract',
languages: ['eng'],
confidence: 0.8,
preprocessing: [
{ type: 'threshold', parameters: { method: 'otsu' } },
{ type: 'noise_removal', parameters: { method: 'median' } }
]
}
};
this.ensembleMethods = {
voting: {
type: 'soft',
weights: [0.3, 0.3, 0.2, 0.2],
models: ['neural_network', 'svm', 'random_forest', 'gradient_boosting']
},
stacking: {
baseModels: ['neural_network', 'svm', 'random_forest'],
metaModel: 'logistic_regression',
crossValidation: 5
},
bagging: {
nEstimators: 10,
maxSamples: 0.8,
maxFeatures: 0.8,
bootstrap: true
},
boosting: {
type: 'gradient',
nEstimators: 100,
learningRate: 0.1,
maxDepth: 3
}
};
this.realTimeLearning = {
onlineLearning: {
enabled: true,
batchSize: 32,
updateFrequency: 100,
forgettingFactor: 0.95
},
incrementalLearning: {
enabled: true,
memorySize: 1000,
importanceSampling: true,
conceptDrift: {
detection: true,
threshold: 0.1,
windowSize: 100
}
},
activeLearning: {
enabled: true,
queryStrategy: 'uncertainty',
budget: 100,
uncertaintyMetrics: ['entropy', 'margin', 'least_confidence']
},
reinforcementLearning: {
enabled: true,
algorithm: 'q_learning',
stateSpace: 100,
actionSpace: 10,
learningRate: 0.1,
discountFactor: 0.9,
epsilon: 0.1
}
};
// Enhanced analytics and performance tracking
this.healingAnalytics = {
totalRequests: 0,
successfulHealings: 0,
failedHealings: 0,
strategySuccessRates: new Map(),
responseTimes: [],
patternTypes: new Map(),
elementTypes: new Map(),
cacheMetrics: {
hits: 0,
misses: 0,
size: 0,
ttl: 300000 // 5 minutes
},
adaptiveLearning: {
successPatterns: new Map(),
failurePatterns: new Map(),
confidenceThresholds: new Map()
}
};
// Enhanced caching with TTL and access tracking
this.healingCache = new Map();
}
fuzzyStringMatch(str1, str2) {
const longer = str1.length > str2.length ? str1 : str2;
const shorter = str1.length > str2.length ? str2 : str1;
if (longer.length === 0)
return 1.0;
const distance = this.levenshteinDistance(longer, shorter);
return (longer.length - distance) / longer.length;
}
levenshteinDistance(str1, str2) {
const matrix = Array(str2.length + 1).fill(null).map(() => Array(str1.length + 1).fill(null));
for (let i = 0; i <= str1.length; i++)
matrix[0][i] = i;
for (let j = 0; j <= str2.length; j++)
matrix[j][0] = j;
for (let j = 1; j <= str2.length; j++) {
for (let i = 1; i <= str1.length; i++) {
const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1;
matrix[j][i] = Math.min(matrix[j][i - 1] + 1, matrix[j - 1][i] + 1, matrix[j - 1][i - 1] + indicator);
}
}
return matrix[str2.length][str1.length];
}
isAnagram(str1, str2) {
const cleanStr1 = str1.toLowerCase().replace(/[^a-z0-9]/g, '');
const cleanStr2 = str2.toLowerCase().replace(/[^a-z0-9]/g, '');
if (cleanStr1.length !== cleanStr2.length)
return false;
const sortedStr1 = cleanStr1.split('').sort().join('');
const sortedStr2 = cleanStr2.split('').sort().join('');
return sortedStr1 === sortedStr2;
}
calculateNeuralNetworkHealth(features) {
const inputs = [
features.neuralScore,
features.fuzzyMatchScore,
features.contextSimilarity,
features.accessibilityScore,
features.historicalSuccess
];
const weightedSum = inputs.reduce((sum, input, index) => {
const weight = this.neuralNetwork.weights[`weight-${index}`] || 0.2;
return sum + (input * weight);
}, 0);
const bias = this.neuralNetwork.biases['base-bias'] || 0.1;
return this.neuralNetwork.activationFunction(weightedSum + bias);
}
processMultiModalAnalysis(element) {
const computedStyle = window.getComputedStyle(element);
const rect = element.getBoundingClientRect();
return {
visualFeatures: {
color: computedStyle.color,
backgroundColor: computedStyle.backgroundColor,
fontSize: computedStyle.fontSize,
fontWeight: computedStyle.fontWeight,
borderStyle: computedStyle.borderStyle,
borderRadius: computedStyle.borderRadius,
boxShadow: computedStyle.boxShadow,
opacity: parseFloat(computedStyle.opacity) || 1,
zIndex: parseInt(computedStyle.zIndex) || 0,
position: computedStyle.position
},
accessibilityFeatures: {
ariaLabel: element.getAttribute('aria-label') || '',
ariaDescribedBy: element.getAttribute('aria-describedby') || '',
ariaLabelledBy: element.getAttribute('aria-labelledby') || '',
ariaHidden: element.getAttribute('aria-hidden') === 'true',
ariaDisabled: element.getAttribute('aria-disabled') === 'true',
ariaRequired: element.getAttribute('aria-required') === 'true',
ariaInvalid: element.getAttribute('aria-invalid') === 'true',
tabIndex: parseInt(element.getAttribute('tabindex') || '0'),
role: element.getAttribute('role') || ''
},
semanticFeatures: {
tagName: element.tagName.toLowerCase(),
textContent: element.textContent || '',
placeholder: element.placeholder || '',
title: element.getAttribute('title') || '',
alt: element.alt || '',
href: element.href || '',
type: element.type || '',
value: element.value || '',
checked: element.checked || false,
selected: element.selected || false
},
layoutFeatures: {
x: rect.x,
y: rect.y,
width: rect.width,
height: rect.height,
offsetTop: element.offsetTop || 0,
offsetLeft: element.offsetLeft || 0,
scrollTop: element.scrollTop || 0,
scrollLeft: element.scrollLeft || 0,
clientWidth: element.clientWidth || 0,
clientHeight: element.clientHeight || 0
},
interactionFeatures: {
clickable: element.tagName === 'BUTTON' || element.tagName === 'A' || element.onclick !== null,
focusable: element.tagName === 'INPUT' || element.tagName === 'BUTTON' || element.tagName === 'A' || element.tabIndex >= 0,
editable: element.tagName === 'INPUT' || element.tagName === 'TEXTAREA' || element.contentEditable === 'true',
draggable: element.draggable || false,
resizable: element.tagName === 'TEXTAREA' || element.style.resize !== 'none',
selectable: element.tagName === 'INPUT' || element.tagName === 'TEXTAREA',
scrollable: element.scrollHeight > element.clientHeight || element.scrollWidth > element.clientWidth
}
};
}
analyzeContextAwareFeatures(element) {
const parent = element.parentElement;
const siblings = Array.from(parent?.children || []);
const form = element.closest('form');
return {
parentContext: {
tagName: parent?.tagName.toLowerCase() || '',
id: parent?.id || '',
className: parent?.className || '',
role: parent?.getAttribute('role') || '',
ariaLabel: parent?.getAttribute('aria-label') || ''
},
siblingContext: siblings.map(sibling => ({
tagName: sibling.tagName.toLowerCase(),
id: sibling.id || '',
className: sibling.className || '',
textContent: sibling.textContent || '',
role: sibling.getAttribute('role') || ''
})),
formContext: {
formId: form?.id || '',
formAction: form?.action || '',
formMethod: form?.method || '',
formEnctype: form?.enctype || ''
},
pageContext: {
title: document.title,
url: window.location.href,
domain: window.location.hostname,
pathname: window.location.pathname,
searchParams: Object.fromEntries(new URLSearchParams(window.location.search))
},
userContext: {
action: 'unknown',
intent: 'unknown',
previousActions: []
}
};
}
calculateAdaptiveScore(element, selector, context) {
const multiModalFeatures = this.processMultiModalAnalysis(element);
const contextFeatures = this.analyzeContextAwareFeatures(element);
let score = 0;
// Visual feature scoring
if (multiModalFeatures.visualFeatures.backgroundColor !== 'rgba(0, 0, 0, 0)')
score += 0.1;
if (multiModalFeatures.visualFeatures.fontSize !== '16px')
score += 0.1;
if (multiModalFeatures.visualFeatures.position !== 'static')
score += 0.1;
// Accessibility feature scoring
if (multiModalFeatures.accessibilityFeatures.ariaLabel)
score += 0.2;
if (multiModalFeatures.accessibilityFeatures.role)
score += 0.15;
if (multiModalFeatures.accessibilityFeatures.tabIndex >= 0)
score += 0.1;
// Semantic feature scoring
if (multiModalFeatures.semanticFeatures.textContent)
score += 0.2;
if (multiModalFeatures.semanticFeatures.placeholder)
score += 0.15;
if (multiModalFeatures.semanticFeatures.type)
score += 0.1;
// Interaction feature scoring
if (multiModalFeatures.interactionFeatures.clickable)
score += 0.15;
if (multiModalFeatures.interactionFeatures.focusable)
score += 0.1;
if (multiModalFeatures.interactionFeatures.editable)
score += 0.1;
// Context-aware scoring
if (contextFeatures.formContext.formId)
score += 0.1;
if (contextFeatures.parentContext.role)
score += 0.1;
return Math.min(score, 1.0);
}
processParallelHealingStrategies(selector, context) {
const strategies = [
{ name: 'neural-network', fn: () => this.processNeuralNetworkStrategy(selector, context) },
{ name: 'fuzzy-logic', fn: () => this.processFuzzyLogicStrategy(selector, context) },
{ name: 'semantic-analysis', fn: () => this.processSemanticAnalysisStrategy(selector, context) },
{ name: 'context-aware', fn: () => this.processContextAwareStrategy(selector, context) },
{ name: 'pattern-recognition', fn: () => this.processPatternRecognitionStrategy(selector, context) },
{ name: 'multi-modal', fn: () => this.processMultiModalStrategy(selector, context) }
];
return Promise.all(strategies.map(async (strategy) => {
try {
const result = await strategy.fn();
return {
strategy: strategy.name,
result,
confidence: result?.confidence || 0
};
}
catch (error) {
return {
strategy: strategy.name,
result: null,
confidence: 0
};
}
}));
}
async processNeuralNetworkStrategy(selector, context) {
// Neural network processing logic
const inputs = this.extractNeuralInputs(selector, context);
const output = this.neuralNetwork.activationFunction(inputs.reduce((sum, input, index) => {
const weight = this.neuralNetwork.weights[`neural-weight-${index}`] || 0.2;
return sum + (input * weight);
}, 0) + (this.neuralNetwork.biases['neural-bias'] || 0.1));
return {
confidence: output,
reasoning: 'Neural network analysis',
strategy: 'neural-network'
};
}
async processFuzzyLogicStrategy(selector, context) {
// Fuzzy logic processing
const fuzzyScore = this.calculateFuzzyScore(selector, context);
return {
confidence: fuzzyScore,
reasoning: 'Fuzzy logic analysis',
strategy: 'fuzzy-logic'
};
}
async processSemanticAnalysisStrategy(selector, context) {
// Semantic analysis processing
const semanticScore = this.calculateSemanticScore(selector, context);
return {
confidence: semanticScore,
reasoning: 'Semantic analysis',
strategy: 'semantic-analysis'
};
}
async processContextAwareStrategy(selector, context) {
// Context-aware processing
const contextScore = this.calculateContextScore(context);
return {
confidence: contextScore,
reasoning: 'Context-aware analysis',
strategy: 'context-aware'
};
}
async processPatternRecognitionStrategy(selector, context) {
// Pattern recognition processing
const patternScore = this.calculateFuzzyScore(selector, context);
return {
confidence: patternScore,
reasoning: 'Pattern recognition analysis',
strategy: 'pattern-recognition'
};
}
async processMultiModalStrategy(selector, context) {
// Multi-modal processing
const multiModalScore = this.calculateFuzzyScore(selector, context);
return {
confidence: multiModalScore,
reasoning: 'Multi-modal analysis',
strategy: 'multi-modal'
};
}
extractNeuralInputs(selector, context) {
return [
selector.length / 100,
selector.split(/[-_\s]/).length / 10,
context?.action === 'click' ? 1 : 0,
context?.action === 'fill' ? 1 : 0,
selector.includes('#') ? 1 : 0,
selector.includes('.') ? 1 : 0,
selector.includes('[') ? 1 : 0
];
}
calculateFuzzyScore(selector, context) {
// Enhanced fuzzy scoring
let score = 0;
const cleanSelector = selector.replace(/[#.]/g, '').toLowerCase();
// Length-based scoring
if (cleanSelector.length > 3)
score += 0.2;
if (cleanSelector.length > 6)
score += 0.2;
// Pattern-based scoring
if (/^[a-z]+$/.test(cleanSelector))
score += 0.3;
if (/^[a-z]+[0-9]+$/.test(cleanSelector))
score += 0.2;
if (/^[a-z]+-[a-z]+$/.test(cleanSelector))
score += 0.2;
// Context-based scoring
if (context?.action)
score += 0.1;
return Math.min(score, 1.0);
}
// Advanced Machine Learning Methods
trainSupportVectorMachine(trainingData) {
// Simplified SVM training
const features = trainingData.map(d => d.features);
const labels = trainingData.map(d => d.label);
// Calculate support vectors (simplified)
const supportVectors = features.slice(0, Math.min(10, features.length));
const dualCoefficients = supportVectors.map(() => [1.0]);
this.machineLearningModels.supportVectorMachine = {
...this.machineLearningModels.supportVectorMachine,
supportVectors,
dualCoefficients,
trained: true
};
}
trainRandomForest(trainingData) {
const trees = [];
const nEstimators = this.machineLearningModels.randomForest.nEstimators;
for (let i = 0; i < nEstimators; i++) {
// Simplified decision tree
const tree = {
nodes: [
{ feature: 0, threshold: 0.5, left: 1, right: 2, value: 0 },
{ feature: -1, threshold: 0, left: -1, right: -1, value: 0 },
{ feature: -1, threshold: 0, left: -1, right: -1, value: 1 }
]
};
trees.push(tree);
}
this.machineLearningModels.randomForest = {
...this.machineLearningModels.randomForest,
trees,
trained: true
};
}
trainGradientBoosting(trainingData) {
const estimators = [];
const nEstimators = this.machineLearningModels.gradientBoosting.nEstimators;
for (let i = 0; i < nEstimators; i++) {
const estimator = {
features: [0, 1, 2],
thresholds: [0.5, 0.3, 0.7],
values: [0.1, 0.2, 0.3]
};
estimators.push(estimator);
}
this.machineLearningModels.gradientBoosting = {
...this.machineLearningModels.gradientBoosting,
estimators,
trained: true
};
}
trainNaiveBayes(trainingData) {
const classCounts = {};
const featureCounts = {};
const classPriors = {};
// Count classes
trainingData.forEach(data => {
classCounts[data.label] = (classCounts[data.label] || 0) + 1;
});
// Calculate priors
const total = trainingData.length;
Object.keys(classCounts).forEach(label => {
classPriors[label] = classCounts[label] / total;
});
// Count features
trainingData.forEach(data => {
data.features.forEach(feature => {
if (!featureCounts[feature])
featureCounts[feature] = {};
featureCounts[feature][data.label] = (featureCounts[feature][data.label] || 0) + 1;
});
});
this.machineLearningModels.naiveBayes = {
...this.machineLearningModels.naiveBayes,
classCounts,
featureCounts,
classPriors,
trained: true
};
}
trainKNearestNeighbors(trainingData) {
this.machineLearningModels.kNearestNeighbors = {
...this.machineLearningModels.kNearestNeighbors,
trainingData,
trained: true
};
}
// Natural Language Processing Methods
tokenizeText(text) {
const tokens = text.toLowerCase()
.replace(/[^\w\s]/g, '')
.split(/\s+/)
.filter(token => token.length > 0);
// Update vocabulary
tokens.forEach(token => {
this.naturalLanguageProcessing.tokenization.vocabulary.add(token);
});
return tokens;
}
calculateWordEmbeddings(tokens) {
const embeddings = [];
tokens.forEach(token => {
if (this.naturalLanguageProcessing.embeddings.vocabulary[token]) {
embeddings.push(this.naturalLanguageProcessing.embeddings.vocabulary[token]);
}
else {
// Generate random embedding for unknown tokens
const embedding = Array(this.naturalLanguageProcessing.embeddings.dimensions)
.fill(0)
.map(() => Math.random() - 0.5);
embeddings.push(embedding);
}
});
return embeddings;
}
calculateSemanticSimilarity(text1, text2) {
const tokens1 = this.tokenizeText(text1);
const tokens2 = this.tokenizeText(text2);
const embeddings1 = this.calculateWordEmbeddings(tokens1);
const embeddings2 = this.calculateWordEmbeddings(tokens2);
if (embeddings1.length === 0 || embeddings2.length === 0)
return 0;
// Calculate cosine similarity
const avgEmbedding1 = embeddings1.reduce((sum, emb) => sum.map((val, i) => val + emb[i]), Array(this.naturalLanguageProcessing.embeddings.dimensions).fill(0)).map(val => val / embeddings1.length);
const avgEmbedding2 = embeddings2.reduce((sum, emb) => sum.map((val, i) => val + emb[i]), Array(this.naturalLanguageProcessing.embeddings.dimensions).fill(0)).map(val => val / embeddings2.length);
const dotProduct = avgEmbedding1.reduce((sum, val, i) => sum + val * avgEmbedding2[i], 0);
const magnitude1 = Math.sqrt(avgEmbedding1.reduce((sum, val) => sum + val * val, 0));
const magnitude2 = Math.sqrt(avgEmbedding2.reduce((sum, val) => sum + val * val, 0));
return dotProduct / (magnitude1 * magnitude2);
}
// Computer Vision Methods
processImageFeatures(element) {
const rect = element.getBoundingClientRect();
const computedStyle = window.getComputedStyle(element);
return {
dimensions: {
width: rect.width,
height: rect.height,
aspectRatio: rect.width / rect.height
},
colors: {
backgroundColor: computedStyle.backgroundColor,
color: computedStyle.color,
borderColor: computedStyle.borderColor
},
visualFeatures: {
fontSize: computedStyle.fontSize,
fontWeight: computedStyle.fontWeight,
opacity: parseFloat(computedStyle.opacity) || 1,
visibility: computedStyle.visibility,
display: computedStyle.display
}
};
}
detectVisualElements(elements) {
const detections = [];
elements.forEach(element => {
const features = this.processImageFeatures(element);
const tagName = element.tagName.toLowerCase();
let type = 'unknown';
let confidence = 0.5;
if (tagName === 'button' || element.getAttribute('role') === 'button') {
type = 'button';
confidence = 0.9;
}
else if (tagName === 'input') {
type = 'input';
confidence = 0.9;
}
else if (tagName === 'a') {
type = 'link';
confidence = 0.8;
}
else if (tagName === 'img') {
type = 'image';
confidence = 0.8;
}
else if (element.textContent && element.textContent.trim().length > 0) {
type = 'text';
confidence = 0.7;
}
detections.push({ element, type, confidence });
});
return detections;
}
// Ensemble Methods
ensemblePrediction(features) {
const predictions = [];
// Neural Network prediction
const nnPrediction = this.neuralNetwork.activationFunction(features.reduce((sum, feature, index) => {
const weight = this.neuralNetwork.weights[`neural-weight-${index}`] || 0.2;
return sum + (feature * weight);
}, 0) + (this.neuralNetwork.biases['neural-bias'] || 0.1));
predictions.push(nnPrediction);
// SVM prediction (simplified)
if (this.machineLearningModels.supportVectorMachine.trained) {
const svmPrediction = 0.8; // Simplified
predictions.push(svmPrediction);
}
// Random Forest prediction (simplified)
if (this.machineLearningModels.randomForest.trained) {
const rfPrediction = 0.75; // Simplified
predictions.push(rfPrediction);
}
// Weighted average
const weights = this.ensembleMethods.voting.weights;
const weightedSum = predictions.reduce((sum, pred, index) => sum + (pred * (weights[index] || 1)), 0);
return weightedSum / predictions.length;
}
// Real-time Learning Methods
updateOnlineLearning(features, prediction, actual) {
if (!this.realTimeLearning.onlineLearning.enabled)
return;
const error = actual - prediction;
const learningRate = this.neuralNetwork.learningRate;
// Update weights online
features.forEach((feature, index) => {
const currentWeight = this.neuralNetwork.weights[`neural-weight-${index}`] || 0.2;
this.neuralNetwork.weights[`neural-weight-${index}`] = currentWeight + (learningRate * error * feature);
});
}
detectConceptDrift(predictions, actuals) {
if (!this.realTimeLearning.incrementalLearning.conceptDrift.detection)
return false;
const windowSize = this.realTimeLearning.incrementalLearning.conceptDrift.windowSize;
const threshold = this.realTimeLearning.incrementalLearning.conceptDrift.threshold;
if (predictions.length < windowSize)
return false;
const recentPredictions = predictions.slice(-windowSize);
const recentActuals = actuals.slice(-windowSize);
const recentError = recentPredictions.reduce((sum, pred, index) => sum + Math.abs(pred - recentActuals[index]), 0) / recentPredictions.length;
const overallError = predictions.reduce((sum, pred, index) => sum + Math.abs(pred - actuals[index]), 0) / predictions.length;
return Math.abs(recentError - overallError) > threshold;
}
activeLearningQuery(features, prediction) {
if (!this.realTimeLearning.activeLearning.enabled)
return false;
const budget = this.realTimeLearning.activeLearning.budget;
const uncertaintyMetrics = this.realTimeLearning.activeLearning.uncertaintyMetrics;
// Calculate uncertainty
let uncertainty = 0;
if (uncertaintyMetrics.includes('entropy')) {
const p = prediction;
const entropy = -p * Math.log(p) - (1 - p) * Math.log(1 - p);
uncertainty = Math.max(uncertainty, entropy);
}
if (uncertaintyMetrics.includes('margin')) {
const margin = Math.abs(prediction - 0.5);
uncertainty = Math.max(uncertainty, 1 - margin);
}
if (uncertaintyMetrics.includes('least_confidence')) {
uncertainty = Math.max(uncertainty, 1 - Math.max(prediction, 1 - prediction));
}
return uncertainty > 0.5 && budget > 0;
}
// Advanced Analytics Methods
calculateAdvancedMetrics() {
const neuralNetworkHealth = this.calculateNeuralNetworkHealth({
neuralScore: 0.8,
fuzzyMatchScore: 0.7,
contextSimilarity: 0.6,
accessibilityScore: 0.9,
historicalSuccess: 0.85
});
const ensembleScore = this.ensemblePrediction([0.8, 0.7, 0.6, 0.9, 0.85]);
const semanticScore = this.calculateSemanticSimilarity('email field', 'email input');
return {
neuralNetworkHealth,
ensembleScore,
semanticScore,
conceptDrift: this.detectConceptDrift([0.8, 0.7, 0.6], [0.8, 0.7, 0.6]),
activeLearning: this.activeLearningQuery([0.8, 0.7, 0.6], 0.75)
};
}
// Multi-Modal Fusion Methods
fuseMultiModalFeatures(visualFeatures, semanticFeatures, contextFeatures, nlpFeatures) {
let fusedScore = 0;
// Visual features (30%)
if (visualFeatures.dimensions.width > 0)
fusedScore += 0.3;
if (visualFeatures.colors.backgroundColor !== 'rgba(0, 0, 0, 0)')
fusedScore += 0.1;
// Semantic features (25%)
if (semanticFeatures.textContent)
fusedScore += 0.25;
if (semanticFeatures.placeholder)
fusedScore += 0.1;
// Context features (25%)
if (contextFeatures.formContext.formId)
fusedScore += 0.25;
if (contextFeatures.parentContext.role)
fusedScore += 0.1;
// NLP features (20%)
if (nlpFeatures.semanticSimilarity > 0.5)
fusedScore += 0.2;
return Math.min(fusedScore, 1.0);
}
processWithCaching(selector, context) {
const cacheKey = `${selector}-${JSON.stringify(context)}`;
const cached = this.healingHistory.find(h => h.originalSelector === selector &&
JSON.stringify(h.context) === JSON.stringify(context));
if (cached) {
return {
selector: cached.healedSelector,
confidence: 0.9,
reasoning: `Cached result from ${new Date(cached.timestamp).toISOString()}`,
patternType: cached.patternType
};
}
return null;
}
updateAdaptiveLearning(selector, result, context) {
const pattern = this.extractPattern(selector);
if (result && result.confidence > 0.7) {
// Success pattern
const existingPattern = this.adaptiveLearningSystem.successPatterns.find(p => p.pattern === pattern);
if (existingPattern) {
existingPattern.successRate = (existingPattern.successRate + result.confidence) / 2;
existingPattern.usageCount++;
existingPattern.lastUsed = Date.now();
}
else {
this.adaptiveLearningSystem.successPatterns.push({
pattern,
successRate: result.confidence,
confidence: result.confidence,
usageCount: 1,
lastUsed: Date.now()
});
}
}
else {
// Failure pattern
const existingPattern = this.adaptiveLearningSystem.failurePatterns.find(p => p.pattern === pattern);
if (existingPattern) {
existingPattern.failureRate = (existingPattern.failureRate + 1) / 2;
existingPattern.attempts++;
existingPattern.lastAttempt = Date.now();
}
else {
this.adaptiveLearningSystem.failurePatterns.push({
pattern,
failureRate: 1,
attempts: 1,
lastAttempt: Date.now(),
suggestedFix: this.generateSuggestedFix(selector, context)
});
}
}
// Update performance metrics
this.adaptiveLearningSystem.performanceMetrics.successRate =
this.adaptiveLearningSystem.successPatterns.length /
(this.adaptiveLearningSystem.successPatterns.length + this.adaptiveLearningSystem.failurePatterns.length);
}
extractPattern(selector) {
// Extract common patterns from selector
if (selector.startsWith('#'))
return 'id-selector';
if (selector.startsWith('.'))
return 'class-selector';
if (selector.includes('['))
return 'attribute-selector';
if (selector.includes(' '))
return 'descendant-selector';
if (selector.includes('>'))
return 'child-selector';
if (selector.includes('+'))
return 'adjacent-selector';
if (selector.includes('~'))
return 'sibling-selector';
return 'simple-selector';
}
generateSuggestedFix(selector, context) {
// Generate suggested fixes based on failure patterns
if (selector.includes('#')) {
return 'Try using class selector instead of ID';
}
if (selector.includes('.')) {
return 'Try using more specific class selector';
}
if (context?.action === 'click') {
return 'Try using button or link selector';
}
if (context?.action === 'fill') {
return 'Try using input selector with type attribute';
}
return 'Try using more specific selector';
}
// Helper methods for analytics and performance tracking
recordStrategySuccess(strategyName) {
const current = this.healingAnalytics.strategySuccessRates.get(strategyName) || { success: 0, total: 0 };
current.success++;
current.total++;
this.healingAnalytics.strategySuccessRates.set(strategyName, current);
}
recordStrategyFailure(strategyName) {
const current = this.healingAnalytics.strategySuccessRates.get(strategyName) || { success: 0, total: 0 };
current.total++;
this.healingAnalytics.strategySuccessRates.set(strategyName, current);
}
recordResponseTime(responseTime) {
this.healingAnalytics.responseTimes.push(responseTime);
// Keep only last 100 response times for performance
if (this.healingAnalytics.responseTimes.length > 100) {
this.healingAnalytics.responseTimes.shift();
}
}
recordPatternType(patternType) {
const current = this.healingAnalytics.patternTypes.get(patternType) || 0;
this.healingAnalytics.patternTypes.set(patternType, current + 1);
}
recordElementType(elementType) {
const current = this.healingAnalytics.elementTypes.get(elementType) || 0;
this.healingAnalytics.elementTypes.set(elementType, current + 1);
}
// Enhanced caching methods
getFromCache(selector) {
const cached = this.healingCache.get(selector);
if (cached) {
const now = Date.now();
if (now - cached.timestamp < cached.ttl) {
// Cache hit - update access metrics
cached.accessCount++;
cached.lastAccessed = now;
this.healingAnalytics.cacheMetrics.hits++;
return cached.result;
}
else {
// Cache expired - remove
this.healingCache.delete(selector);
this.healingAnalytics.cacheMetrics.size--;
}
}
this.healingAnalytics.cacheMetrics.misses++;
return null;
}
setCache(selector, result, ttl = 300000) {
// Clean up old cache entries if size exceeds 100
if (this.healingCache.size >= 100) {
const entries = Array.from(this.healingCache.entries());
entries.sort((a, b) => a[1].lastAccessed - b[1].lastAccessed);
const toRemove = entries.slice(0, 20); // Remove 20 oldest entries
toRemove.forEach(([key]) => {
this.healingCache.delete(key);
this.healingAnalytics.cacheMetrics.size--;
});
}
this.healingCache.set(selector, {
result,
timestamp: Date.now(),
accessCount: 1,
lastAccessed: Date.now(),
ttl
});
this.healingAnalytics.cacheMetrics.size++;
}
// Enhanced healing with caching and analytics
async healWithAdvancedML(page, originalSelector, context) {
const startTime = Date.now();
this.healingAnalytics.totalRequests++;
// Check cache first
const cachedResult = this.getFromCache(originalSelector);
if (cachedResult) {
this.healingAnalytics.successfulHealings++;
this.recordResponseTime(Date.now() - startTime);
return cachedResult;
}
try {
// Try fast path first
const fastResult = await this.performOptimizedMLHealing(page, originalSelector, context, startTime);
if (fastResult) {
this.healingAnalytics.successfulHealings++;
this.recordResponseTime(Date.now() - startTime);
this.setCache(originalSelector, fastResult);
return fastResult;
}
// Try parallel healing for better performance
const parallelResult = await this.executeParallelHealing(page, originalSelector, context, startTime);
if (parallelResult) {
this.healingAnalytics.successfulHealings++;
this.recordResponseTime(Date.now() - startTime);
this.setCache(originalSelector, parallelResult);
return parallelResult;
}
// Fallback to comprehensive healing
const comprehensiveResult = await this.performComprehensiveHealing(page, originalSelector, context, startTime);
if (comprehensiveResult) {
this.healingAnalytics.successfulHealings++;
this.recordResponseTime(Date.now() - startTime);
this.setCache(originalSelector, comprehensiveResult);
return comprehensiveResult;
}
// All healing attempts failed
this.healingAnalytics.failedHealings++;
this.recordResponseTime(Date.now() - startTime);
return null;
}
catch (error) {
this.healingAnalytics.failedHealings++;
this.recordResponseT