@simonecoelhosfo/optimizely-mcp-server
Version:
Optimizely MCP Server for AI assistants with integrated CLI tools
487 lines • 20.7 kB
JavaScript
/**
* JSONata-based Field Extractor
*
* Uses JSONata expressions to dynamically search for and extract required fields
* from payloads regardless of nesting depth or field naming variations.
*
* This addresses the core problem of AI agents sending data in unpredictable
* structures with varying field names and nesting levels.
*/
import jsonata from 'jsonata';
import { getLogger } from '../logging/Logger.js';
import { FIELDS } from '../generated/fields.generated.js';
import { UNIFIED_FIELD_SYNONYMS } from './UnifiedFieldMappings.js';
export class JSONataFieldExtractor {
logger = getLogger();
// Use unified field synonyms from centralized source
FIELD_SYNONYMS = UNIFIED_FIELD_SYNONYMS;
// Pre-compiled JSONata expressions for common searches
expressions = new Map();
constructor() {
this.initializeExpressions();
}
/**
* Initialize commonly used JSONata expressions
*/
initializeExpressions() {
// TEMPORARY FIX: Disable JSONata expressions that cause parsing errors
// The field synonyms contain spaces and special characters that break JSONata
// For now, we'll rely on direct property access and manual nested search
this.logger.debug('JSONata expressions disabled due to parsing issues with field names containing spaces');
// TODO: Implement proper JSONata expression handling for field names with spaces/special chars
// For now, the findField method will use direct property access and manual traversal
}
/**
* Extract required fields for an entity type from a payload
*/
async extractFields(entityType, payload, options) {
try {
// Input validation - ensure payload is an object
if (!payload || typeof payload !== 'object') {
this.logger.error({
entityType,
payload,
payloadType: typeof payload
}, 'JSONataFieldExtractor: Invalid payload - not an object');
return {
success: false,
extractedFields: {},
missingRequired: [],
transformations: [],
confidence: 0
};
}
// Handle array payloads (shouldn't happen, but better to handle gracefully)
if (Array.isArray(payload)) {
this.logger.error({
entityType,
payloadLength: payload.length,
payloadSample: payload.slice(0, 3)
}, 'JSONataFieldExtractor: Received array payload instead of object');
return {
success: false,
extractedFields: {},
missingRequired: [],
transformations: [],
confidence: 0
};
}
this.logger.info({
entityType,
payloadKeys: Object.keys(payload || {}),
platform: options?.platform
}, 'JSONataFieldExtractor: Starting field extraction');
const schema = FIELDS[entityType];
if (!schema) {
this.logger.error({
entityType,
availableSchemas: Object.keys(FIELDS)
}, 'JSONataFieldExtractor: No schema found for entity type');
return {
success: false,
extractedFields: {},
missingRequired: [],
transformations: [],
confidence: 0
};
}
const extractedFields = {};
const transformations = [];
const missingRequired = [];
// Extract all fields (required + optional)
const allFields = [
...schema.required,
...schema.optional
];
for (const field of allFields) {
try {
const result = await this.findField(field, payload);
if (result.found) {
extractedFields[field] = result.value;
if (result.sourcePath !== field) {
transformations.push({
field,
sourcePath: result.sourcePath,
targetPath: field,
value: result.value
});
}
}
else if (schema.required.includes(field)) {
// Check if we can use a default
if (schema.defaults && schema.defaults[field] !== undefined) {
extractedFields[field] = schema.defaults[field];
transformations.push({
field,
sourcePath: 'default',
targetPath: field,
value: schema.defaults[field]
});
}
else {
missingRequired.push(field);
}
}
}
catch (fieldError) {
this.logger.error({
entityType,
field,
error: fieldError instanceof Error ? fieldError.message : 'Unknown field extraction error',
stack: fieldError instanceof Error ? fieldError.stack : undefined
}, 'JSONataFieldExtractor: Error extracting individual field');
// Continue processing other fields
continue;
}
}
// Special handling for platform-specific fields
if (options?.platform) {
this.applyPlatformSpecificLogic(entityType, extractedFields, options.platform);
}
// Handle nested structures that should be flattened
this.handleNestedStructures(entityType, payload, extractedFields, transformations);
const confidence = this.calculateConfidence(allFields.length, Object.keys(extractedFields).length, missingRequired.length, transformations.length);
this.logger.info({
entityType,
fieldsFound: Object.keys(extractedFields).length,
missingRequired,
transformationCount: transformations.length,
confidence
}, 'JSONataFieldExtractor: Field extraction complete');
return {
success: missingRequired.length === 0,
extractedFields,
missingRequired,
transformations,
confidence
};
}
catch (error) {
this.logger.error({
entityType,
payload: payload ? Object.keys(payload) : 'null/undefined',
error: error instanceof Error ? error.message : 'Unknown extraction error',
stack: error instanceof Error ? error.stack : undefined
}, 'JSONataFieldExtractor: Fatal error during field extraction');
return {
success: false,
extractedFields: {},
missingRequired: [],
transformations: [],
confidence: 0
};
}
}
/**
* Find a field in the payload using JSONata
*/
async findField(fieldName, payload) {
// First try direct access
if (payload[fieldName] !== undefined) {
return {
found: true,
value: payload[fieldName],
sourcePath: fieldName
};
}
// Try direct access with all synonyms (for field names with spaces/special chars)
const synonyms = this.FIELD_SYNONYMS[fieldName] || [fieldName];
for (const synonym of synonyms) {
if (payload[synonym] !== undefined) {
return {
found: true,
value: payload[synonym],
sourcePath: synonym
};
}
}
// Manual nested search as fallback
const result = this.searchNestedObject(payload, synonyms);
if (result.found) {
return result;
}
// Use JSONata to search all synonyms
const expression = this.expressions.get(fieldName);
if (!expression) {
// Create expression on the fly if not cached
const synonyms = this.FIELD_SYNONYMS[fieldName] || [fieldName];
const validSynonyms = synonyms
.filter(syn => syn && typeof syn === 'string')
.filter(syn => /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(syn)); // Only valid identifiers
if (validSynonyms.length === 0) {
return {
found: false,
value: undefined,
sourcePath: ''
};
}
const paths = validSynonyms.map(syn => `**.${syn}`).join(' | ');
const dynamicExpression = jsonata(`(${paths})[0]`);
try {
const result = await dynamicExpression.evaluate(payload);
if (result !== undefined) {
// Find which path was used
const sourcePath = await this.findSourcePath(payload, result, synonyms);
return {
found: true,
value: result,
sourcePath
};
}
}
catch (error) {
this.logger.debug({
field: fieldName,
error: error instanceof Error ? error.message : 'Unknown error'
}, 'JSONata evaluation error for field');
}
}
else {
try {
const result = await expression.evaluate(payload);
if (result !== undefined) {
const synonyms = this.FIELD_SYNONYMS[fieldName] || [fieldName];
const sourcePath = await this.findSourcePath(payload, result, synonyms);
return {
found: true,
value: result,
sourcePath
};
}
}
catch (error) {
this.logger.debug({
field: fieldName,
error: error instanceof Error ? error.message : 'Unknown error'
}, 'JSONata evaluation error for field');
}
}
return {
found: false,
value: undefined,
sourcePath: ''
};
}
/**
* Manual nested object search as fallback when JSONata fails
*/
searchNestedObject(obj, synonyms, path = '') {
if (!obj || typeof obj !== 'object') {
return { found: false, value: undefined, sourcePath: '' };
}
// Check current level
for (const synonym of synonyms) {
if (obj[synonym] !== undefined) {
return {
found: true,
value: obj[synonym],
sourcePath: path ? `${path}.${synonym}` : synonym
};
}
}
// Search nested objects (including arrays for complex structures like variations)
for (const [key, value] of Object.entries(obj)) {
if (value && typeof value === 'object') {
if (Array.isArray(value)) {
// Special handling for arrays - search within array elements
for (let i = 0; i < value.length; i++) {
const element = value[i];
if (element && typeof element === 'object') {
const nestedResult = this.searchNestedObject(element, synonyms, path ? `${path}.${key}[${i}]` : `${key}[${i}]`);
if (nestedResult.found) {
return nestedResult;
}
}
}
}
else {
// Regular nested object search
const nestedResult = this.searchNestedObject(value, synonyms, path ? `${path}.${key}` : key);
if (nestedResult.found) {
return nestedResult;
}
}
}
}
return { found: false, value: undefined, sourcePath: '' };
}
/**
* Find the source path that was used to extract a value
*/
async findSourcePath(payload, value, synonyms) {
for (const synonym of synonyms) {
try {
const expr = jsonata(`**."${synonym}"`); // Use quoted notation
const results = await expr.evaluate(payload);
if (Array.isArray(results) ? results.includes(value) : results === value) {
// Find the full path
const pathExpr = jsonata(`**."${synonym}" ~> $path()`);
const paths = await pathExpr.evaluate(payload);
if (paths && paths.length > 0) {
return paths[0];
}
return synonym;
}
}
catch (error) {
// Continue searching - this synonym might have invalid characters
this.logger.debug({
synonym,
error: error instanceof Error ? error.message : 'Unknown error'
}, 'Failed to search for synonym in payload');
}
}
return 'unknown';
}
/**
* Handle special nested structures (like ab_test containing variations)
*/
handleNestedStructures(entityType, payload, extractedFields, transformations) {
// Handle ab_test structure for flags
if (entityType === 'flag' && payload.ab_test) {
const abTest = payload.ab_test;
// Extract nested fields that should be at top level for certain operations
if (abTest.variations && !extractedFields.variations) {
extractedFields.variations = abTest.variations;
transformations.push({
field: 'variations',
sourcePath: 'ab_test.variations',
targetPath: 'variations',
value: abTest.variations
});
}
if (abTest.metrics && !extractedFields.metrics) {
extractedFields.metrics = abTest.metrics;
transformations.push({
field: 'metrics',
sourcePath: 'ab_test.metrics',
targetPath: 'metrics',
value: abTest.metrics
});
}
}
// Handle nested entity_data or template_data
if (payload.entity_data) {
Object.entries(payload.entity_data).forEach(([key, value]) => {
if (!extractedFields[key]) {
extractedFields[key] = value;
transformations.push({
field: key,
sourcePath: `entity_data.${key}`,
targetPath: key,
value
});
}
});
}
// CRITICAL: Preserve complete array structures like variations
this.preserveCompleteArrays(payload, extractedFields, transformations);
}
/**
* Preserve complete array structures without field-by-field extraction
*/
preserveCompleteArrays(payload, extractedFields, transformations) {
// Arrays that should be preserved completely without field extraction
const preserveCompleteArrays = ['variations', 'rules', 'metrics', 'audience_conditions', 'actions'];
for (const arrayField of preserveCompleteArrays) {
if (payload[arrayField] && Array.isArray(payload[arrayField]) && !extractedFields[arrayField]) {
// Preserve the entire array with all nested fields intact
extractedFields[arrayField] = this.deepCloneArray(payload[arrayField]);
transformations.push({
field: arrayField,
sourcePath: arrayField,
targetPath: arrayField,
value: extractedFields[arrayField]
});
this.logger.debug({
field: arrayField,
elementCount: payload[arrayField].length,
firstElement: payload[arrayField][0] ? Object.keys(payload[arrayField][0]) : []
}, 'Preserved complete array structure');
}
}
// Also check for arrays nested one level deep
Object.entries(payload).forEach(([key, value]) => {
if (value && typeof value === 'object' && !Array.isArray(value)) {
const nestedObj = value;
for (const arrayField of preserveCompleteArrays) {
if (nestedObj[arrayField] && Array.isArray(nestedObj[arrayField]) && !extractedFields[arrayField]) {
extractedFields[arrayField] = this.deepCloneArray(nestedObj[arrayField]);
transformations.push({
field: arrayField,
sourcePath: `${key}.${arrayField}`,
targetPath: arrayField,
value: extractedFields[arrayField]
});
this.logger.debug({
field: arrayField,
sourcePath: `${key}.${arrayField}`,
elementCount: nestedObj[arrayField].length
}, 'Preserved nested complete array structure');
}
}
}
});
}
/**
* Deep clone array preserving all nested structure
*/
deepCloneArray(arr) {
return arr.map(item => {
if (item && typeof item === 'object') {
if (Array.isArray(item)) {
return this.deepCloneArray(item);
}
else {
// Deep clone object
const cloned = {};
Object.entries(item).forEach(([key, value]) => {
if (Array.isArray(value)) {
cloned[key] = this.deepCloneArray(value);
}
else if (value && typeof value === 'object') {
cloned[key] = { ...value };
}
else {
cloned[key] = value;
}
});
return cloned;
}
}
return item;
});
}
/**
* Apply platform-specific logic
*/
applyPlatformSpecificLogic(entityType, fields, platform) {
// Example: Remove weight from Feature Experimentation variations
if (entityType === 'variation' && platform === 'feature') {
delete fields.weight;
}
// Add more platform-specific logic as needed
}
/**
* Calculate confidence score for the extraction
*/
calculateConfidence(totalFields, foundFields, missingRequired, transformations) {
const foundRatio = foundFields / totalFields;
const requiredPenalty = missingRequired * 0.2;
const transformPenalty = Math.min(transformations * 0.05, 0.3);
return Math.max(0, Math.min(1, foundRatio - requiredPenalty - transformPenalty));
}
/**
* Add custom field synonyms for a specific entity type
*/
addFieldSynonyms(field, synonyms) {
if (!this.FIELD_SYNONYMS[field]) {
this.FIELD_SYNONYMS[field] = [];
}
this.FIELD_SYNONYMS[field].push(...synonyms);
// Re-compile the expression
const expr = this.FIELD_SYNONYMS[field].map(syn => `**.${syn}`).join(' | ');
this.expressions.set(field, jsonata(`(${expr})[0]`));
}
}
//# sourceMappingURL=JSONataFieldExtractor.js.map