@simonecoelhosfo/optimizely-mcp-server
Version:
Optimizely MCP Server for AI assistants with integrated CLI tools
1,106 lines (1,105 loc) • 50.5 kB
JavaScript
/**
* HybridQueryBuilder - Builds SQL queries with JSONata post-processing for complex nested data
*/
import { QUERY_LIMITS } from './constants.js';
import { IntelligentFieldMapper } from './IntelligentFieldMapper.js';
import { getLogger } from '../logging/Logger.js';
export class HybridQueryBuilder {
schemaMap;
fieldMapper;
constructor() {
this.schemaMap = this.initializeSchemaMap();
this.fieldMapper = new IntelligentFieldMapper();
}
/**
* Build a hybrid SQL + JSONata query from enhanced intent
*/
buildHybridQuery(intent) {
// Step 1: Build base SQL query
const sqlQuery = this.buildBaseSQLQuery(intent);
// Step 2: Determine if JSONata post-processing is needed
const needsJsonProcessing = this.requiresJsonProcessing(intent);
if (!needsJsonProcessing) {
return {
type: 'sql-only',
sql: sqlQuery.sql,
params: sqlQuery.params
};
}
// Step 3: Build processing pipeline for complex operations
const processingPipeline = this.buildProcessingPipeline(intent);
return {
type: 'hybrid',
sql: sqlQuery.sql,
params: sqlQuery.params,
jsonataExpression: intent.jsonataExpression,
processingPipeline
};
}
/**
* Build basic SQL query from intent
*/
buildBaseSQLQuery(intent) {
const schema = this.schemaMap[intent.primaryEntity];
if (!schema) {
throw new Error(`Unknown entity type: ${intent.primaryEntity}`);
}
const query = new SQLQueryBuilder();
// Build SELECT clause
this.buildSelectClause(query, intent, schema);
// Build FROM clause with JOINs
this.buildFromClause(query, intent, schema);
// Build WHERE clause
this.buildWhereClause(query, intent, schema);
// Build GROUP BY clause
this.buildGroupByClause(query, intent, schema);
// Build ORDER BY clause
this.buildOrderByClause(query, intent, schema);
// Build LIMIT clause
this.buildLimitClause(query, intent);
return query.compile();
}
buildSelectClause(query, intent, schema) {
const selectFields = [];
const mainTable = schema.table;
// Always include primary key and basic fields with aliases to remove table prefix
if (intent.primaryEntity === 'projects') {
selectFields.push(`${mainTable}.id AS id`, `${mainTable}.name AS name`);
}
else {
selectFields.push(`${mainTable}.id AS id`, `${mainTable}.project_id AS project_id`, `${mainTable}.name AS name`);
}
// Add entity-specific fields with aliases to remove table prefix
switch (intent.primaryEntity) {
case 'flags':
selectFields.push(`${mainTable}.key AS key`, `${mainTable}.description AS description`, `${mainTable}.archived AS archived`, `${mainTable}.data_json AS data_json`);
// Add timestamp fields for flags
selectFields.push(`${mainTable}.created_time AS created_time`, `${mainTable}.updated_time AS updated_time`);
break;
case 'experiments':
selectFields.push(`${mainTable}.status AS status`, `${mainTable}.type AS type`, `${mainTable}.flag_key AS flag_key`, `${mainTable}.environment AS environment`, `${mainTable}.data_json AS data_json`);
// Add timestamp fields for experiments
selectFields.push(`${mainTable}.created AS created`, `${mainTable}.last_modified AS last_modified`);
break;
case 'audiences':
selectFields.push(`${mainTable}.description AS description`, `${mainTable}.conditions AS audience_conditions`, `${mainTable}.archived AS archived`, `${mainTable}.data_json AS data_json`);
// Add timestamp fields for audiences
selectFields.push(`${mainTable}.created AS created`, `${mainTable}.last_modified AS last_modified`);
break;
case 'variations':
selectFields.push(`${mainTable}.key AS key`, `${mainTable}.flag_key AS flag_key`, `${mainTable}.enabled AS enabled`, `${mainTable}.variables AS variables`, `${mainTable}.data_json AS data_json`);
break;
case 'rules':
selectFields.push(`${mainTable}.key AS key`, `${mainTable}.flag_key AS flag_key`, `${mainTable}.type AS type`, `${mainTable}.percentage_included AS percentage_included`, `${mainTable}.audience_conditions AS audience_conditions`, `${mainTable}.data_json AS data_json`);
break;
case 'events':
selectFields.push(`${mainTable}.key AS key`, `${mainTable}.event_type AS event_type`, `${mainTable}.category AS category`, `${mainTable}.data_json AS data_json`);
// Add timestamp fields for events
selectFields.push(`${mainTable}.created AS created`, `${mainTable}.last_modified AS last_modified`);
break;
case 'campaigns':
selectFields.push(`${mainTable}.description AS description`, `${mainTable}.status AS status`, `${mainTable}.data_json AS data_json`);
// Add timestamp fields for campaigns
selectFields.push(`${mainTable}.created AS created`, `${mainTable}.last_modified AS last_modified`);
break;
case 'pages':
selectFields.push(`${mainTable}.key AS key`, `${mainTable}.page_type AS page_type`, `${mainTable}.activation_code AS activation_code`, `${mainTable}.data_json AS data_json`);
// Add timestamp fields for pages
selectFields.push(`${mainTable}.created AS created`, `${mainTable}.last_modified AS last_modified`);
break;
default:
selectFields.push(`${mainTable}.data_json AS data_json`);
}
// Add aggregation fields if grouping
if (intent.groupBy && intent.groupBy.length > 0) {
const entityType = this.getEntityTypeFromSchema(schema);
// Use IntelligentFieldMapper for GROUP BY fields
for (const field of intent.groupBy) {
try {
const fieldMapping = this.fieldMapper.resolveField(entityType, field);
// Add the resolved field to SELECT
if (fieldMapping.requiresJsonProcessing && fieldMapping.jsonataPath) {
// Special case: grouping by array elements requires different handling
if (fieldMapping.jsonataPath.includes('variations.') && entityType === 'experiments') {
// Don't add to SELECT - we'll get the full data_json and process later
if (!selectFields.some(f => f.includes('data_json'))) {
selectFields.push(`${mainTable}.data_json`);
}
}
else {
// For non-array JSON fields in GROUP BY, extract the value
const jsonPath = fieldMapping.jsonataPath.replace(/\./g, '.');
const jsonExtractExpr = `json_extract(${fieldMapping.sqlField}, '$.${jsonPath}') as ${field.replace(/\./g, '_')}`;
if (!selectFields.some(f => f.includes(jsonExtractExpr))) {
selectFields.push(jsonExtractExpr);
}
}
}
else if (!selectFields.includes(fieldMapping.sqlField)) {
selectFields.push(fieldMapping.sqlField);
}
}
catch (error) {
// Fallback to original logic
const prefixedField = field.includes('.') ? field : `${mainTable}.${field}`;
if (!selectFields.includes(prefixedField)) {
selectFields.push(prefixedField);
}
}
}
// Add count
selectFields.push('COUNT(*) as count');
// Add entity-specific aggregations
if (schema.countable) {
for (const [alias, expression] of Object.entries(schema.countable)) {
selectFields.push(`${expression} as ${alias}`);
}
}
}
// Add time fields if trending
if (intent.action === 'trend' && schema.aggregatable) {
for (const field of schema.aggregatable) {
selectFields.push(field);
}
}
query.select(selectFields);
}
/**
* Sort JOINs by dependency order to avoid "ON clause references tables to its right" errors
*/
sortJoinsByDependency(joins) {
const sorted = [];
const remaining = [...joins];
const joinedTables = new Set(['flags', 'experiments', 'rules', 'audiences', 'events', 'pages', 'campaigns', 'extensions', 'groups']); // Start with base tables
while (remaining.length > 0) {
let progress = false;
for (let i = remaining.length - 1; i >= 0; i--) {
const join = remaining[i];
// Extract table names from the join condition
const referencedTables = this.extractTablesFromCondition(join.condition || '');
// Remove the target table from dependencies (it's being joined)
const dependencies = referencedTables.filter(t => t !== join.table);
// Check if all dependencies are already joined
if (dependencies.every(dep => joinedTables.has(dep))) {
sorted.push(join);
joinedTables.add(join.table);
remaining.splice(i, 1);
progress = true;
}
}
// If no progress, add remaining joins in original order (fallback)
if (!progress) {
sorted.push(...remaining);
break;
}
}
return sorted;
}
/**
* Extract table names from a JOIN condition
*/
extractTablesFromCondition(condition) {
const tables = new Set();
const tablePattern = /(\w+)\.\w+/g;
let match;
while ((match = tablePattern.exec(condition)) !== null) {
tables.add(match[1]);
}
return Array.from(tables);
}
buildFromClause(query, intent, schema) {
query.from(schema.table);
const entityType = this.getEntityTypeFromSchema(schema);
const fieldMappings = [];
// Collect all field mappings to determine required JOINs
if (intent.filters) {
for (const filter of intent.filters) {
try {
const mapping = this.fieldMapper.resolveField(entityType, filter.field);
fieldMappings.push(mapping);
}
catch (error) {
// Ignore mapping errors for now, will be handled in buildFilterCondition
}
}
}
// CRITICAL FIX: Also collect GROUP BY field mappings for JOIN requirements
if (intent.groupBy) {
for (const groupField of intent.groupBy) {
try {
const mapping = this.fieldMapper.resolveField(entityType, groupField);
fieldMappings.push(mapping);
getLogger().debug({
groupField,
sqlField: mapping.sqlField,
requiredJoin: mapping.requiredJoin
}, 'HybridQueryBuilder: GROUP BY field mapping collected for JOIN requirements');
}
catch (error) {
// Ignore mapping errors for now, will be handled in buildGroupByClause
getLogger().debug({
groupField,
error: error instanceof Error ? error.message : String(error)
}, 'HybridQueryBuilder: GROUP BY field mapping failed, will use fallback');
}
}
}
// Get required JOINs from field mappings
const allFields = [
...(intent.filters?.map(f => f.field) || []),
...(intent.groupBy || []),
...(intent.orderBy?.map(o => o.field) || [])
];
const requiredJoins = this.fieldMapper.getRequiredJoins(entityType, allFields);
// Sort JOINs by dependency order
const sortedJoins = this.sortJoinsByDependency(requiredJoins);
// Add intelligent JOINs in sorted order
for (const join of sortedJoins) {
if (join.condition && join.type) {
query.join(join.type, join.table, join.condition);
}
if (join.condition) {
getLogger().debug({
joinType: join.type,
table: join.table,
condition: join.condition
}, 'HybridQueryBuilder: Added intelligent JOIN');
}
}
// Add legacy JOINs for related entities (fallback)
if (intent.relatedEntities && schema.joins) {
for (const relatedEntity of intent.relatedEntities) {
const joinInfo = schema.joins[relatedEntity];
if (joinInfo) {
// Check if this JOIN is already added by intelligent mapping
const alreadyAdded = requiredJoins.some(rj => rj.table === joinInfo.table);
if (!alreadyAdded) {
query.join(joinInfo.type, joinInfo.table, joinInfo.on);
}
}
}
}
// Add legacy automatic JOINs for filtering (fallback)
if (intent.filters) {
for (const filter of intent.filters) {
// If filtering on related entity field, add JOIN
const fieldParts = filter.field.split('.');
if (fieldParts.length > 1) {
const relatedTable = fieldParts[0];
if (schema.joins && schema.joins[relatedTable]) {
const joinInfo = schema.joins[relatedTable];
// Check if this JOIN is already added
const alreadyAdded = requiredJoins.some(rj => rj.table === joinInfo.table);
if (!alreadyAdded) {
query.join(joinInfo.type, joinInfo.table, joinInfo.on);
}
}
}
}
}
}
buildWhereClause(query, intent, schema) {
const conditions = [];
const params = [];
// Check if we need to add JOIN for enabled filter on flags
const hasEnabledFilter = intent.filters?.some(f => f.field === 'enabled' && intent.primaryEntity === 'flags');
if (hasEnabledFilter && !intent.relatedEntities?.includes('environments')) {
// Force the JOIN to be added
if (!intent.relatedEntities)
intent.relatedEntities = [];
intent.relatedEntities.push('environments');
}
// Add filters
if (intent.filters) {
for (const filter of intent.filters) {
const condition = this.buildFilterCondition(filter, schema, params);
if (condition) {
conditions.push(condition);
}
}
}
// Add time range filter
if (intent.timeRange) {
const timeCondition = this.buildTimeRangeCondition(intent.timeRange, schema, params);
if (timeCondition) {
conditions.push(timeCondition);
}
}
// Add default filters
if (!intent.filters?.some(f => f.field === 'archived')) {
conditions.push(`${schema.table}.archived = 0`);
}
if (conditions.length > 0) {
query.where(conditions.join(' AND '), params);
}
}
buildGroupByClause(query, intent, schema) {
if (intent.groupBy && intent.groupBy.length > 0) {
const entityType = this.getEntityTypeFromSchema(schema);
const groupByFields = [];
for (const field of intent.groupBy) {
try {
// Use IntelligentFieldMapper to resolve complex field references
const fieldMapping = this.fieldMapper.resolveField(entityType, field);
getLogger().debug({
originalField: field,
resolvedField: fieldMapping.sqlField,
requiresJoin: !!fieldMapping.requiredJoin,
requiresJson: fieldMapping.requiresJsonProcessing
}, 'HybridQueryBuilder: GROUP BY field mapping resolved');
// For JSON fields, we need special handling in GROUP BY
if (fieldMapping.requiresJsonProcessing && fieldMapping.jsonataPath) {
// Special case: grouping by array elements (e.g., variations.key)
if (fieldMapping.jsonataPath.includes('variations.') && entityType === 'experiments') {
// This requires expanding the variations array - mark for hybrid processing
getLogger().info({
field,
jsonataPath: fieldMapping.jsonataPath,
message: 'GROUP BY on array element detected - will use hybrid processing'
}, 'HybridQueryBuilder: Array GROUP BY requires post-processing');
// For now, skip adding to SQL GROUP BY - will handle in post-processing
// Mark the query as requiring hybrid processing
query.__requiresHybridProcessing = true;
query.__hybridGroupBy = field;
}
else {
// For non-array JSON fields, use json_extract
const jsonPath = fieldMapping.jsonataPath.replace(/\./g, '.');
const jsonExtractExpr = `json_extract(${fieldMapping.sqlField}, '$.${jsonPath}')`;
groupByFields.push(jsonExtractExpr);
}
}
else {
groupByFields.push(fieldMapping.sqlField);
}
}
catch (error) {
// Fallback to original logic if mapping fails
getLogger().debug({
field,
error: error instanceof Error ? error.message : String(error)
}, 'HybridQueryBuilder: GROUP BY field mapping failed, using fallback');
// Original fallback logic
if (schema.groupable?.includes(field) || schema.columns.includes(field)) {
groupByFields.push(field.includes('.') ? field : `${schema.table}.${field}`);
}
}
}
if (groupByFields.length > 0) {
query.groupBy(groupByFields);
}
}
}
buildOrderByClause(query, intent, schema) {
if (intent.orderBy && intent.orderBy.length > 0) {
for (const orderBy of intent.orderBy) {
// Prefix field with table name if not already prefixed
const field = orderBy.field.includes('.') ? orderBy.field :
(orderBy.field === 'count' ? orderBy.field : `${schema.table}.${orderBy.field}`);
query.orderBy(field, orderBy.direction);
}
}
else if (intent.groupBy && intent.groupBy.length > 0) {
// Default ordering for grouped results
query.orderBy('count', 'desc');
}
}
buildLimitClause(query, intent) {
const limit = intent.limit || QUERY_LIMITS.defaultLimit;
const offset = intent.offset || 0;
query.limit(Math.min(limit, QUERY_LIMITS.maxLimit), offset);
}
buildFilterCondition(filter, schema, params) {
let field;
try {
// Use IntelligentFieldMapper to resolve complex field references
const fieldMapping = this.fieldMapper.resolveField(this.getEntityTypeFromSchema(schema), filter.field);
getLogger().debug({
originalField: filter.field,
resolvedField: fieldMapping.sqlField,
requiresJoin: !!fieldMapping.requiredJoin,
requiresJson: fieldMapping.requiresJsonProcessing
}, 'HybridQueryBuilder: Field mapping resolved');
// If this field requires JSON processing, delegate to JSON filter builder
if (fieldMapping.requiresJsonProcessing) {
return this.buildJsonFilterCondition(filter, params);
}
// Use the resolved SQL field
field = fieldMapping.sqlField;
}
catch (mappingError) {
getLogger().debug({
field: filter.field,
error: mappingError.message
}, 'HybridQueryBuilder: Field mapping failed, falling back to legacy logic');
// Fallback to legacy field mapping
field = filter.field;
// Special handling for flags.enabled - transform to proper JOIN condition
if (field === 'enabled' && schema.table === 'flags') {
// Return a condition that will be applied after JOIN with flag_environments
params.push(filter.value);
return `flag_environments.enabled = ?`;
}
// Handle JSON path filters or json_contains operator
if (filter.jsonPath || field.includes('.') || filter.operator === 'json_contains') {
return this.buildJsonFilterCondition(filter, params);
}
// Validate field exists
if (!schema.columns.includes(field) && !schema.groupable?.includes(field)) {
// Try to map to a valid field
const mappedField = this.mapToValidField(field, schema);
if (!mappedField)
return null;
field = mappedField;
}
// Prefix field with table name if not already prefixed
if (!field.includes('.')) {
field = `${schema.table}.${field}`;
}
}
switch (filter.operator) {
case 'eq':
params.push(filter.value);
return `${field} = ?`;
case 'ne':
params.push(filter.value);
return `${field} != ?`;
case 'gt':
params.push(filter.value);
return `${field} > ?`;
case 'lt':
params.push(filter.value);
return `${field} < ?`;
case 'gte':
params.push(filter.value);
return `${field} >= ?`;
case 'lte':
params.push(filter.value);
return `${field} <= ?`;
case 'in':
const placeholders = filter.value.map(() => '?').join(', ');
params.push(...filter.value);
return `${field} IN (${placeholders})`;
case 'not_in':
const notInPlaceholders = filter.value.map(() => '?').join(', ');
params.push(...filter.value);
return `${field} NOT IN (${notInPlaceholders})`;
case 'contains':
params.push(`%${filter.value}%`);
return `${field} LIKE ?`;
case 'not_contains':
params.push(`%${filter.value}%`);
return `${field} NOT LIKE ?`;
case 'exists':
return `${field} IS NOT NULL`;
case 'not_exists':
return `${field} IS NULL`;
default:
return null;
}
}
buildJsonFilterCondition(filter, params) {
const jsonPath = filter.jsonPath || filter.field;
const pathParts = jsonPath.split('.');
// Determine which JSON column to use
let jsonColumn = 'data_json';
if (pathParts[0] === 'conditions') {
jsonColumn = 'conditions';
}
else if (pathParts[0] === 'variables') {
jsonColumn = 'variables';
}
// Build JSON path for SQLite
const pathElements = pathParts.slice(jsonColumn === 'data_json' ? 0 : 1);
const sqlitePath = pathElements.length > 0 ? `$.${pathElements.join('.')}` : `$`;
switch (filter.operator) {
case 'exists':
return `JSON_EXTRACT(${jsonColumn}, '${sqlitePath}') IS NOT NULL`;
case 'not_exists':
return `JSON_EXTRACT(${jsonColumn}, '${sqlitePath}') IS NULL`;
case 'eq':
params.push(JSON.stringify(filter.value));
return `JSON_EXTRACT(${jsonColumn}, '${sqlitePath}') = ?`;
case 'contains':
params.push(`%${filter.value}%`);
return `JSON_EXTRACT(${jsonColumn}, '${sqlitePath}') LIKE ?`;
case 'array_contains':
// Use JSON array contains check
params.push(JSON.stringify(filter.value));
return `EXISTS (SELECT 1 FROM json_each(${jsonColumn}, '${sqlitePath}') WHERE value = ?)`;
case 'array_length':
params.push(filter.value);
return `JSON_ARRAY_LENGTH(${jsonColumn}, '${sqlitePath}') ${filter.value.operator || '='} ?`;
case 'json_contains':
// Check if any of the provided values exist as keys in the JSON object
if (Array.isArray(filter.value)) {
const conditions = filter.value.map(val => {
params.push(val);
return `JSON_EXTRACT(${jsonColumn}, '${sqlitePath}' || '.' || ?) IS NOT NULL`;
});
return `(${conditions.join(' OR ')})`;
}
else {
params.push(filter.value);
return `JSON_EXTRACT(${jsonColumn}, '${sqlitePath}' || '.' || ?) IS NOT NULL`;
}
default:
return `JSON_EXTRACT(${jsonColumn}, '${sqlitePath}') IS NOT NULL`;
}
}
buildTimeRangeCondition(timeRange, schema, params) {
// Determine time field based on entity
let timeField = 'created_time';
if (schema.aggregatable?.includes('updated_time')) {
timeField = 'updated_time';
}
if (schema.aggregatable?.includes('timestamp')) {
timeField = 'timestamp';
}
const conditions = [];
if (timeRange.start) {
params.push(timeRange.start);
conditions.push(`${timeField} >= ?`);
}
if (timeRange.end) {
params.push(timeRange.end);
conditions.push(`${timeField} <= ?`);
}
if (timeRange.relative) {
const now = new Date();
const { value, unit } = timeRange.relative;
switch (unit) {
case 'hours':
now.setHours(now.getHours() - value);
break;
case 'days':
now.setDate(now.getDate() - value);
break;
case 'weeks':
now.setDate(now.getDate() - (value * 7));
break;
case 'months':
now.setMonth(now.getMonth() - value);
break;
}
params.push(now.toISOString());
conditions.push(`${timeField} >= ?`);
}
return conditions.length > 0 ? `(${conditions.join(' AND ')})` : null;
}
mapToValidField(field, schema) {
// Common field mappings
const mappings = {
'created': 'created_time',
'updated': 'updated_time',
'modified': 'updated_time',
'last_modified': 'updated_time',
'is_archived': 'archived',
'status': 'status',
'type': 'type',
'platform': 'platform'
};
return mappings[field] || null;
}
getEntityTypeFromSchema(schema) {
// Map schema table names to entity types
const tableToEntityMap = {
'flags': 'flags',
'experiments': 'experiments',
'audiences': 'audiences',
'campaigns': 'campaigns',
'pages': 'pages',
'projects': 'projects'
};
return tableToEntityMap[schema.table] || 'flags';
}
requiresJsonProcessing(intent) {
// Check if we need JSONata processing
if (intent.jsonPaths && intent.jsonPaths.length > 0)
return true;
if (intent.jsonFilters && intent.jsonFilters.length > 0)
return true;
if (intent.transforms && intent.transforms.length > 0)
return true;
// Check if GROUP BY includes array elements (e.g., variations.key)
if (intent.groupBy) {
for (const field of intent.groupBy) {
if (field.includes('variations.') && intent.primaryEntity === 'experiments') {
return true; // Requires hybrid processing for array expansion
}
}
}
// Phase 2: Enhanced conditions for complex queries
if (intent.groupBy && intent.groupBy.includes('environment'))
return true;
if (intent.action === 'analyze' && intent.metrics?.includes('complexity'))
return true;
if (intent.relatedEntities && intent.relatedEntities.length > 0)
return true;
if (intent.metrics?.some(m => m.includes('traffic') || m.includes('allocation')))
return true;
// Check if filters require JSON processing
if (intent.filters) {
for (const filter of intent.filters) {
if (filter.jsonPath || filter.field.includes('.'))
return true;
if (['array_contains', 'array_length'].includes(filter.operator))
return true;
}
}
// Check if aggregations require JSON processing
if (intent.aggregations) {
for (const agg of intent.aggregations) {
if (['count_if', 'percent'].includes(agg))
return true;
}
}
return false;
}
buildProcessingPipeline(intent) {
const pipeline = [];
// Handle array GROUP BY (e.g., variations.key)
if (intent.groupBy) {
for (const field of intent.groupBy) {
if (field.includes('variations.') && intent.primaryEntity === 'experiments') {
// Add a transform to expand variations array and group
pipeline.push({
type: 'transform',
operation: 'expand_and_group',
params: {
arrayPath: 'variations',
groupByField: field.split('.').pop(), // 'key' from 'variations.key'
aggregation: 'count'
}
});
}
}
}
// Add JSON filtering step
if (intent.jsonFilters && intent.jsonFilters.length > 0) {
pipeline.push({
type: 'filter',
operation: 'jsonata_filter',
params: intent.jsonFilters
});
}
// Add transformation steps
if (intent.transforms) {
for (const transform of intent.transforms) {
pipeline.push({
type: 'transform',
operation: transform.type,
params: transform.config
});
}
}
// Add aggregation step if needed
if (intent.aggregations && this.requiresPostAggregation(intent)) {
pipeline.push({
type: 'aggregate',
operation: 'jsonata_aggregate',
params: intent.aggregations
});
}
// Add final sorting if needed
if (intent.orderBy && pipeline.length > 0) {
pipeline.push({
type: 'sort',
operation: 'sort',
params: intent.orderBy
});
}
// Add limit if processing changed result count
if (intent.limit && pipeline.some(p => p.type === 'filter' || p.type === 'aggregate')) {
pipeline.push({
type: 'limit',
operation: 'limit',
params: { limit: intent.limit, offset: intent.offset || 0 }
});
}
return pipeline;
}
requiresPostAggregation(intent) {
// Check if aggregations can't be done in SQL
if (!intent.aggregations)
return false;
for (const agg of intent.aggregations) {
if (['count_if', 'percent'].includes(agg))
return true;
}
// If grouping by JSON fields, need post-aggregation
if (intent.groupBy) {
for (const field of intent.groupBy) {
if (field.includes('.'))
return true;
}
}
return false;
}
initializeSchemaMap() {
const baseSchema = {
projects: {
table: 'projects',
columns: ['id', 'name', 'description', 'platform', 'status', 'account_id',
'is_flags_enabled', 'archived', 'created_at', 'last_modified', 'data_json'],
jsonColumns: ['data_json'],
aggregatable: ['created_at', 'last_modified'],
groupable: ['platform', 'status', 'archived', 'account_id']
},
flags: {
table: 'flags',
columns: ['project_id', 'key', 'id', 'name', 'description', 'archived',
'created_time', 'updated_time', 'data_json'],
jsonColumns: ['data_json'],
joins: {
environments: {
table: 'flag_environments',
on: 'flags.project_id = flag_environments.project_id AND flags.key = flag_environments.flag_key',
type: 'LEFT'
},
variations: {
table: 'variations',
on: 'flags.project_id = variations.project_id AND flags.key = variations.flag_key',
type: 'LEFT'
},
rules: {
table: 'rules',
on: 'flags.project_id = rules.project_id AND flags.key = rules.flag_key',
type: 'LEFT'
},
rulesets: {
table: 'rulesets',
on: 'flags.project_id = rulesets.project_id AND flags.key = rulesets.flag_key',
type: 'LEFT'
}
},
aggregatable: ['created_time', 'updated_time'],
groupable: ['archived', 'project_id'],
countable: {
variation_count: 'JSON_ARRAY_LENGTH(flags.data_json, \'$.variations\')',
environment_count: 'COUNT(DISTINCT flag_environments.environment_key)',
rule_count: 'COUNT(DISTINCT rules.id)',
variable_count: 'JSON_ARRAY_LENGTH(flags.data_json, \'$.variable_definitions\')'
}
},
experiments: {
table: 'experiments',
columns: ['id', 'project_id', 'name', 'description', 'status', 'flag_key',
'environment', 'type', 'archived', 'created_time', 'updated_time', 'data_json'],
jsonColumns: ['data_json'],
joins: {
variations: {
table: 'variations',
on: 'experiments.flag_key = variations.flag_key AND experiments.project_id = variations.project_id',
type: 'LEFT'
},
results: {
table: 'experiment_results',
on: 'experiments.id = experiment_results.experiment_id',
type: 'LEFT'
},
campaigns: {
table: 'campaigns',
on: 'JSON_EXTRACT(experiments.data_json, \'$.campaign_id\') = campaigns.id',
type: 'LEFT'
},
audiences: {
table: 'audiences',
on: 'EXISTS (SELECT 1 FROM json_each(experiments.data_json, \'$.audience_conditions\') WHERE value = audiences.id)',
type: 'LEFT'
}
},
aggregatable: ['created_time', 'updated_time'],
groupable: ['status', 'type', 'environment', 'archived', 'project_id'],
countable: {
variation_count: 'JSON_ARRAY_LENGTH(experiments.data_json, \'$.variations\')',
audience_count: 'JSON_ARRAY_LENGTH(experiments.data_json, \'$.audience_conditions\')',
page_count: 'JSON_ARRAY_LENGTH(experiments.data_json, \'$.page_ids\')'
}
},
audiences: {
table: 'audiences',
columns: ['id', 'project_id', 'name', 'description', 'conditions', 'archived',
'created_time', 'last_modified', 'data_json'],
jsonColumns: ['conditions', 'data_json'],
joins: {
experiments: {
table: 'experiments',
on: 'EXISTS (SELECT 1 FROM json_each(experiments.data_json, \'$.audience_conditions\') WHERE value = audiences.id)',
type: 'LEFT'
}
},
aggregatable: ['created_time', 'last_modified'],
groupable: ['archived', 'project_id'],
countable: {
experiment_count: 'COUNT(DISTINCT experiments.id)',
condition_count: 'JSON_ARRAY_LENGTH(audiences.conditions)'
}
},
variations: {
table: 'variations',
columns: ['id', 'key', 'project_id', 'flag_key', 'name', 'description',
'enabled', 'archived', 'variables', 'created_time', 'updated_time', 'data_json'],
jsonColumns: ['variables', 'data_json'],
joins: {
flags: {
table: 'flags',
on: 'variations.project_id = flags.project_id AND variations.flag_key = flags.key',
type: 'LEFT'
}
},
aggregatable: ['created_time', 'updated_time'],
groupable: ['enabled', 'archived', 'project_id', 'flag_key'],
countable: {
variable_count: 'JSON_ARRAY_LENGTH(variations.variables)'
}
},
rules: {
table: 'rules',
columns: ['id', 'key', 'project_id', 'flag_key', 'environment_key', 'type',
'name', 'audience_conditions', 'percentage_included', 'variations',
'metrics', 'enabled', 'created_time', 'updated_time', 'data_json'],
jsonColumns: ['audience_conditions', 'variations', 'metrics', 'data_json'],
joins: {
rulesets: {
table: 'rulesets',
on: 'rules.project_id = rulesets.project_id AND rules.flag_key = rulesets.flag_key AND rules.environment_key = rulesets.environment_key',
type: 'LEFT'
}
},
aggregatable: ['created_time', 'updated_time'],
groupable: ['type', 'enabled', 'project_id', 'flag_key', 'environment_key'],
countable: {
audience_condition_count: 'JSON_ARRAY_LENGTH(rules.audience_conditions)',
variation_count: 'JSON_ARRAY_LENGTH(rules.variations)',
metric_count: 'JSON_ARRAY_LENGTH(rules.metrics)'
}
},
events: {
table: 'events',
columns: ['id', 'project_id', 'key', 'name', 'description', 'event_type',
'category', 'archived', 'created_time', 'data_json'],
jsonColumns: ['data_json'],
joins: {},
aggregatable: ['created_time'],
groupable: ['event_type', 'category', 'archived', 'project_id'],
countable: {}
},
attributes: {
table: 'attributes',
columns: ['id', 'project_id', 'key', 'name', 'condition_type', 'archived',
'last_modified', 'data_json'],
jsonColumns: ['data_json'],
joins: {},
aggregatable: ['last_modified'],
groupable: ['condition_type', 'archived', 'project_id'],
countable: {}
},
pages: {
table: 'pages',
columns: ['id', 'project_id', 'name', 'edit_url', 'activation_mode',
'activation_code', 'conditions', 'archived', 'created_time',
'updated_time', 'data_json'],
jsonColumns: ['conditions', 'data_json'],
joins: {},
aggregatable: ['created_time', 'updated_time'],
groupable: ['activation_mode', 'archived', 'project_id'],
countable: {}
},
experiment_results: {
table: 'experiment_results',
columns: ['id', 'experiment_id', 'project_id', 'confidence_level',
'use_stats_engine', 'stats_engine_version', 'baseline_count',
'treatment_count', 'total_count', 'start_time', 'last_update',
'results_json', 'reach_json', 'stats_config_json', 'data_json'],
jsonColumns: ['results_json', 'reach_json', 'stats_config_json', 'data_json'],
joins: {
experiments: {
table: 'experiments',
on: 'experiment_results.experiment_id = experiments.id',
type: 'LEFT'
}
},
aggregatable: ['start_time', 'last_update', 'confidence_level', 'total_count', 'baseline_count', 'treatment_count'],
groupable: ['experiment_id', 'project_id', 'use_stats_engine'],
countable: {}
},
environments: {
table: 'environments',
columns: ['project_id', 'key', 'name', 'is_primary', 'priority', 'archived', 'data_json'],
jsonColumns: ['data_json'],
joins: {},
aggregatable: [],
groupable: ['is_primary', 'archived', 'project_id'],
countable: {}
},
rulesets: {
table: 'rulesets',
columns: ['id', 'project_id', 'flag_key', 'environment_key', 'enabled',
'rules_count', 'revision', 'created_time', 'updated_time', 'data_json'],
jsonColumns: ['data_json'],
joins: {
rules: {
table: 'rules',
on: 'rulesets.project_id = rules.project_id AND rulesets.flag_key = rules.flag_key AND rulesets.environment_key = rules.environment_key',
type: 'LEFT'
}
},
aggregatable: ['created_time', 'updated_time'],
groupable: ['enabled', 'project_id', 'flag_key', 'environment_key'],
countable: {
rule_count: 'COUNT(DISTINCT rules.id)'
}
},
changes: {
table: 'change_history',
columns: ['id', 'project_id', 'entity_type', 'entity_id', 'entity_name',
'action', 'timestamp', 'changed_by', 'change_summary', 'archived'],
joins: {},
aggregatable: ['timestamp'],
groupable: ['entity_type', 'action', 'changed_by', 'project_id'],
countable: {}
},
campaigns: {
table: 'campaigns',
columns: ['id', 'name', 'status', 'description', 'holdback', 'page_ids',
'metrics', 'project_id', 'created', 'last_modified', 'archived', 'data_json'],
jsonColumns: ['metrics', 'page_ids', 'data_json'],
joins: {
experiments: {
table: 'experiments',
on: 'campaigns.id = experiments.campaign_id',
type: 'LEFT'
}
},
aggregatable: ['created', 'last_modified'],
groupable: ['status', 'archived', 'project_id'],
countable: {
experiment_count: 'COUNT(DISTINCT experiments.id)'
}
}
};
// Add singular forms that reference the same schema as plural forms
const schemaWithSingular = {
...baseSchema,
// Singular forms (alias to plural)
flag: baseSchema.flags,
experiment: baseSchema.experiments,
audience: baseSchema.audiences,
variation: baseSchema.variations,
rule: baseSchema.rules,
event: baseSchema.events,
attribute: baseSchema.attributes,
page: baseSchema.pages,
project: baseSchema.projects,
environment: baseSchema.environments,
ruleset: baseSchema.rulesets,
campaign: baseSchema.campaigns
};
return schemaWithSingular;
}
}
/**
* Internal SQL query builder helper
*/
class SQLQueryBuilder {
selectClause = [];
fromClause = '';
joins = [];
whereClause = '';
groupByClause = [];
orderByClause = [];
limitClause = '';
params = [];
select(fields) {
this.selectClause = fields;
return this;
}
from(table) {
this.fromClause = table;
return this;
}
join(type, table, on, alias) {
// Avoid duplicate joins
const tableRef = alias && alias !== table ? `${table} AS ${alias}` : table;
const joinStr = `${type} JOIN ${tableRef} ON ${on}`;
if (!this.joins.includes(joinStr)) {
this.joins.push(joinStr);
}
return this;
}
where(condition, params) {
this.whereClause = condition;
if (params) {
this.params.push(...params);
}
return this;
}
groupBy(fields) {
this.groupByClause = fields;
return this;
}
orderBy(field, direction = 'asc') {
this.orderByClause.push(`${field} ${direction.toUpperCase()}`);
return this;
}
limit(limit, offset = 0) {
this.limitClause = `LIMIT ${limit}`;
if (offset > 0) {
this.limitClause += ` OFFSET ${offset}`;
}
return this;
}
compile() {
const parts = [];
// SELECT
parts.push(`SELECT ${this.selectClause.join(', ')}`);
// FROM
parts.push(`FROM ${this.fromClause}`);
// JOINs
if (this.joins.length > 0) {
parts.push(...this.joins);
}
// WHERE
if (this.whereClause) {
parts.push(`WHERE ${this.whereClause}`);
}
// GROUP BY
if (this.groupByClause.length > 0) {
parts.push(`GROUP BY ${this.groupByClause.join(', ')}`);
}
// ORDER BY
if (this.orderByClause.length > 0) {
parts.push(`ORDER BY ${this.orderByClause.join(', ')}`);
}
// LIMIT
if (this.limitClause) {
parts.push(this.limitClause);
}
const sql = parts.join('\n');
const compiledQuery = {
sql,
params: this.params,
metadata: {
tables: this.extractTables(sql),
complexity: this.calculateComplexity()
}
};
// Pass along any special processing requirements
if (this.__requiresHybridProcessing) {
compiledQuery.__requiresHybridProcessing = true;
compiledQuery.__hybridGroupBy = this.__hybridGroupBy;
}
return compiledQuery;
}
extractTables(sql) {
const tables = [this.fromClause];
// Extract tables from joins
const joinMatches = sql.matchAll(/JOIN\s+(\w+)\s+ON/gi);
for (const match of joinMatches) {
if (match[1] && !tables.includes(match[1])) {
tables.push(match[1]);
}
}
return tables;
}