@simonecoelhosfo/optimizely-mcp-server
Version:
Optimizely MCP Server for AI assistants with integrated CLI tools
957 lines (956 loc) • 67.1 kB
JavaScript
/**
* SQL Builder for Intelligent Query Engine
*
* Builds SQL queries using field location information from the Field Catalog
*/
import { getLogger } from '../../logging/Logger.js';
import { DateFunctionHandler } from './DateFunctionHandler.js';
import { FieldDisambiguator } from './FieldDisambiguator.js';
import { JSONPathHandler } from './JSONPathHandler.js';
import { CardinalityGuard } from './CardinalityGuard.js';
import { QueryComplexityFirewall } from '../QueryComplexityFirewall.js';
import { entityTableMapper } from './EntityTableMapper.js';
import { ViewOnlySQLBuilder } from './ViewOnlySQLBuilder.js';
const logger = getLogger();
export class SQLBuilder {
fieldCatalog;
requiredJoins = new Set();
joinClauses = new Map();
dateHandler;
fieldDisambiguator;
jsonHandler;
cardinalityGuard;
complexityFirewall;
currentQuery = null;
viewOnlyBuilder;
useViewOnlyMode = true; // Default to view-only mode
constructor(fieldCatalog) {
this.fieldCatalog = fieldCatalog;
this.dateHandler = new DateFunctionHandler();
this.fieldDisambiguator = new FieldDisambiguator(fieldCatalog);
this.jsonHandler = new JSONPathHandler();
this.cardinalityGuard = new CardinalityGuard({
enabled: true,
debugMode: true,
autoDistinct: true
});
this.complexityFirewall = new QueryComplexityFirewall({
enabled: true,
debugMode: true,
enableFallbacks: true,
enableLearning: true
});
this.viewOnlyBuilder = new ViewOnlySQLBuilder();
// Check environment variable to override view-only mode
if (process.env.ANALYTICS_DISABLE_VIEW_ONLY === 'true') {
this.useViewOnlyMode = false;
logger.warn('[SQLBuilder] View-only mode disabled via environment variable');
}
}
/**
* Build SQL query from UniversalQuery
*/
async buildSQL(query) {
logger.debug(`Building SQL for entity: ${query.find}`);
// If in view-only mode, delegate to ViewOnlySQLBuilder
if (this.useViewOnlyMode) {
logger.info(`[SQLBuilder] Using VIEW-ONLY mode for query on entity: ${query.find}`);
try {
return await this.viewOnlyBuilder.buildSQL(query);
}
catch (error) {
// Log the error and re-throw - NO FALLBACK to legacy mode
logger.error(`[SQLBuilder] View-only query failed: ${error instanceof Error ? error.message : String(error)}`);
throw error;
}
}
// Legacy mode (only if explicitly disabled)
logger.warn(`[SQLBuilder] Using LEGACY mode with complex JOINs for entity: ${query.find}`);
// Reset state
this.requiredJoins.clear();
this.joinClauses.clear();
this.currentQuery = query;
// Always disambiguate fields - no bypassing!
const disambiguationResult = this.fieldDisambiguator.disambiguateQuery(query);
if (disambiguationResult.errors.length > 0) {
throw new Error(`Field disambiguation errors: ${disambiguationResult.errors.join(', ')}`);
}
// Apply disambiguation to query
const disambiguatedQuery = this.fieldDisambiguator.applyDisambiguation(query, disambiguationResult);
// Log warnings if any
if (disambiguationResult.warnings.length > 0) {
logger.warn(`Field disambiguation warnings: ${disambiguationResult.warnings.join(', ')}`);
}
// L1-3 DEBUG: Check if disambiguation changed the select fields
// DEBUG buildSQL: original query.select = ${JSON.stringify(query.select)}
// DEBUG buildSQL: disambiguatedQuery.select = ${JSON.stringify(disambiguatedQuery.select)}
// Build SELECT clause
const selectClause = await this.buildSelectClause(disambiguatedQuery);
// Build FROM clause
const fromClause = this.buildFromClause(disambiguatedQuery);
// Build WHERE clause (may add joins)
const whereClause = await this.buildWhereClause(disambiguatedQuery);
// Build GROUP BY clause (may add joins)
const groupByClause = await this.buildGroupByClause(disambiguatedQuery);
// Build JOIN clauses
const joinClause = this.buildJoinClauses(disambiguatedQuery);
// CRITICAL: Validate JOIN cardinality to prevent explosions
const cardinalityValidation = this.validateQueryCardinality(disambiguatedQuery, joinClause);
// L2-1 DEBUG: Track cardinality validation
logger.info(`L2-1 DEBUG: Cardinality validation result:`);
logger.info(`L2-1 DEBUG: - isValid: ${cardinalityValidation.isValid}`);
logger.info(`L2-1 DEBUG: - totalMultiplier: ${cardinalityValidation.totalMultiplier}`);
logger.info(`L2-1 DEBUG: - Has GROUP BY: ${groupByClause !== ''}`);
logger.info(`L2-1 DEBUG: - GROUP BY clause: ${groupByClause}`);
logger.info(`L2-1 DEBUG: - Fallback SQL: ${cardinalityValidation.fallbackSQL}`);
if (!cardinalityValidation.isValid && cardinalityValidation.fallbackSQL) {
logger.warn(`Query rejected due to cardinality risk (${cardinalityValidation.totalMultiplier}x), using fallback`);
logger.warn(`L2-1 DEBUG: GROUP BY query is being replaced with fallback SQL!`);
return cardinalityValidation.fallbackSQL;
}
// CRITICAL: Final complexity firewall analysis (Third line of defense)
const complexityAnalysis = this.analyzeQueryComplexity(disambiguatedQuery, joinClause, cardinalityValidation);
// DEBUG complexity firewall: allowed=${complexityAnalysis.allowed}, score=${complexityAnalysis.complexity.score}
if (!complexityAnalysis.allowed && complexityAnalysis.fallback) {
logger.warn(`Query blocked by complexity firewall (score: ${complexityAnalysis.complexity.score}), using fallback`);
// DEBUG: Using fallback SQL: ${complexityAnalysis.fallback.sql}
if (complexityAnalysis.recommendations.length > 0) {
logger.info(`Complexity recommendations: ${complexityAnalysis.recommendations.join(', ')}`);
}
return complexityAnalysis.fallback.sql;
}
// Build ORDER BY clause
const orderByClause = await this.buildOrderByClause(disambiguatedQuery);
// Build HAVING clause
const havingClause = this.buildHavingClause(query);
// Assemble final SQL
let sql = `SELECT ${selectClause}\nFROM ${fromClause}`;
if (joinClause) {
sql += `\n${joinClause}`;
}
// Apply cardinality protection (inject DISTINCT if needed)
if (cardinalityValidation.totalMultiplier > 1.5) {
sql = this.cardinalityGuard.injectDistinctIfNeeded(sql, cardinalityValidation.totalMultiplier, entityTableMapper.toTableName(query.find));
}
if (whereClause) {
sql += `\nWHERE ${whereClause}`;
}
if (groupByClause) {
sql += `\nGROUP BY ${groupByClause}`;
}
if (havingClause) {
sql += `\nHAVING ${havingClause}`;
}
if (orderByClause) {
sql += `\nORDER BY ${orderByClause}`;
}
if (query.limit) {
sql += `\nLIMIT ${query.limit}`;
}
if (query.offset) {
sql += `\nOFFSET ${query.offset}`;
}
logger.info(`Generated SQL: ${sql}`);
// Log SQL for COUNT queries to debug empty results
if (query.aggregations && query.aggregations.length > 0) {
logger.debug('COUNT SQL =', sql);
}
return sql;
}
/**
* Build SELECT clause with field mapping
*/
async buildSelectClause(query) {
const mappedFields = [];
// CRITICAL L6-7 FIX: Check if this is a "list with aggregations" query
// The action field is in the parse result, not the universal query
const isListWithAggregations = query.aggregations && query.aggregations.length > 0 && query.groupBy && query.groupBy.length > 0;
// Handle regular fields first for list queries with aggregations
if (isListWithAggregations && query.select && query.select.length > 0 && query.select[0] !== '*') {
// Add the regular fields for GROUP BY
for (const field of query.select) {
if (!field.includes('(')) { // Skip aggregation functions
const convertedField = this.convertProblemFields(field, true); // Enable JOIN addition
mappedFields.push(convertedField);
}
}
}
// CRITICAL FIX: Handle aggregations
if (query.aggregations && query.aggregations.length > 0) {
for (const agg of query.aggregations) {
const aggField = agg.field === '*' ? '*' : agg.field;
const aggFunction = agg.function.toUpperCase();
const aggAlias = agg.alias || `${aggFunction.toLowerCase()}_${aggField}`;
// Build aggregation expression
if (aggField === '*') {
mappedFields.push(`${aggFunction}(*) as ${aggAlias}`);
}
else {
// CRITICAL FIX: If field matches the entity name, use COUNT(*) instead of COUNT(entityName)
// This fixes "no such column: flags" error when parsing "count flags"
const isEntityCount = aggField === query.find && aggFunction === 'COUNT';
if (isEntityCount) {
mappedFields.push(`${aggFunction}(*) as ${aggAlias}`);
}
else {
// Handle field with proper table qualification if needed
let qualifiedField = aggField.includes('.') ? aggField : aggField;
// CRITICAL FIX: Convert flags.status to flags.archived
qualifiedField = this.convertProblemFields(qualifiedField);
mappedFields.push(`${aggFunction}(${qualifiedField}) as ${aggAlias}`);
}
}
}
// If we have aggregations, we might also need GROUP BY fields
if (query.groupBy && query.groupBy.length > 0) {
for (const groupField of query.groupBy) {
if (!mappedFields.some(f => f.includes(groupField))) {
// CRITICAL FIX: Convert problematic fields in GROUP BY
const convertedField = this.convertProblemFields(groupField, true); // Enable JOIN addition
mappedFields.push(convertedField);
}
}
}
// For non-list queries with aggregations, return early
if (!isListWithAggregations) {
return mappedFields.join(', ');
}
}
// Original logic for non-aggregation queries
// DEBUG buildSelectClause: query.select = ${JSON.stringify(query.select)}
const selectFields = query.select || ['*'];
// DEBUG buildSelectClause: selectFields = ${JSON.stringify(selectFields)}
if (selectFields[0] === '*') {
// DEBUG buildSelectClause: Returning '*' because selectFields[0] === '*'
return '*';
}
// CRITICAL FIX: Detect joins from both explicit query joins AND field mappings
const hasJoins = this.requiredJoins.size > 0 || Boolean(query.joins && query.joins.length > 0);
for (const field of selectFields) {
// Handle JSON functions specially
if (field.includes('(') && field.includes(')')) {
const processedField = this.processJSONFunction(field, hasJoins);
mappedFields.push(processedField);
continue;
}
// Handle aliased fields
const aliasMatch = field.match(/^(.+?)\s+as\s+(.+)$/i);
const fieldName = aliasMatch ? aliasMatch[1].trim() : field;
const alias = aliasMatch ? aliasMatch[2].trim() : null;
// Check if this is a JSON path
if (this.jsonHandler.isJSONPath(fieldName) && fieldName.includes('$')) {
// Extract base field and JSON path
const baseField = this.jsonHandler.getBaseFieldName(fieldName);
const jsonExtraction = this.jsonHandler.generateJSONExtractSQL('data_json', fieldName, alias || undefined);
mappedFields.push(jsonExtraction);
}
// Check if field already has table prefix
else if (fieldName.includes('.')) {
const [entityOrTable, ...fieldParts] = fieldName.split('.');
const fieldPart = fieldParts.join('.');
const tableName = entityTableMapper.toTableName(entityOrTable);
// Check if the field part is a JSON path
if (fieldParts.length > 1 && this.jsonHandler.isJSONPath(fieldPart)) {
// This is like flag.environments.production
const jsonExtraction = this.jsonHandler.generateJSONExtractSQL(hasJoins ? `${tableName}.data_json` : 'data_json', `$.${fieldPart}`, alias || undefined);
mappedFields.push(jsonExtraction);
}
else {
// Regular table.field reference
const rawField = hasJoins ? `${tableName}.${fieldPart}` : fieldPart;
// CRITICAL FIX: Apply field conversion for consistency with WHERE clause
const sqlField = this.convertProblemFields(rawField);
if (alias) {
mappedFields.push(`${sqlField} as ${alias}`);
}
else {
mappedFields.push(sqlField);
}
}
}
else {
// Simple field name - apply field conversion for consistency with WHERE clause
const convertedField = this.convertProblemFields(fieldName);
if (alias) {
mappedFields.push(`${convertedField} as ${alias}`);
}
else {
mappedFields.push(convertedField);
}
}
}
const result = mappedFields.join(', ');
// DEBUG buildSelectClause final return: "${result}"
return result;
}
/**
* Build FROM clause
*/
buildFromClause(query) {
// Detect environment comparison queries and use optimal table
const entity = query.find;
// For environment comparison queries, use flag_environments as primary table
// This avoids self-JOINs to environments table and resolves column ambiguity
if (entity === 'environments' && this.isEnvironmentComparisonQuery(query)) {
// L4-2 FIX: Environment comparison detected, using flag_environments as primary table
return 'flag_environments';
}
// Use the mapper to get the correct table name
return entityTableMapper.toTableName(query.find);
}
/**
* Detect if this is an environment comparison query
*/
isEnvironmentComparisonQuery(query) {
// Look for conditions that indicate environment comparison:
// 1. Multiple environment_key conditions (development vs production)
// 2. Fields that exist in flag_environments (enabled, environment_key)
// 3. WHERE conditions referencing flag environment fields
if (!query.where)
return false;
let hasEnvironmentFields = false;
let hasEnvironmentKeyCondition = false;
for (const condition of query.where) {
// Check for flag-environment specific fields
if (condition.field === 'enabled' ||
condition.field === 'flag_environments.enabled' ||
condition.field === 'environment_key' ||
condition.field === 'flag_environments.environment_key') {
hasEnvironmentFields = true;
}
// Check for environment_key filtering
if (condition.field.includes('environment_key')) {
hasEnvironmentKeyCondition = true;
}
}
return hasEnvironmentFields && hasEnvironmentKeyCondition;
}
/**
* Build WHERE clause with field mapping
*/
async buildWhereClause(query) {
if (!query.where || query.where.length === 0) {
// LEVEL 7: Check for anti-join conditions that need to be added
if (query.joins?.some((join) => join.isAntiJoin)) {
logger.debug('No WHERE clause but anti-join detected, adding IS NULL conditions');
const antiJoinConditions = [];
for (const join of query.joins) {
if (join.isAntiJoin) {
const tableName = entityTableMapper.toTableName(join.entity);
antiJoinConditions.push(`${tableName}.id IS NULL`);
logger.debug(`Added condition: ${tableName}.id IS NULL`);
}
}
return antiJoinConditions.join(' AND ');
}
return '';
}
const conditions = [];
// CRITICAL FIX: Detect joins from both explicit query joins AND field mappings
const hasJoins = this.requiredJoins.size > 0 || Boolean(query.joins && query.joins.length > 0);
for (const condition of query.where) {
let sqlField = condition.field;
// Handle COUNT operations with enhanced fields first
if (condition.field.includes('COUNT(') && condition.field.includes(')')) {
const countMatch = condition.field.match(/COUNT\s*\(\s*([^)]+)\s*\)/i);
if (countMatch) {
const innerField = countMatch[1].trim();
// In view-only mode, we don't need enhanced mappings
// Views already handle the complexity
sqlField = this.processJSONFunction(condition.field, hasJoins);
}
else {
sqlField = this.processJSONFunction(condition.field, hasJoins);
}
}
// Handle other JSON functions
else if (condition.field.includes('(') && condition.field.includes(')')) {
sqlField = this.processJSONFunction(condition.field, hasJoins);
}
// Handle JSON paths
else if (this.jsonHandler.isJSONPath(condition.field) && condition.field.includes('$')) {
// Direct JSON path like $.environments.production.enabled
sqlField = `JSON_EXTRACT(data_json, '${condition.field}')`;
}
// Handle fields with entity/table prefix
else if (condition.field.includes('.')) {
const [entityOrTable, ...fieldParts] = condition.field.split('.');
const fieldPart = fieldParts.join('.');
const tableName = entityTableMapper.toTableName(entityOrTable);
// Check if this is a JSON path reference
if (fieldParts.length > 1 && this.jsonHandler.isJSONPath(fieldPart)) {
// Like flag.environments.production.enabled
const tableRef = hasJoins ? `${tableName}.data_json` : 'data_json';
sqlField = `JSON_EXTRACT(${tableRef}, '$.${fieldPart}')`;
}
else {
// Regular field reference
let baseField = hasJoins ? `${tableName}.${fieldPart}` : fieldPart;
// Convert visitor fields to experiment_results JSON path
if (entityOrTable === 'experiments' && (fieldPart === 'visitor_count' || fieldPart === 'visitors' || fieldPart === 'total_visitors')) {
const hasResultsJoin = query.joins?.some(j => j.entity === 'experiment_results');
if (hasResultsJoin) {
// L7-16 FIX: Converting visitor field to JSON_EXTRACT from experiment_results
sqlField = "JSON_EXTRACT(experiment_results.data_json, '$.reach.total_count')";
}
else {
logger.warn('Missing JOIN to experiment_results for visitor data');
sqlField = baseField;
}
}
else if (entityOrTable === 'experiments' && fieldPart === 'unique_conversions') {
const hasResultsJoin = query.joins?.some(j => j.entity === 'experiment_results');
if (hasResultsJoin) {
logger.debug('Converting unique_conversions to JSON_EXTRACT from experiment_results');
// This is complex - need to sum samples across all variations
sqlField = "JSON_EXTRACT(experiment_results.data_json, '$.metrics[0].results')";
}
else {
logger.warn('Missing JOIN to experiment_results for conversion data');
sqlField = baseField;
}
}
else if (entityOrTable === 'experiments' && (fieldPart === 'confidence_level' || fieldPart === 'confidence')) {
const hasResultsJoin = query.joins?.some(j => j.entity === 'experiment_results');
if (hasResultsJoin) {
logger.debug('Converting confidence_level to JSON_EXTRACT from experiment_results');
sqlField = "JSON_EXTRACT(experiment_results.data_json, '$.stats_config.confidence_level')";
}
else {
logger.warn('Missing JOIN to experiment_results for confidence data');
sqlField = baseField;
}
}
else {
// CRITICAL FIX: Convert problematic field references
// Pass true to add JOIN if needed since we're in WHERE clause
sqlField = this.convertProblemFields(baseField, true);
}
}
}
// CRITICAL FIX: Handle fields without table prefix when JOINs are present
else if (hasJoins) {
// Need to resolve which table this field belongs to
try {
logger.debug({ field: condition.field, entity: query.find }, 'FIELD CATALOG RESOLUTION: Resolving field');
const location = await this.fieldCatalog.resolveField(query.find, condition.field);
logger.debug('FIELD CATALOG RESULT:', JSON.stringify(location, null, 2));
sqlField = this.mapFieldToSQL(location, query.find);
logger.debug('MAPPED SQL FIELD:', sqlField);
}
catch (error) {
logger.debug('FIELD CATALOG ERROR:', error instanceof Error ? error.message : String(error));
// If field resolution fails, check if it's in a joined table
if (query.joins) {
for (const join of query.joins) {
const joinedTable = entityTableMapper.toTableName(join.entity);
// Common field mappings
if (condition.field === 'environment_key' && joinedTable === 'flag_environments') {
sqlField = 'flag_environments.environment_key';
break;
}
else if (condition.field === 'enabled' && joinedTable === 'flag_environments') {
sqlField = 'flag_environments.enabled';
break;
}
}
}
// Fallback to assuming it's in the primary table
if (sqlField === condition.field) {
const primaryTable = entityTableMapper.toTableName(query.find);
let qualifiedField = `${primaryTable}.${condition.field}`;
logger.debug('FALLBACK QUALIFIED FIELD:', qualifiedField);
// CRITICAL FIX: Apply field conversion for problematic fields
// Pass true to add JOIN if needed since we're in WHERE clause
sqlField = this.convertProblemFields(qualifiedField, true);
}
}
}
// CRITICAL FIX: Handle standalone fields without JOINs
else {
// Handle visitor-related fields for experiments
const fieldWithoutTable = condition.field.includes('.') ? condition.field.split('.')[1] : condition.field;
if ((fieldWithoutTable === 'visitor_count' || fieldWithoutTable === 'visitors' || fieldWithoutTable === 'total_visitors') && query.find === 'experiments') {
logger.debug('Converting visitor field to experiment_results.total_count');
// Check if we have a JOIN to experiment_results
const hasResultsJoin = query.joins?.some(j => j.entity === 'experiment_results');
if (hasResultsJoin) {
sqlField = 'experiment_results.total_count';
}
else {
// Need to add the JOIN - this should have been done by the parser
logger.warn('Missing JOIN to experiment_results for visitor data');
sqlField = '0'; // Fallback
}
}
else {
// Apply field conversion for standalone fields that need special handling
// Pass true to add JOIN if the conversion requires it
sqlField = this.convertProblemFields(condition.field, true);
}
}
// Check if this is a date-related condition
const isDateField = this.dateHandler.isDateField(condition.field);
const hasRelativeDate = typeof condition.value === 'string' && this.dateHandler.getRelativeDateSQL(condition.value) !== null;
const isDateOperator = ['BETWEEN', 'YEAR', 'MONTH', 'DAY', 'LAST_N_DAYS'].includes(condition.operator) || hasRelativeDate;
let sqlCondition = '';
if (isDateField || isDateOperator) {
// Handle date/time conditions
const dateResult = this.dateHandler.parseDateFilter(condition);
if (dateResult.isValid) {
sqlCondition = dateResult.sqlExpression.replace(condition.field, sqlField);
}
else {
// Fallback to standard processing
sqlCondition = this.buildStandardCondition(sqlField, condition);
}
}
else {
// Handle standard conditions
sqlCondition = this.buildStandardCondition(sqlField, condition);
}
conditions.push(sqlCondition);
}
return conditions.join(' AND ');
}
/**
* Process JSON functions to inject proper column names
*/
processJSONFunction(field, hasJoins) {
// Extract alias if present
const aliasMatch = field.match(/^(.+?)\s+as\s+(.+)$/i);
const funcPart = aliasMatch ? aliasMatch[1].trim() : field;
const alias = aliasMatch ? ` AS ${aliasMatch[2].trim()}` : '';
// JSON function patterns that need column injection
const jsonFunctions = [
'JSON_ARRAY_LENGTH',
'JSON_TYPE',
'JSON_VALID',
'JSON_QUOTE',
'JSON_GROUP_ARRAY',
'JSON_GROUP_OBJECT'
];
// Check if this is a JSON function with a path
for (const func of jsonFunctions) {
// First check for JSON path with $ prefix
const pathPattern = new RegExp(`${func}\\s*\\(\\s*\\$([^)]+)\\)`, 'i');
const pathMatch = funcPart.match(pathPattern);
if (pathMatch) {
const jsonPath = `$${pathMatch[1]}`;
const primaryEntity = this.currentQuery?.find || 'flags';
const tableName = entityTableMapper.toTableName(primaryEntity);
const columnName = hasJoins ? `${tableName}.data_json` : 'data_json';
return `${func}(${columnName}, '${jsonPath}')${alias}`;
}
// Check for direct column reference without $
const directPattern = new RegExp(`${func}\\s*\\(\\s*([^)]+)\\)`, 'i');
const directMatch = funcPart.match(directPattern);
if (directMatch) {
const columnName = directMatch[1].trim();
logger.debug(`Processing ${func}(${columnName}) - adding table qualification`);
// Get the primary entity/table
const primaryEntity = this.currentQuery?.find || 'flags';
const tableName = entityTableMapper.toTableName(primaryEntity);
// Add table qualification if not already present
if (!columnName.includes('.')) {
// First convert the field reference to handle special cases
const convertedField = this.convertProblemFields(columnName, true);
// If the field was converted (e.g., variations -> rules.variations), use that
if (convertedField !== columnName) {
return `${func}(${convertedField})${alias}`;
}
// Otherwise, use standard qualification
const qualifiedColumn = hasJoins ? `${tableName}.${columnName}` : columnName;
return `${func}(${qualifiedColumn})${alias}`;
}
// Already qualified, return as-is
return field;
}
}
// Check for JSON_EXTRACT with incorrect syntax
if (funcPart.includes('JSON_EXTRACT')) {
// Fix quote issues - ensure single quotes for paths
const fixed = funcPart.replace(/JSON_EXTRACT\s*\(\s*([^,]+),\s*"([^"]+)"\s*\)/gi, "JSON_EXTRACT($1, '$2')");
return fixed + alias;
}
// Not a JSON function, return as-is
return field;
}
/**
* Build standard (non-date) condition
*/
buildStandardCondition(sqlField, condition) {
let value = condition.value;
// CRITICAL DEBUG: Log condition building
logger.debug({ field: sqlField, operator: condition.operator, value, valueType: typeof value }, 'BUILDING CONDITION');
logger.debug('BUILDING CONDITION: sqlField parameter received:', sqlField);
// Handle confidence level percentage vs decimal conversion
if (sqlField.includes('confidence_level') && typeof value === 'string') {
const numValue = parseFloat(value);
if (!isNaN(numValue) && numValue > 1) {
// Convert percentage to decimal (95 -> 0.95)
value = numValue / 100;
logger.debug({ from: numValue, to: value }, '[FIX] CONFIDENCE CONVERSION: Converting percentage to decimal');
}
}
// Handle percentage_included conversion (stored as basis points)
if (sqlField.includes('percentage_included')) {
const numValue = parseFloat(String(value));
if (!isNaN(numValue) && numValue <= 100) {
// Convert percentage to basis points (50% -> 5000)
value = numValue * 100;
logger.debug({ from: numValue, to: value }, 'Converting percentage to basis points');
}
}
// LEVEL 7 FIX: Handle boolean field conversions for archived fields
if ((sqlField === 'pages.archived' || sqlField === 'flags.archived') && typeof value === 'string') {
// Convert string status values to boolean equivalents for SQLite
if (value === 'active' || value === 'running') {
value = 0; // In SQLite, 0 = false (not archived)
logger.debug('[FIX] BOOLEAN CONVERSION: Converting "active/running" to archived=0 (not archived)');
}
else if (value === 'inactive' || value === 'paused' || value === 'archived') {
value = 1; // In SQLite, 1 = true (archived)
logger.debug('[FIX] BOOLEAN CONVERSION: Converting "inactive/paused/archived" to archived=1 (archived)');
}
}
// Handle different value types and operators
if (condition.operator === 'IN' && Array.isArray(value)) {
// Handle IN operator with array values
const quotedValues = value.map(v => `'${String(v).replace(/'/g, "''")}'`);
const result = `${sqlField} IN (${quotedValues.join(', ')})`;
logger.debug('CONDITION RESULT (IN array):', result);
return result;
}
else if (condition.operator === 'IN' && typeof value === 'string') {
// Handle IN operator with comma-separated string
const values = value.split(',').map(v => v.trim());
const quotedValues = values.map(v => `'${v.replace(/'/g, "''")}'`);
const result = `${sqlField} IN (${quotedValues.join(', ')})`;
logger.debug('CONDITION RESULT (IN string):', result);
return result;
}
else if (condition.operator === 'BETWEEN' && Array.isArray(value) && value.length === 2) {
// Handle BETWEEN operator
const result = `${sqlField} BETWEEN '${value[0]}' AND '${value[1]}'`;
logger.debug('CONDITION RESULT (BETWEEN):', result);
return result;
}
else if (typeof value === 'string') {
value = `'${value.replace(/'/g, "''")}'`;
const result = `${sqlField} ${condition.operator} ${value}`;
logger.debug('CONDITION RESULT (string):', result);
return result;
}
else if (value === null) {
let result;
if (condition.operator === '=' || condition.operator === 'IS NULL') {
result = `${sqlField} IS NULL`;
}
else if (condition.operator === '!=' || condition.operator === 'IS NOT NULL') {
result = `${sqlField} IS NOT NULL`;
}
else if (condition.operator.includes('NULL')) {
// Operator already contains NULL, don't append it
result = `${sqlField} ${condition.operator}`;
}
else {
result = `${sqlField} ${condition.operator} NULL`;
}
logger.debug('CONDITION RESULT (null):', result);
return result;
}
else if (Array.isArray(value)) {
// Handle other array cases
const quotedValues = value.map(v => `'${String(v).replace(/'/g, "''")}'`);
const result = `${sqlField} ${condition.operator} (${quotedValues.join(', ')})`;
logger.debug('CONDITION RESULT (array):', result);
return result;
}
else {
// Numbers, booleans, etc.
const result = `${sqlField} ${condition.operator} ${value}`;
logger.debug('CONDITION RESULT (number/other):', result);
return result;
}
}
/**
* Build GROUP BY clause with field mapping
*/
async buildGroupByClause(query) {
if (!query.groupBy || query.groupBy.length === 0) {
return '';
}
logger.debug(`Building GROUP BY clause with fields: ${query.groupBy.join(', ')}`);
const groupFields = [];
// CRITICAL FIX: Detect joins from both explicit query joins AND field mappings
const hasJoins = this.requiredJoins.size > 0 || Boolean(query.joins && query.joins.length > 0);
for (const field of query.groupBy) {
// LEVEL 7: Check for complex aggregation patterns (e.g., breakdown by variation count)
if (field === 'variation_count_range' || field.includes('breakdown')) {
logger.debug(`Detected complex grouping pattern: ${field}`);
// Example: GROUP BY variation count ranges (2, 3, 4, 5+)
if (field === 'variation_count_range') {
const caseStatement = `CASE
WHEN JSON_ARRAY_LENGTH(variations) = 2 THEN '2 variations'
WHEN JSON_ARRAY_LENGTH(variations) = 3 THEN '3 variations'
WHEN JSON_ARRAY_LENGTH(variations) = 4 THEN '4 variations'
WHEN JSON_ARRAY_LENGTH(variations) >= 5 THEN '5+ variations'
ELSE 'No variations'
END`;
groupFields.push(caseStatement);
logger.debug('Generated CASE statement for variation count ranges');
continue;
}
}
// Check if field is already a SQL expression (e.g., JSON_EXTRACT, function call, etc.)
if (field.includes('(') && field.includes(')')) {
// It's already a SQL expression, use it as-is
logger.debug(`Field is already SQL expression: ${field}`);
groupFields.push(field);
}
// Check if this is a JSON path
else if (this.jsonHandler.isJSONPath(field) && field.includes('$')) {
// Direct JSON path
groupFields.push(`JSON_EXTRACT(data_json, '${field}')`);
}
// Check for entity-prefixed JSON paths
else if (field.includes('.')) {
const [entityOrTable, ...fieldParts] = field.split('.');
const fieldPart = fieldParts.join('.');
const tableName = entityTableMapper.toTableName(entityOrTable);
if (fieldParts.length > 1 && this.jsonHandler.isJSONPath(fieldPart)) {
// JSON path with entity prefix
const tableRef = hasJoins ? `${tableName}.data_json` : 'data_json';
groupFields.push(`JSON_EXTRACT(${tableRef}, '$.${fieldPart}')`);
}
else {
// Regular field
let sqlField = hasJoins ? `${tableName}.${fieldPart}` : fieldPart;
// CRITICAL FIX: Convert problematic field references
sqlField = this.convertProblemFields(sqlField);
groupFields.push(sqlField);
}
}
else {
// Try field catalog resolution
try {
let location;
try {
location = await this.fieldCatalog.resolveField(query.find, field);
}
catch (error) {
// If that fails, try the singular form
const singularEntity = query.find.endsWith('s') ? query.find.slice(0, -1) : query.find;
location = await this.fieldCatalog.resolveField(singularEntity, field);
}
const sqlField = this.mapFieldToSQL(location, query.find);
groupFields.push(sqlField);
}
catch (error) {
// Field not found, use as-is with field conversion
logger.warn(`Could not resolve field ${field}, using as-is`);
const convertedField = this.convertProblemFields(field);
groupFields.push(convertedField);
}
}
}
return groupFields.join(', ');
}
/**
* Map field location to SQL expression
*/
mapFieldToSQL(location, baseEntity) {
logger.debug({
type: location.physicalLocation.type,
path: location.physicalLocation.path,
jsonPath: location.physicalLocation.jsonPath,
baseEntity
}, 'MAP FIELD TO SQL');
switch (location.physicalLocation.type) {
case 'column':
// CRITICAL FIX: Convert problematic field references
const columnResult = this.convertProblemFields(location.physicalLocation.path);
logger.debug('COLUMN RESULT:', columnResult);
return columnResult;
case 'json_path':
const jsonResult = `JSON_EXTRACT(data_json, '${location.physicalLocation.jsonPath}')`;
logger.debug('JSON_PATH RESULT:', jsonResult);
return jsonResult;
case 'related':
const relationship = location.physicalLocation.relationship;
if (relationship) {
// Add required join
this.addRequiredJoin(relationship);
// Return qualified field name
const qualifiedField = `${relationship.to.entity}.${location.physicalLocation.path}`;
const relatedResult = this.convertProblemFields(qualifiedField);
logger.debug('RELATED RESULT:', relatedResult);
return relatedResult;
}
const fallbackRelated = this.convertProblemFields(location.physicalLocation.path);
logger.debug('RELATED FALLBACK:', fallbackRelated);
return fallbackRelated;
case 'computed':
const computedResult = this.convertProblemFields(location.physicalLocation.path);
logger.debug('COMPUTED RESULT:', computedResult);
return computedResult;
default:
const defaultResult = this.convertProblemFields(location.physicalLocation.path);
logger.debug('DEFAULT RESULT:', defaultResult);
return defaultResult;
}
}
/**
* Add a required join
*/
addRequiredJoin(relationship) {
const joinKey = `${relationship.from.entity}_${relationship.to.entity}`;
if (!this.requiredJoins.has(joinKey)) {
this.requiredJoins.add(joinKey);
this.joinClauses.set(joinKey, {
type: 'LEFT',
entity: relationship.to.entity,
on: {
leftField: `${relationship.from.entity}.${relationship.from.field}`,
rightField: `${relationship.to.entity}.${relationship.to.field}`
}
});
}
}
/**
* Build JOIN clauses
*/
buildJoinClauses(query) {
const joins = [];
const primaryTable = this.buildFromClause(query);
// Add explicit joins
if (query.joins) {
for (const join of query.joins) {
// LEVEL 7: Check for anti-join marker
if (join.isAntiJoin) {
logger.debug(`Building anti-join for ${join.entity}`);
// Anti-join will be handled in WHERE clause generation
// We still need the LEFT JOIN here
}
// Fix table names in join conditions
let leftField = join.on.leftField;
let rightField = join.on.rightField;
// Map singular entity names to plural table names
if (leftField.startsWith('flag.')) {
leftField = leftField.replace('flag.', 'flags.');
}
if (rightField.startsWith('flag.')) {
rightField = rightField.replace('flag.', 'flags.');
}
// Map entity name to table name for JOIN
const tableName = entityTableMapper.toTableName(join.entity);
// [FIX] L2-1 FIX: Check if this exact JOIN already exists before adding
const joinCondition = `${leftField} = ${rightField}`;
const proposedJoin = `${join.type} JOIN ${tableName} ON ${joinCondition}`;
const existingIdenticalJoin = joins.find(j => j.includes(`JOIN ${tableName}`) &&
j.includes(joinCondition));
if (existingIdenticalJoin) {
logger.debug(`[FIX] L2-1 FIX: Skipping duplicate JOIN to ${tableName} with condition ${joinCondition}`);
logger.debug(`[L2-1] Skipping duplicate JOIN to ${tableName}`);
continue;
}
// LEVEL 7: Check for self-join (same table joined multiple times)
const existingTableJoin = joins.find(j => j.includes(`JOIN ${tableName}`));
if (existingTableJoin && tableName === primaryTable) {
logger.debug(`Detected self-join on ${tableName}, adding alias`);
// TODO: Implement table aliasing for self-joins
// e.g., JOIN flag_environments dev ON ... JOIN flag_environments prod ON ...
}
// [FIX] L4-2 FIX: Skip JOINs to the same table that's already the primary table
if (tableName === primaryTable) {
logger.debug(`[FIX] L4-2 FIX: Skipping duplicate JOIN to ${tableName} (already primary table)`);
logger.debug(`Skipping duplicate JOIN to ${tableName}`);
continue;
}
// LEVEL 7: Handle special page-experiment relationship for anti-joins
if (primaryTable === 'pages' && tableName === 'experiments' && join.isAntiJoin) {
logger.debug('Special handling for pages-experiments anti-join');
// Experiments have page_ids array, need to check if page.id is in that array
joins.push(`LEFT JOIN experiments ON experiments.project_id = pages.project_id ` +
`AND JSON_EXTRACT(experiments.data_json, '$.page_ids') LIKE '%' || pages.id || '%'`);
}
else {
joins.push(`${join.type} JOIN ${tableName} ON ${leftField} = ${rightField}`);
}
}
}
// Add required joins from field mapping
for (const [, join] of this.joinClauses) {
// Fix table names in join conditions
let leftField = join.on.leftField;
let rightField = join.on.rightField;
// Map singular entity names to plural table names
if (leftField.startsWith('flag.')) {
leftField = leftField.replace('flag.', 'flags.');
}
if (rightField.startsWith('flag.')) {
rightField = rightField.replace('flag.', 'flags.');
}
// Map entity name to table name for JOIN
const tableName = entityTableMapper.toTableName(join.entity);
// [FIX] L4-2 FIX: Skip JOINs to the same table that's already the primary table
if (tableName === primaryTable) {
logger.debug(`[FIX] L4-2 FIX: Skipping duplicate field mapping JOIN to ${tableName} (already primary table)`);
continue;
}
// [FIX] L6-7 FIX: Check if this JOIN already exists to prevent duplicates
const baseJoinCondition = `${leftField} = ${rightField}`;
const existingJoin = joins.find(j => j.includes(baseJoinCondition) && j.includes(tableName));
if (existingJoin) {
logger.debug(`[FIX] L6-7 FIX: Skipping duplicate JOIN to ${tableName} - already exists`);
continue;
}
// Build JOIN condition with additional conditions if present
let joinCondition = `${leftField} = ${rightField}`;
// Handle additional conditions (e.g., for composite keys)
if (join.on.additionalConditions) {
const additionalConditions = join.on.additionalConditions;
for (const condition of additionalConditions) {
joinCondition += ` AND ${condition.leftField} = ${condition.rightField}`;
}
}
joins.push(`${join.type} JOIN ${tableName} ON ${joinCondition}`);
}
return joins.join('\n');
}
/**
* Build ORDER BY clause
*/
async buildOrderByClause(query) {
if (!query.orderBy || query.orderBy.length === 0) {
return '';
}
const orderClauses = [];
for (const order of query.orderBy) {
try {
const location = await this.fieldCatalog.resolveField(query.find, order.field);
const sqlField = this.mapFieldToSQL(location, query.find);
orderClauses.push(`${sqlField} ${order.direction}`);
}
catch (error) {
// Field not found, use as-is
orderClauses.push(`${order.field} ${order.direction}`);
}
}
return orderClauses.join(', ');
}
/**
* Build HAVING clause
*/
buildHavingClause(query) {
if (!query.having || query.having.length === 0) {
return '';
}
const conditions = query.having.map(h => `${h.field} ${h.operator} ${h.value}`).join(' AND ');
return conditions;
}
/**
* Validate query cardinality to prevent JOIN explosion
*/
validateQueryCardinality(query, joinClause) {