UNPKG

@the_cfdude/productboard-mcp

Version:

Model Context Protocol server for Productboard REST API with dynamic tool loading

845 lines (844 loc) 39.5 kB
import { ValidationError } from '../errors/index.js'; import { EntityFieldMappings } from './search-field-mappings.js'; import { OutputProcessor } from './search-output-processor.js'; import { handleFeaturesTool } from '../tools/features.js'; import { handleProductsTool } from '../tools/products.js'; import { handleComponentsTool } from '../tools/components.js'; import { handleNotesTool } from '../tools/notes.js'; import { handleCompaniesTool } from '../tools/companies.js'; import { handleUsersTool } from '../tools/users.js'; import { handleReleasesTool } from '../tools/releases.js'; import { handleObjectivesTool } from '../tools/objectives.js'; import { handleCustomFieldsTool } from '../tools/custom-fields.js'; import { handleWebhooksTool } from '../tools/webhooks.js'; import { handlePluginIntegrationsTool } from '../tools/plugin-integrations.js'; import { handleJiraIntegrationsTool } from '../tools/jira-integrations.js'; import { compilePattern, applyPatternFilter, generateSearchSuggestions, expandFieldPatterns, validatePatternComplexity, } from './search-pattern-utils.js'; import { debugLog } from './debug-logger.js'; export class SearchEngine { entityMappings = EntityFieldMappings; outputProcessor = new OutputProcessor(); // Cache removed - was causing stale data issues /** * Validate and normalize search parameters */ async validateAndNormalizeParams(params) { debugLog('search-engine', 'validateAndNormalizeParams called', { params }); // Normalize entityType to array const entityTypes = Array.isArray(params.entityType) ? params.entityType : [params.entityType]; // Validate all entity types for (const entityType of entityTypes) { if (!this.entityMappings[entityType]) { throw new ValidationError(`Unsupported entity type: ${entityType}. Supported types: ${Object.keys(this.entityMappings).join(', ')}`, 'entityType'); } } // Output parameter should already be parsed at the search.ts level const output = params.output || 'full'; // Normalize parameters with defaults const normalized = { entityType: params.entityType, // Keep original for compatibility entityTypes: entityTypes, // Always normalized to array filters: params.filters || {}, operators: params.operators || {}, output: output, limit: Math.min(Math.max(params.limit ?? 50, 1), 100), // Ensure minimum limit of 1 startWith: Math.max(params.startWith || 0, 0), detail: params.detail || 'standard', includeSubData: params.includeSubData || false, includeCustomFields: params.includeCustomFields || false, // Enhanced search parameters patternMatchMode: params.patternMatchMode || 'wildcard', caseSensitive: params.caseSensitive || false, suggestAlternatives: params.suggestAlternatives || false, maxSuggestions: Math.min(params.maxSuggestions || 5, 10), ...(params.instance && { instance: params.instance }), ...(params.workspaceId && { workspaceId: params.workspaceId }), }; // Validate filters for all entity types const filterValidation = this.validateFiltersForMultipleTypes(normalized.entityTypes, normalized.filters); if (!filterValidation.isValid) { throw new ValidationError(`Invalid filters: ${filterValidation.errors.join(', ')}`, 'filters'); } // Use normalized filters normalized.filters = filterValidation.normalizedFilters; debugLog('search-engine', 'Parameters normalized', { normalized }); // Validate operators (now includes enhanced operators) const operatorValidation = this.validateOperators(normalized.operators); if (!operatorValidation.isValid) { throw new ValidationError(`Invalid operators: ${operatorValidation.errors.join(', ')}`, 'operators'); } // Validate output parameter for all entity types if (Array.isArray(normalized.output)) { const outputValidation = this.validateOutputFieldsForMultipleTypes(normalized.entityTypes, normalized.output); if (!outputValidation.isValid) { throw new ValidationError(`Invalid output fields: ${outputValidation.errors.join(', ')}`, 'output'); } } return normalized; } /** * Execute search against the appropriate entity endpoint(s) */ async executeEntitySearch(_context, params) { const startTime = Date.now(); // If single entity type, use existing logic if (params.entityTypes.length === 1) { return this.executeSingleEntitySearch(_context, { ...params, entityType: params.entityTypes[0], }, startTime); } // For multiple entity types, execute searches in parallel const searchPromises = params.entityTypes.map(entityType => this.executeSingleEntitySearch(_context, { ...params, entityType, }, startTime).then(result => ({ entityType, result, }))); try { const results = await Promise.all(searchPromises); // Aggregate results from all entity types let allData = []; let totalRecords = 0; let hasMore = false; const warnings = []; for (const { entityType, result } of results) { // Add entity type to each item if not already present const dataWithType = result.data.map(item => ({ ...item, _entityType: entityType, // Add entity type identifier })); allData = allData.concat(dataWithType); totalRecords += result.totalRecords; hasMore = hasMore || result.hasMore; warnings.push(...result.warnings); } const queryTime = Date.now() - startTime; return { data: allData, totalRecords, hasMore, warnings: [...new Set(warnings)], // Remove duplicates queryTimeMs: queryTime, // cacheHit removed with cache removal }; } catch (error) { throw new Error(`Multi-entity search failed for ${params.entityTypes.join(', ')}: ${error.message}`); } } /** * Execute search for a single entity type */ async executeSingleEntitySearch(_context, params, startTime) { debugLog('search-engine', 'executeSingleEntitySearch called', { entityType: params.entityType, filters: params.filters, limit: params.limit, }); // Cache removed - was causing stale data issues try { const entityConfig = this.entityMappings[params.entityType]; // Build parameters for the underlying list function const listParams = this.buildListParams(params, params.entityType); // Route to appropriate handler based on entity type const response = await this.routeToEntityHandler(params.entityType, entityConfig.listFunction, listParams); // Parse response from the handler debugLog('search-engine', 'Parsing handler response', { entityType: params.entityType, responseType: typeof response, hasContent: !!response?.content, contentType: response?.content?.[0]?.type, }); const parsedResponse = this.parseHandlerResponse(response); debugLog('search-engine', 'Handler response parsed', { entityType: params.entityType, dataLength: parsedResponse.data?.length || 0, totalRecords: parsedResponse.totalRecords, }); const queryTime = Date.now() - startTime; const results = { data: parsedResponse.data || [], totalRecords: parsedResponse.totalRecords || parsedResponse.data?.length || 0, hasMore: parsedResponse.hasMore || false, warnings: parsedResponse.warnings || [], queryTimeMs: queryTime, }; // Cache removed - was causing stale data issues return results; } catch (error) { // Re-throw with additional context throw new Error(`Entity search failed for ${params.entityType}: ${error.message}`); } } /** * Process raw results with filtering and output formatting */ async processResults(rawResults, params) { let processedData = rawResults.data; const warnings = [...rawResults.warnings]; // Apply client-side filtering for complex operators processedData = await this.applyClientSideFiltering(processedData, params); // Check for parameter conflicts and add warnings if (params.output !== 'full' && params.detail !== 'standard') { warnings.push(`output parameter overrides detail level "${params.detail}" - exact fields and order determined by output specification`); } // Apply output processing if (params.output !== 'full') { // When processing multi-entity results, we need to handle entity type per item if (params.entityTypes.length > 1) { // Special handling for ids-only mode - can't add properties to strings if (params.output === 'ids-only') { processedData = this.outputProcessor.processOutput(processedData, params.entityTypes[0], // Entity type doesn't matter for ids-only params.output); } else { processedData = processedData.map(item => { const entityType = item._entityType || params.entityTypes[0]; const processed = this.outputProcessor.processOutput([item], entityType, params.output)[0]; // Preserve the _entityType field if it exists and we're not in ids-only mode if (item._entityType && typeof processed === 'object') { processed._entityType = item._entityType; } return processed; }); } } else { processedData = this.outputProcessor.processOutput(processedData, params.entityTypes[0], params.output); } } return { ...rawResults, data: processedData, warnings, }; } /** * Check if a field is searchable for the given entity type */ isFieldSearchable(entityType, field) { const entityConfig = this.entityMappings[entityType]; if (!entityConfig) return false; // Direct field match if (entityConfig.searchableFields.includes(field)) { return true; } // Check for dot notation fields (e.g., "owner.email") const baseField = field.split('.')[0]; return entityConfig.searchableFields.some(searchableField => searchableField.startsWith(baseField + '.') || searchableField === baseField); } /** * Validate filters for multiple entity types */ validateFiltersForMultipleTypes(entityTypes, filters) { const errors = []; const warnings = []; const normalizedFilters = {}; for (const [field, value] of Object.entries(filters)) { // Check if field is searchable in at least one entity type const searchableInTypes = entityTypes.filter(entityType => this.isFieldSearchable(entityType, field)); if (searchableInTypes.length === 0) { errors.push(`Field "${field}" is not searchable in any of the specified entity types`); continue; } if (searchableInTypes.length < entityTypes.length) { warnings.push(`Field "${field}" is only searchable in: ${searchableInTypes.join(', ')}`); } // Normalize filter value normalizedFilters[field] = this.normalizeFilterValue(value); // Add warnings for potentially problematic filters if (value === '' || value === null || value === undefined) { warnings.push(`Searching for empty/missing values in field "${field}"`); } } return { isValid: errors.length === 0, errors, warnings, normalizedFilters, }; } /** * Validate output fields for multiple entity types */ validateOutputFieldsForMultipleTypes(entityTypes, fields) { const errors = []; const warnings = []; for (const field of fields) { // Skip the special _entityType field we add if (field === '_entityType') continue; // Check if field is available in at least one entity type const availableInTypes = entityTypes.filter(entityType => this.isFieldSearchable(entityType, field)); if (availableInTypes.length === 0) { errors.push(`Output field "${field}" is not available in any of the specified entity types`); continue; } if (availableInTypes.length < entityTypes.length) { warnings.push(`Output field "${field}" is only available in: ${availableInTypes.join(', ')}`); } } return { isValid: errors.length === 0, errors, warnings, normalizedFilters: {}, }; } /** * Validate operator fields and values */ validateOperators(operators) { const validOperators = [ 'equals', 'contains', 'isEmpty', 'startsWith', 'endsWith', 'before', 'after', 'regex', 'wildcard', 'not', 'in', 'not_in', ]; const errors = []; const warnings = []; for (const [field, operator] of Object.entries(operators)) { if (!validOperators.includes(operator)) { errors.push(`Invalid operator "${operator}" for field "${field}". Valid operators: ${validOperators.join(', ')}`); } // Add warnings for operators that might not work as expected if (operator === 'isEmpty') { warnings.push(`Using "isEmpty" operator for field "${field}" - filter value will be ignored`); } if (operator === 'regex') { warnings.push(`Using "regex" operator for field "${field}" - ensure pattern is safe and not overly complex`); } if (operator === 'wildcard') { warnings.push(`Using "wildcard" operator for field "${field}" - supports * and ? patterns`); } } return { isValid: errors.length === 0, errors, warnings, normalizedFilters: {}, }; } /** * Build parameters for the underlying list function */ buildListParams(params, entityType) { const listParams = { limit: params.limit, startWith: params.startWith, detail: params.detail, includeSubData: params.includeSubData, includeCustomFields: params.includeCustomFields, instance: params.instance, workspaceId: params.workspaceId, }; // Add cursor for pagination if present if (params.pageCursor) { listParams.pageCursor = params.pageCursor; } // Add entity-specific filters if we have a single entity type const singleEntityType = entityType || (params.entityTypes.length === 1 ? params.entityTypes[0] : null); if (singleEntityType) { for (const [field, value] of Object.entries(params.filters)) { if (this.canFilterServerSide(singleEntityType, field)) { listParams[this.mapFilterToApiParam(singleEntityType, field)] = value; } } } return listParams; } /** * Check if a filter can be applied server-side */ canFilterServerSide(entityType, field) { const entityConfig = this.entityMappings[entityType]; return entityConfig.serverSideFilters?.includes(field) || false; } /** * Map filter field to API parameter name */ mapFilterToApiParam(entityType, field) { const entityConfig = this.entityMappings[entityType]; return entityConfig.filterMappings?.[field] || field; } /** * Route to appropriate entity handler with pagination support */ async routeToEntityHandler(entityType, functionName, params) { debugLog('search-engine', 'Routing to entity handler', { entityType, functionName, paramsKeys: Object.keys(params), }); // For pagination, we need to fetch ALL pages and combine results // The API returns max 100 records per page with links.next for pagination const allData = []; let currentParams = { ...params }; let hasMorePages = true; let pageCount = 0; const maxPages = 50; // Safety limit to prevent infinite loops while (hasMorePages && pageCount < maxPages) { pageCount++; debugLog('search-engine', `Fetching page ${pageCount} for ${entityType}`, { offset: currentParams.startWith || 0, limit: currentParams.limit, }); let response; switch (entityType) { case 'features': response = await handleFeaturesTool(functionName, currentParams); break; case 'products': debugLog('search-engine', 'Calling handleProductsTool', { functionName, params: currentParams, }); response = await handleProductsTool(functionName, currentParams); debugLog('search-engine', 'handleProductsTool returned', { hasContent: !!response?.content, contentLength: typeof response?.content?.[0]?.text === 'string' ? response.content[0].text.length : 0, }); break; case 'components': response = await handleComponentsTool(functionName, currentParams); break; case 'notes': response = await handleNotesTool(functionName, currentParams); break; case 'companies': response = await handleCompaniesTool(functionName, currentParams); break; case 'users': response = await handleUsersTool(functionName, currentParams); break; case 'releases': case 'release_groups': response = await handleReleasesTool(functionName, currentParams); break; case 'objectives': case 'initiatives': case 'key_results': response = await handleObjectivesTool(functionName, currentParams); break; case 'custom_fields': response = await handleCustomFieldsTool(functionName, currentParams); break; case 'webhooks': response = await handleWebhooksTool(functionName, currentParams); break; case 'plugin_integrations': response = await handlePluginIntegrationsTool(functionName, currentParams); break; case 'jira_integrations': response = await handleJiraIntegrationsTool(functionName, currentParams); break; default: throw new Error(`No handler available for entity type: ${entityType}`); } // Parse the response to check for data and pagination if (response?.content?.[0]?.text) { try { const parsed = JSON.parse(response.content[0].text); // Handle different response formats if (Array.isArray(parsed)) { // Simple array response - add all items allData.push(...parsed); hasMorePages = false; // Arrays don't have pagination info } else if (parsed.data) { // Structured response with data array allData.push(...(parsed.data || [])); // Check for pagination via links.next if (parsed.links?.next) { // Extract cursor from next URL for cursor-based pagination const nextUrl = new URL(parsed.links.next); const pageCursor = nextUrl.searchParams.get('pageCursor'); if (pageCursor) { currentParams = { ...currentParams, pageCursor, }; debugLog('search-engine', `Found links.next, fetching next page with cursor ${pageCursor}`); } else { // Fallback to offset-based pagination if no cursor const nextOffset = (currentParams.startWith || 0) + (currentParams.limit || 100); currentParams = { ...currentParams, startWith: nextOffset, }; debugLog('search-engine', `Found links.next, fetching next page at offset ${nextOffset}`); } } else { hasMorePages = false; } } else if (parsed.links?.next) { // Response with pagination but data at root level // This shouldn't happen but handle it just in case allData.push(parsed); const nextOffset = (currentParams.startWith || 0) + (currentParams.limit || 100); currentParams = { ...currentParams, startWith: nextOffset, }; } else { // Single object response allData.push(parsed); hasMorePages = false; } } catch (error) { debugLog('search-engine', 'Failed to parse response for pagination', { error: error.message, }); hasMorePages = false; } } else { hasMorePages = false; } } if (pageCount >= maxPages) { debugLog('search-engine', `Warning: Reached max page limit (${maxPages}) for ${entityType}`); } debugLog('search-engine', `Fetched ${pageCount} page(s) with ${allData.length} total records for ${entityType}`); // Apply limit to final results if specified const limitedData = params.limit && params.limit < allData.length ? allData.slice(0, params.limit) : allData; const hasMoreDueToLimit = params.limit && allData.length > params.limit; const hasMoreDueToPagination = pageCount >= maxPages; debugLog('search-engine', `Applied limit ${params.limit} - returning ${limitedData.length} of ${allData.length} records`); // Return the combined results in the expected format return { content: [ { type: 'text', text: JSON.stringify({ data: limitedData, totalRecords: allData.length, // Keep total for pagination info hasMore: hasMoreDueToLimit || hasMoreDueToPagination, }), }, ], }; } /** * Parse response from tool handlers */ parseHandlerResponse(response) { debugLog('search-engine', 'parseHandlerResponse called', { hasResponse: !!response, hasContent: !!response?.content, contentLength: response?.content?.[0]?.text?.length || 0, }); if (response?.content?.[0]?.text) { try { const parsed = JSON.parse(response.content[0].text); // Handle both structured responses and raw arrays let result; if (Array.isArray(parsed)) { // Raw array response - wrap it in the expected structure result = { data: parsed, totalRecords: parsed.length, hasMore: false, }; } else if (parsed.data) { // Structured response - use as is result = parsed; } else { // Object that's not an array and doesn't have data property - treat as single item result = { data: [parsed], totalRecords: 1, hasMore: false, }; } debugLog('search-engine', 'Successfully parsed response', { dataLength: result.data?.length || 0, totalRecords: result.totalRecords || 0, }); return result; } catch (error) { debugLog('search-engine', 'Failed to parse response', { error: error.message, }); return { data: [], totalRecords: 0, hasMore: false }; } } debugLog('search-engine', 'No content to parse, returning empty result'); return { data: [], totalRecords: 0, hasMore: false }; } /** * Apply client-side filtering for complex operators */ async applyClientSideFiltering(data, params) { let filtered = data; // Compile patterns once for reuse const compiledPatterns = new Map(); for (const [field, value] of Object.entries(params.filters)) { // Skip server-side filters only if we have a single entity type if (params.entityTypes.length === 1 && this.canFilterServerSide(params.entityTypes[0], field)) { continue; // Already filtered server-side } // Handle hierarchical filtering for cross-entity relationships const hierarchicalResult = await this.handleHierarchicalFiltering(filtered, field, value, params); if (hierarchicalResult !== null) { filtered = hierarchicalResult; continue; // Skip normal filtering for this field } const operator = params.operators[field] || 'equals'; // Compile pattern if not already compiled if (!compiledPatterns.has(field)) { try { // Validate pattern complexity for wildcard/regex operators if ((operator === 'wildcard' || operator === 'regex') && !validatePatternComplexity(String(value))) { throw new ValidationError(`Pattern too complex for field "${field}": ${value}`, 'filters'); } const patternMode = operator === 'regex' ? 'regex' : operator === 'wildcard' ? 'wildcard' : params.patternMatchMode || 'wildcard'; const pattern = compilePattern(String(value), patternMode, { caseSensitive: params.caseSensitive, }); compiledPatterns.set(field, pattern); } catch { // Failed to compile pattern, fallback to exact pattern (removed console.warn for production) const exactPattern = compilePattern(String(value), 'exact', { caseSensitive: params.caseSensitive, }); compiledPatterns.set(field, exactPattern); } } // Apply pattern-based filtering using the compiled pattern const pattern = compiledPatterns.get(field); debugLog('search-engine', `Applying client-side filter for field "${field}"`, { operator, patternValue: pattern.pattern, patternMode: pattern.mode, isWildcard: pattern.isWildcard, caseSensitive: !pattern.regex.flags.includes('i'), itemsBeforeFilter: filtered.length, }); const filteredResults = filtered.filter(item => { const fieldValue = this.getNestedFieldValue(item, field); const matches = applyPatternFilter(item, field, pattern, operator); // Debug log first few matches/non-matches if (filtered.indexOf(item) < 3) { debugLog('search-engine', `Filter test for item ${filtered.indexOf(item)}`, { field, fieldValue, searchPattern: pattern.pattern, operator, matches, }); } return matches; }); debugLog('search-engine', `Filter "${field}" completed`, { itemsAfterFilter: filteredResults.length, itemsFiltered: filtered.length - filteredResults.length, }); filtered = filteredResults; } return filtered; } /** * Handle hierarchical filtering for cross-entity relationships * Inspects actual parent object structure rather than assuming fixed hierarchy */ async handleHierarchicalFiltering(data, field, value, params) { // Only handle specific hierarchical cases if (field === 'parent.product.id' && params.entityTypes.includes('features')) { debugLog('search-engine', 'Handling hierarchical filtering for parent.product.id on features', { targetProductId: value, itemCount: data.length, }); const matchingFeatures = data.filter(item => { // Check direct parent.product.id (Product -> Feature) if (item.parent?.product?.id === value) { debugLog('search-engine', 'Feature matches via direct parent.product.id', { featureId: item.id, parentProductId: item.parent.product.id, }); return true; } // Check indirect via parent.component.id (Product -> Component -> Feature) if (item.parent?.component?.id) { debugLog('search-engine', 'Feature has component parent, checking for component match', { featureId: item.id, parentComponentId: item.parent.component.id, }); // For now, we need to get components for this product to check if this component belongs to it // This would require a separate API call, but for immediate testing we can check // if the component structure has a parent reference to the product if (item.parent.component.parent?.product?.id === value) { debugLog('search-engine', 'Feature matches via component parent.product.id', { featureId: item.id, componentId: item.parent.component.id, productId: item.parent.component.parent.product.id, }); return true; } return false; } // Check if it's a sub-feature (Feature -> Sub-feature) if (item.parent?.feature?.id) { debugLog('search-engine', 'Found sub-feature, checking parent feature hierarchy', { subFeatureId: item.id, parentFeatureId: item.parent.feature.id, }); // For sub-features, we need to check if the parent feature belongs to our target product // This could be done by checking the parent feature's parent structure if (item.parent.feature.parent?.product?.id === value) { debugLog('search-engine', 'Sub-feature matches via parent feature product', { subFeatureId: item.id, parentFeatureId: item.parent.feature.id, productId: item.parent.feature.parent.product.id, }); return true; } // Also check if parent feature goes through a component to the product if (item.parent.feature.parent?.component?.parent?.product?.id === value) { debugLog('search-engine', 'Sub-feature matches via parent feature component product', { subFeatureId: item.id, parentFeatureId: item.parent.feature.id, componentId: item.parent.feature.parent.component.id, productId: item.parent.feature.parent.component.parent.product.id, }); return true; } return false; } return false; }); debugLog('search-engine', 'Hierarchical filtering completed', { field, matchingCount: matchingFeatures.length, totalCount: data.length, }); return matchingFeatures; } // Return null to indicate this field doesn't need hierarchical processing return null; } /** * Get nested field value using dot notation (helper for debugging) */ getNestedFieldValue(obj, path) { return path.split('.').reduce((current, key) => current?.[key], obj); } /** * Normalize filter values */ normalizeFilterValue(value) { if (value === '' || value === null || value === undefined) { return ''; } return value; } /** * Generate smart suggestions when no results are found */ async generateSmartSuggestions(params) { try { const suggestions = []; // Generate field expansion suggestions for (const [field] of Object.entries(params.filters)) { // Get available fields for the entity types const availableFields = []; for (const entityType of params.entityTypes) { const entityConfig = this.entityMappings[entityType]; if (entityConfig) { availableFields.push(...entityConfig.searchableFields); } } const expandedFields = expandFieldPatterns([field], availableFields); if (expandedFields.length > 1) { // More than just the original field suggestions.push({ type: 'field_expansion', originalField: field, suggestedFields: expandedFields.filter(f => f !== field), message: `Try expanding "${field}" to: ${expandedFields.filter(f => f !== field).join(', ')}`, }); } } // Generate search pattern suggestions using the pattern utilities const searchTerms = Object.values(params.filters).map(v => String(v)); for (const term of searchTerms) { if (term && term.length > 2) { const patternSuggestions = generateSearchSuggestions(term, [], // Available values would need to be fetched from API in real implementation 3 // maxSuggestions ); if (patternSuggestions.length > 0) { suggestions.push({ type: 'pattern_suggestion', originalTerm: term, suggestions: patternSuggestions, message: `Try similar terms: ${patternSuggestions.join(', ')}`, }); } } } // Generate operator suggestions for complex patterns for (const [field, value] of Object.entries(params.filters)) { const operator = params.operators[field] || 'equals'; if (operator === 'equals' && String(value).includes('*')) { suggestions.push({ type: 'operator_suggestion', field, currentOperator: operator, suggestedOperator: 'wildcard', message: `Use "wildcard" operator for "${field}" to enable * pattern matching`, }); } if (operator === 'equals' && /[.*+?^${}()|[\]\\]/.test(String(value))) { suggestions.push({ type: 'operator_suggestion', field, currentOperator: operator, suggestedOperator: 'regex', message: `Use "regex" operator for "${field}" to enable regex pattern matching`, }); } } return suggestions; } catch { // Failed to generate smart suggestions (removed console.warn for production) return []; } } }