@the_cfdude/productboard-mcp
Version:
Model Context Protocol server for Productboard REST API with dynamic tool loading
531 lines (530 loc) • 20.9 kB
JavaScript
/**
* Lightweight query engine for high-performance operations
* Optimized for status checks, existence validation, and progress tracking
*/
/* eslint-disable no-undef */
import { IntelligentCache, performanceCollector, memoryMonitor, } from './performance-monitor.js';
import { ValidationError, NetworkError } from '../errors/index.js';
import { withContext } from './tool-wrapper.js';
/**
* Lightweight query engine optimized for performance-critical operations
*/
export class LightweightQueryEngine {
cache;
defaultBatchSize = 50;
defaultConcurrency = 5;
maxBatchSize = 100;
constructor(cache) {
this.cache = cache || new IntelligentCache(2000, 180000, 'LRU'); // 3min TTL for lightweight data
}
/**
* Check status for multiple entities with minimal data transfer
*/
async checkMultipleStatus(entityType, ids, options = {}) {
const metric = performanceCollector.start('lightweight_status_check');
try {
const { batchSize = this.defaultBatchSize, concurrency = this.defaultConcurrency, cache = true, cacheTtl = 180000, // 3 minutes
fields = ['id', 'status', 'updatedAt'], } = options;
// Validate inputs
if (ids.length === 0) {
throw new ValidationError('Entity IDs array cannot be empty', 'ids');
}
if (ids.length > 500) {
throw new ValidationError('Too many IDs provided (max 500)', 'ids');
}
// Check cache first
const cacheKey = `status_${entityType}_${this.hashIds(ids)}_${fields.join(',')}`;
if (cache) {
const cached = this.cache.get(cacheKey);
if (cached) {
performanceCollector.recordCacheHit(metric, true);
performanceCollector.end(metric, true);
return cached;
}
}
performanceCollector.recordCacheHit(metric, false);
// Batch the requests
const batches = this.createBatches(ids, Math.min(batchSize, this.maxBatchSize));
const results = [];
// Process batches with controlled concurrency
for (let i = 0; i < batches.length; i += concurrency) {
const batchSlice = batches.slice(i, i + concurrency);
const batchPromises = batchSlice.map(batch => this.fetchEntityBatch(entityType, batch, fields));
const batchResults = await Promise.allSettled(batchPromises);
for (const result of batchResults) {
if (result.status === 'fulfilled') {
results.push(...result.value);
}
else {
console.warn(`Batch request failed:`, result.reason);
// Continue with other batches instead of failing completely
}
}
}
// Create status summary
const summary = this.createStatusSummary(results, entityType);
// Cache the result
if (cache && summary.total > 0) {
this.cache.set(cacheKey, summary, cacheTtl);
}
performanceCollector.recordDataSize(metric, JSON.stringify(summary).length);
performanceCollector.end(metric, true);
return summary;
}
catch (error) {
performanceCollector.end(metric, false);
throw error;
}
}
/**
* Validate existence of multiple entities
*/
async validateExistence(entityType, ids, options = {}) {
const metric = performanceCollector.start('lightweight_existence_check');
try {
const { batchSize = this.defaultBatchSize, concurrency = this.defaultConcurrency, cache = true, cacheTtl = 300000, // 5 minutes - existence data is more stable
} = options;
if (ids.length === 0) {
return {
missing: [],
existing: [],
total: 0,
existingCount: 0,
missingCount: 0,
};
}
if (ids.length > 1000) {
throw new ValidationError('Too many IDs for existence check (max 1000)', 'ids');
}
// Check cache
const cacheKey = `exists_${entityType}_${this.hashIds(ids)}`;
if (cache) {
const cached = this.cache.get(cacheKey);
if (cached) {
performanceCollector.recordCacheHit(metric, true);
performanceCollector.end(metric, true);
return cached;
}
}
performanceCollector.recordCacheHit(metric, false);
// Use minimal fields for existence check
const minimalFields = ['id'];
const batches = this.createBatches(ids, Math.min(batchSize, this.maxBatchSize));
const existingEntities = [];
// Process batches
for (let i = 0; i < batches.length; i += concurrency) {
const batchSlice = batches.slice(i, i + concurrency);
const batchPromises = batchSlice.map(batch => this.fetchEntityBatch(entityType, batch, minimalFields, true) // Allow partial results
);
const batchResults = await Promise.allSettled(batchPromises);
for (const result of batchResults) {
if (result.status === 'fulfilled') {
existingEntities.push(...result.value);
}
// For existence checks, we continue even if some batches fail
}
}
const existingIds = existingEntities.map(e => e.id);
const missingIds = ids.filter(id => !existingIds.includes(id));
const result = {
missing: missingIds,
existing: existingIds,
total: ids.length,
existingCount: existingIds.length,
missingCount: missingIds.length,
};
// Cache the result
if (cache) {
this.cache.set(cacheKey, result, cacheTtl);
}
performanceCollector.recordDataSize(metric, JSON.stringify(result).length);
performanceCollector.end(metric, true);
return result;
}
catch (error) {
performanceCollector.end(metric, false);
throw error;
}
}
/**
* Track progress for multiple entities with custom markers
*/
async trackBatchProgress(entityType, ids, progressMarker, options = {}) {
const metric = performanceCollector.start('lightweight_progress_tracking');
try {
const { batchSize = this.defaultBatchSize, concurrency = this.defaultConcurrency, cache = true, cacheTtl = 120000, // 2 minutes - progress data changes frequently
fields = ['id', 'status', 'customFields', 'updatedAt'], } = options;
if (ids.length === 0) {
return {
completed: 0,
pending: 0,
total: 0,
summary: 'No entities to track',
};
}
// Check cache
const cacheKey = `progress_${entityType}_${progressMarker}_${this.hashIds(ids)}`;
if (cache) {
const cached = this.cache.get(cacheKey);
if (cached) {
performanceCollector.recordCacheHit(metric, true);
performanceCollector.end(metric, true);
return cached;
}
}
performanceCollector.recordCacheHit(metric, false);
// Fetch entities with progress-relevant fields
const batches = this.createBatches(ids, Math.min(batchSize, this.maxBatchSize));
const entities = [];
for (let i = 0; i < batches.length; i += concurrency) {
const batchSlice = batches.slice(i, i + concurrency);
const batchPromises = batchSlice.map(batch => this.fetchEntityBatch(entityType, batch, fields));
const batchResults = await Promise.allSettled(batchPromises);
for (const result of batchResults) {
if (result.status === 'fulfilled') {
entities.push(...result.value);
}
}
}
// Analyze progress
const progressDetails = entities.map(entity => {
const hasMarker = this.checkProgressMarker(entity, progressMarker);
const detail = {
id: entity.id,
status: entity.status || 'unknown',
progress: hasMarker,
};
if (hasMarker) {
detail.marker = progressMarker;
}
return detail;
});
const completed = progressDetails.filter(p => p.progress).length;
const pending = progressDetails.length - completed;
const result = {
completed,
pending,
total: progressDetails.length,
details: progressDetails,
summary: `${completed}/${progressDetails.length} completed (${Math.round((completed / progressDetails.length) * 100)}%)`,
};
// Cache with shorter TTL for progress data
if (cache) {
this.cache.set(cacheKey, result, cacheTtl);
}
performanceCollector.recordDataSize(metric, JSON.stringify(result).length);
performanceCollector.end(metric, true);
return result;
}
catch (error) {
performanceCollector.end(metric, false);
throw error;
}
}
/**
* Perform health check on the system
*/
async healthCheck() {
const startTime = Date.now();
const checks = {
api: false,
cache: false,
memory: false,
};
try {
// Test API connectivity with a minimal request
await this.testApiConnectivity();
checks.api = true;
}
catch (error) {
console.warn('API health check failed:', error);
}
try {
// Test cache functionality
const testKey = `health_${Date.now()}`;
this.cache.set(testKey, { test: true });
const retrieved = this.cache.get(testKey);
checks.cache = retrieved !== null;
}
catch (error) {
console.warn('Cache health check failed:', error);
}
try {
// Check memory usage
memoryMonitor.getStats();
checks.memory = !memoryMonitor.isCritical(1500); // 1.5GB threshold
}
catch (error) {
console.warn('Memory health check failed:', error);
}
const responseTime = Date.now() - startTime;
const memoryUsage = process.memoryUsage();
const cacheStats = this.cache.getStats();
let status = 'healthy';
if (!checks.api || !checks.cache) {
status = 'degraded';
}
if (!checks.api && !checks.cache) {
status = 'unhealthy';
}
if (!checks.memory) {
status = status === 'healthy' ? 'degraded' : 'unhealthy';
}
return {
status,
responseTime,
timestamp: Date.now(),
checks,
details: {
memoryUsage,
cacheStats,
apiLatency: responseTime,
},
};
}
/**
* Get count of entities without fetching full data
*/
async getEntityCount(entityType, filters, options = {}) {
const metric = performanceCollector.start('lightweight_entity_count');
try {
const { cache = true, cacheTtl = 300000 } = options;
// Create cache key
const filterStr = filters ? JSON.stringify(filters) : 'all';
const cacheKey = `count_${entityType}_${this.hashString(filterStr)}`;
if (cache) {
const cached = this.cache.get(cacheKey);
if (cached) {
performanceCollector.recordCacheHit(metric, true);
performanceCollector.end(metric, true);
return cached;
}
}
performanceCollector.recordCacheHit(metric, false);
// Make a minimal request to get count
const result = await this.fetchEntityCount(entityType, filters);
if (cache) {
this.cache.set(cacheKey, result, cacheTtl);
}
performanceCollector.end(metric, true);
return result;
}
catch (error) {
performanceCollector.end(metric, false);
throw error;
}
}
/**
* Fetch entity batch with minimal fields
*/
async fetchEntityBatch(entityType, ids, fields, allowPartialResults = false) {
try {
// Use the appropriate handler based on entity type
const handler = this.getEntityHandler(entityType);
const params = {
limit: ids.length,
detail: 'basic',
includeSubData: false,
fields: fields.join(','),
// Add ID filter if supported by the API
ids: ids.join(','),
};
const response = await handler('get_' + entityType, params);
const parsedResponse = this.parseResponse(response);
// Filter to only requested IDs and map to lightweight format
const entities = parsedResponse.data
.filter(item => ids.includes(item.id))
.map(item => this.mapToLightweightEntity(item, fields));
return entities;
}
catch (error) {
if (allowPartialResults) {
console.warn(`Partial batch failure for ${entityType}:`, error);
return [];
}
throw error;
}
}
/**
* Test API connectivity with minimal request
*/
async testApiConnectivity() {
// Make a very lightweight API call
try {
await withContext(async () => {
// Simple connectivity test - could be a health endpoint if available
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 5000);
try {
const response = await globalThis.fetch('https://api.productboard.com/health', {
method: 'HEAD',
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok && response.status !== 404) {
throw new NetworkError('API connectivity test failed');
}
}
catch (fetchError) {
clearTimeout(timeoutId);
throw fetchError;
}
});
}
catch {
// If health endpoint doesn't exist, try a minimal actual API call
const handler = this.getEntityHandler('features');
await handler('get_features', { limit: 1, detail: 'basic' });
}
}
/**
* Fetch entity count
*/
async fetchEntityCount(entityType, filters) {
const handler = this.getEntityHandler(entityType);
const params = {
limit: 1, // Minimal request
detail: 'basic',
...filters,
};
const response = await handler('get_' + entityType, params);
const parsedResponse = this.parseResponse(response);
return {
count: parsedResponse.totalRecords || parsedResponse.data?.length || 0,
timestamp: Date.now(),
};
}
/**
* Get appropriate handler for entity type
*/
getEntityHandler(entityType) {
// Simple handler that returns a promise to avoid ESLint issues
return async (operation, _params) => {
// This is a placeholder - in real implementation, would use proper dynamic imports
// For now, throw error to indicate unsupported operation in lightweight mode
throw new ValidationError(`Lightweight query engine does not support ${operation} on ${entityType}`, 'operation');
};
}
/**
* Parse response from tool handlers
*/
parseResponse(response) {
const typedResponse = response;
if (typedResponse?.content?.[0]?.text) {
try {
const parsed = JSON.parse(typedResponse.content[0].text);
return {
data: Array.isArray(parsed.data) ? parsed.data : [],
totalRecords: parsed.totalRecords || 0,
};
}
catch {
return { data: [], totalRecords: 0 };
}
}
return { data: [], totalRecords: 0 };
}
/**
* Map API response to lightweight entity
*/
mapToLightweightEntity(item, fields) {
const entity = { id: item.id };
for (const field of fields) {
if (field === 'status') {
const status = item.status;
entity.status =
(typeof status === 'object' ? status?.name : status) || 'unknown';
}
else if (field === 'updatedAt') {
entity.updatedAt = (item.updatedAt ||
item.updated_at ||
new Date().toISOString());
}
else if (item[field] !== undefined) {
entity[field] = item[field];
}
}
entity.exists = true;
return entity;
}
/**
* Create status summary from entities
*/
createStatusSummary(entities, entityType) {
const statusCounts = {};
let latestUpdate = '';
for (const entity of entities) {
const status = entity.status || 'unknown';
statusCounts[status] = (statusCounts[status] || 0) + 1;
if (entity.updatedAt && entity.updatedAt > latestUpdate) {
latestUpdate = entity.updatedAt;
}
}
// Create human-readable summary
const statusEntries = Object.entries(statusCounts);
const summary = statusEntries.length > 0
? statusEntries
.map(([status, count]) => `${count} ${status}`)
.join(', ')
: 'No entities found';
return {
total: entities.length,
byStatus: statusCounts,
lastUpdated: latestUpdate || new Date().toISOString(),
summary: `${entities.length} ${entityType}: ${summary}`,
entities,
};
}
/**
* Check if entity has progress marker
*/
checkProgressMarker(entity, progressMarker) {
// Check custom fields first
if (entity.customFields &&
entity.customFields[progressMarker]) {
return true;
}
// Check status-based markers
if (progressMarker.startsWith('status:')) {
const targetStatus = progressMarker.replace('status:', '');
return entity.status === targetStatus;
}
// Check field-based markers
if (progressMarker.includes(':')) {
const [field, value] = progressMarker.split(':', 2);
return entity[field] === value;
}
// Default: check if field exists and is truthy
return Boolean(entity[progressMarker]);
}
/**
* Create batches from array of IDs
*/
createBatches(items, batchSize) {
const batches = [];
for (let i = 0; i < items.length; i += batchSize) {
batches.push(items.slice(i, i + batchSize));
}
return batches;
}
/**
* Hash array of IDs for cache keys
*/
hashIds(ids) {
return this.hashString(ids.sort().join(','));
}
/**
* Simple string hash for cache keys
*/
hashString(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = (hash << 5) - hash + char;
hash = hash & hash; // Convert to 32-bit integer
}
return Math.abs(hash).toString(36);
}
}
// Export singleton instance
export const lightweightQueryEngine = new LightweightQueryEngine();