@dollhousemcp/mcp-server
Version:
DollhouseMCP - A Model Context Protocol (MCP) server that enables dynamic AI persona management from markdown files, allowing Claude and other compatible AI assistants to activate and switch between different behavioral personas.
1,254 lines • 249 kB
JavaScript
/**
* Unified Index Manager - Combines local, GitHub, and collection portfolio indexing
*
* Features:
* - Unified search across local, GitHub, and collection portfolios
* - Intelligent result merging and deduplication
* - Version conflict detection and resolution
* - Performance optimization with parallel indexing
* - Advanced fallback strategies for resilient operation
* - Comprehensive search capabilities with pagination
* - Smart result ranking and duplicate detection
*/
import { ElementType } from './types.js';
import { logger } from '../utils/logger.js';
import { ErrorHandler, ErrorCategory } from '../utils/ErrorHandler.js';
import { CacheFactory } from '../cache/LRUCache.js';
import { UnicodeValidator } from '../security/validators/unicodeValidator.js';
import { SecurityMonitor } from '../security/securityMonitor.js';
import { getSourcePriorityConfig, ElementSource, getSourceDisplayName } from '../config/sourcePriority.js';
export class UnifiedIndexManager {
localIndexManager;
githubIndexer;
collectionIndexCache;
githubClient;
// Performance monitoring and caching
performanceMonitor;
resultCache;
indexCache;
BATCH_SIZE = 50; // For streaming results
MAX_CONCURRENT_SOURCES = 3;
ownsLocalIndexManager;
ownsGithubIndexer;
ownsCollectionIndexCache;
ownsPerformanceMonitor;
ownsResultCache;
ownsIndexCache;
// Source priority configuration (Issue #1446)
sourcePriorityConfig;
// Lookup tables for optimized enum conversions (Issue #1446 - Code Review)
static SOURCE_ENUM_TO_STRING_MAP = new Map([
[ElementSource.LOCAL, 'local'],
[ElementSource.GITHUB, 'github'],
[ElementSource.COLLECTION, 'collection']
]);
static SOURCE_STRING_TO_ENUM_MAP = new Map([
['local', ElementSource.LOCAL],
['github', ElementSource.GITHUB],
['collection', ElementSource.COLLECTION]
]);
// Cache for source availability checks (Issue #1446 - Code Review)
sourceAvailabilityCache = new Map();
// Telemetry for source usage patterns (Issue #1446 - Code Review)
sourceUsageTelemetry = new Map();
constructor(dependencies) {
this.localIndexManager = dependencies.portfolioIndexManager;
this.ownsLocalIndexManager = false;
this.githubIndexer = dependencies.githubIndexer;
this.ownsGithubIndexer = false;
// Initialize GitHubClient with required dependencies
this.githubClient = dependencies.githubClient;
this.collectionIndexCache = dependencies.collectionIndexCache;
this.ownsCollectionIndexCache = false;
// Initialize and validate source priority configuration (Issue #1446)
this.sourcePriorityConfig = this.initializeSourcePriorityConfig();
// Initialize performance monitoring and caching
this.performanceMonitor = dependencies.performanceMonitor;
this.ownsPerformanceMonitor = false;
if (this.ownsPerformanceMonitor) {
this.performanceMonitor.startMonitoring();
}
this.resultCache =
dependencies.resultCache ??
CacheFactory.createSearchResultCache({
maxSize: 200,
maxMemoryMB: 15,
ttlMs: 5 * 60 * 1000, // 5 minutes
onEviction: (key, value) => {
logger.debug('Search result cache eviction', { key, resultCount: value.length });
}
});
this.ownsResultCache = !dependencies.resultCache;
this.indexCache =
dependencies.indexCache ??
CacheFactory.createIndexCache({
maxSize: 100,
maxMemoryMB: 20,
ttlMs: 15 * 60 * 1000, // 15 minutes
onEviction: (key, _value) => {
logger.debug('Index cache eviction', { key });
}
});
this.ownsIndexCache = !dependencies.indexCache;
logger.debug('UnifiedIndexManager created with performance optimization');
}
/**
* Enhanced search across local, GitHub, and collection portfolios with performance optimization
*
* Implements source priority-based search (Issue #1446):
* - Sources are searched sequentially in priority order (local → github → collection by default)
* - Early termination when stopOnFirst is enabled and results are found
* - Configurable priority order via sourcePriority or preferredSource options
* - includeAll option forces search of all sources for comprehensive results
*/
async search(searchOptions) {
const startTime = Date.now();
const memoryBefore = process.memoryUsage().heapUsed;
// Normalize and validate search options
const normalizedOptions = this.normalizeSearchOptions(searchOptions);
// Log security audit event
this.logSearchSecurityEvent(normalizedOptions, searchOptions);
logger.debug('Starting optimized unified portfolio search', normalizedOptions);
// Check cache first
const cachedResult = await this.checkSearchCache(normalizedOptions, startTime, memoryBefore);
if (cachedResult) {
return cachedResult;
}
try {
// Handle streaming search separately
if (normalizedOptions.streamResults) {
return await this.streamSearch(normalizedOptions);
}
// Perform priority-based search
const { results, sourceCount, enabledSources } = await this.performPriorityBasedSearch(normalizedOptions);
// Process, paginate, and cache results
const finalResults = await this.processFinalResults(results, normalizedOptions, startTime, memoryBefore, sourceCount, enabledSources);
return finalResults;
}
catch (error) {
const duration = Date.now() - startTime;
ErrorHandler.logError('UnifiedIndexManager.search', error, { query: normalizedOptions, duration });
throw ErrorHandler.wrapError(error, 'Failed to perform unified portfolio search', ErrorCategory.SYSTEM_ERROR);
}
}
// ========================================================================
// Search Helper Methods (extracted for reduced cognitive complexity)
// ========================================================================
/**
* Normalize search options (extracted from search())
*/
normalizeSearchOptions(searchOptions) {
const { query } = searchOptions;
const validationResult = UnicodeValidator.normalize(query);
const normalizedQuery = validationResult.normalizedContent;
return {
...searchOptions,
query: normalizedQuery
};
}
/**
* Log security event for search operation (extracted from search())
*/
logSearchSecurityEvent(normalizedOptions, originalOptions) {
const { includeLocal = true, includeGitHub = true, includeCollection = false } = originalOptions;
SecurityMonitor.logSecurityEvent({
type: 'PORTFOLIO_FETCH_SUCCESS',
severity: 'LOW',
source: 'UnifiedIndexManager.search',
details: `Unified search performed with query length: ${normalizedOptions.query.length}, sources: ${JSON.stringify({
local: includeLocal,
github: includeGitHub,
collection: includeCollection
})}`
});
}
/**
* Check search cache and return cached results if available (extracted from search())
*/
async checkSearchCache(normalizedOptions, startTime, memoryBefore) {
const cacheKey = this.createCacheKey(normalizedOptions);
const cached = this.resultCache.get(cacheKey);
if (cached) {
const duration = Date.now() - startTime;
this.recordSearchMetrics({
query: normalizedOptions.query,
duration,
resultCount: cached.length,
sources: this.getEnabledSources(normalizedOptions),
cacheHit: true,
memoryBefore,
memoryAfter: process.memoryUsage().heapUsed,
timestamp: new Date()
});
logger.debug('Using cached search results', { resultCount: cached.length });
return cached;
}
return null;
}
/**
* Perform priority-based search across sources (extracted from search())
*
* This is the core implementation of source priority search (Issue #1446):
* - Searches sources sequentially in priority order
* - Supports early termination with stopOnFirst
* - Handles fallback on error
* - Records telemetry for each source
*/
async performPriorityBasedSearch(normalizedOptions) {
const allResults = [];
const sourceCount = { local: 0, github: 0, collection: 0 };
// Get enabled sources in priority order
const enabledSources = this.getEnabledSourcesByPriority(normalizedOptions);
// Determine if we should stop on first result
const stopOnFirst = normalizedOptions.includeAll ? false : this.sourcePriorityConfig.stopOnFirst;
// Search sources sequentially in priority order
for (const source of enabledSources) {
const shouldContinue = await this.searchSingleSource(source, normalizedOptions, allResults, sourceCount, stopOnFirst);
if (!shouldContinue) {
break;
}
// Memory check between sources
this.checkMemoryUsage();
}
return { results: allResults, sourceCount, enabledSources };
}
/**
* Search a single source and update results (extracted from performPriorityBasedSearch())
*/
async searchSingleSource(source, normalizedOptions, allResults, sourceCount, stopOnFirst) {
const sourceStartTime = Date.now();
try {
const sourceResults = await this.searchWithFallback(source, normalizedOptions.query, normalizedOptions);
const sourceDuration = Date.now() - sourceStartTime;
// Record telemetry for this source
this.recordSourceUsage(source, sourceResults.length, sourceDuration);
if (sourceResults.length > 0) {
sourceCount[source] += sourceResults.length;
allResults.push(...sourceResults);
logger.debug(`Found ${sourceResults.length} results in ${getSourceDisplayName(this.mapSourceStringToEnum(source))}`, {
source,
resultCount: sourceResults.length,
duration: `${sourceDuration}ms`
});
// Early termination if stopOnFirst is enabled and we found results
if (stopOnFirst && allResults.length > 0) {
logger.debug('Stopping search early (stopOnFirst enabled)', {
source,
totalResults: allResults.length
});
return false;
}
}
}
catch (error) {
const shouldFallback = this.sourcePriorityConfig.fallbackOnError;
if (shouldFallback) {
logger.warn(`Search failed for source ${source}, continuing to next source`, {
error: error instanceof Error ? error.message : String(error),
source
});
}
else {
logger.error(`Search failed for source ${source}, halting search`, {
error: error instanceof Error ? error.message : String(error),
source
});
throw error;
}
}
return true; // Continue to next source
}
/**
* Check memory usage and trigger cleanup if needed (extracted from performPriorityBasedSearch())
*/
checkMemoryUsage() {
const currentMemory = process.memoryUsage().heapUsed / (1024 * 1024);
if (currentMemory > 200) { // 200MB threshold
logger.warn('High memory usage during search, triggering cleanup', {
memoryMB: currentMemory
});
this.triggerMemoryCleanup();
}
}
/**
* Process final results: apply processing, pagination, caching, and metrics (extracted from search())
*/
async processFinalResults(results, normalizedOptions, startTime, memoryBefore, sourceCount, enabledSources) {
// Apply advanced processing with memory-efficient batching
const processedResults = await this.processSearchResultsOptimized(results, normalizedOptions);
// Apply pagination
const paginatedResults = this.applyPagination(processedResults, normalizedOptions);
// Cache results with memory limit check
if (paginatedResults.length < 1000) {
const cacheKey = this.createCacheKey(normalizedOptions);
this.resultCache.set(cacheKey, paginatedResults);
}
// Record metrics and log completion
const duration = Date.now() - startTime;
const memoryAfter = process.memoryUsage().heapUsed;
this.recordSearchMetrics({
query: normalizedOptions.query,
duration,
resultCount: paginatedResults.length,
sources: enabledSources,
cacheHit: false,
memoryBefore,
memoryAfter,
timestamp: new Date()
});
logger.info('Optimized unified portfolio search completed', {
query: normalizedOptions.query.substring(0, 50),
sources: { ...sourceCount, total: results.length },
finalResults: paginatedResults.length,
duration: `${duration}ms`,
memoryUsageMB: (memoryAfter - memoryBefore) / (1024 * 1024)
});
return paginatedResults;
}
/**
* Find element by name across all portfolios
*/
async findByName(name, options = {}) {
try {
const searchOptions = {
query: name,
includeLocal: options.includeLocal ?? true,
includeGitHub: options.includeGitHub ?? true,
includeCollection: options.includeCollection ?? false,
pageSize: 1,
...options
};
const results = await this.search(searchOptions);
// Return exact name match first, then best match
const exactMatch = results.find(result => result.entry.name.toLowerCase() === name.toLowerCase());
return exactMatch?.entry || results[0]?.entry || null;
}
catch (error) {
ErrorHandler.logError('UnifiedIndexManager.findByName', error, { name });
return null;
}
}
/**
* Get elements by type from all portfolios
*/
async getElementsByType(elementType, options = {}) {
try {
const searchOptions = {
query: '', // Empty query to get all elements
elementType,
includeLocal: options.includeLocal ?? true,
includeGitHub: options.includeGitHub ?? true,
includeCollection: options.includeCollection ?? false,
pageSize: 1000, // Large page size to get all
...options
};
const results = await this.getAllElementsByType(elementType, searchOptions);
return this.deduplicateEntries(results.map(r => r.entry));
}
catch (error) {
ErrorHandler.logError('UnifiedIndexManager.getElementsByType', error, { elementType });
return [];
}
}
/**
* Check for duplicates across all sources
*/
async checkDuplicates(name) {
try {
const searchOptions = {
query: name,
includeLocal: true,
includeGitHub: true,
includeCollection: true,
pageSize: 100
};
const results = await this.search(searchOptions);
const duplicateMap = new Map();
for (const result of results) {
const key = `${result.entry.elementType}:${result.entry.name.toLowerCase()}`;
if (!duplicateMap.has(key)) {
duplicateMap.set(key, {
name: result.entry.name,
elementType: result.entry.elementType,
sources: [],
hasVersionConflict: false
});
}
const duplicate = duplicateMap.get(key);
duplicate.sources.push({
source: result.source,
version: result.entry.version,
lastModified: result.entry.lastModified,
path: this.getPathFromEntry(result.entry)
});
}
// Filter to only items with multiple sources and check version conflicts
const actualDuplicates = Array.from(duplicateMap.values())
.filter(item => item.sources.length > 1)
.map(item => {
const versionConflict = this.detectVersionConflict(item.sources);
return {
...item,
hasVersionConflict: !!versionConflict,
versionConflict
};
});
return actualDuplicates;
}
catch (error) {
ErrorHandler.logError('UnifiedIndexManager.checkDuplicates', error, { name });
return [];
}
}
/**
* Get version comparison across all sources
*/
async getVersionComparison(name) {
try {
const duplicates = await this.checkDuplicates(name);
if (duplicates.length === 0) {
return null;
}
const duplicate = duplicates[0];
const versions = {};
// Build version info
for (const source of duplicate.sources) {
if (source.source === 'local') {
versions.local = {
version: source.version || 'unknown',
lastModified: source.lastModified,
path: source.path || 'unknown'
};
}
else if (source.source === 'github') {
versions.github = {
version: source.version || 'unknown',
lastModified: source.lastModified,
path: source.path || 'unknown'
};
}
else if (source.source === 'collection') {
versions.collection = {
version: source.version || 'unknown',
lastModified: source.lastModified,
path: source.path || 'unknown'
};
}
}
// Determine recommendation
const recommendation = this.determineVersionRecommendation(versions);
return {
name: duplicate.name,
elementType: duplicate.elementType,
versions,
recommended: recommendation,
updateAvailable: recommendation.source !== 'local' && !!versions.local,
updateFrom: recommendation.source !== 'local' ? recommendation.source : undefined
};
}
catch (error) {
ErrorHandler.logError('UnifiedIndexManager.getVersionComparison', error, { name });
return null;
}
}
/**
* Get comprehensive statistics across all sources
*/
async getStats() {
try {
const [localStats, githubStats, collectionStats] = await Promise.allSettled([
this.getLocalStats(),
this.getGitHubStats(),
this.getCollectionStats()
]);
const local = localStats.status === 'fulfilled' ? localStats.value : {
totalElements: 0,
elementsByType: {},
lastBuilt: null,
isStale: true
};
const github = githubStats.status === 'fulfilled' ? githubStats.value : {
totalElements: 0,
elementsByType: {},
lastFetched: null,
isStale: true
};
const collection = collectionStats.status === 'fulfilled' ? collectionStats.value : {
totalElements: 0,
elementsByType: {},
lastFetched: null,
isStale: true
};
// Calculate combined statistics
const totalElements = local.totalElements + github.totalElements + collection.totalElements;
const duplicatesCount = await this.calculateDuplicatesCount();
const uniqueElements = totalElements - duplicatesCount;
return {
local,
github,
collection,
combined: {
totalElements,
uniqueElements,
duplicates: duplicatesCount
},
performance: {
averageSearchTime: this.getPerformanceStats().searchStats.averageTime || 0,
cacheHitRate: this.getPerformanceStats().searchStats.cacheHitRate || 0,
lastOptimized: null
}
};
}
catch (error) {
ErrorHandler.logError('UnifiedIndexManager.getStats', error);
throw error;
}
}
/**
* Invalidate caches after user actions with performance monitoring
*/
invalidateAfterAction(action) {
logger.info('Invalidating unified portfolio caches after user action', { action });
// Clear result and index caches
this.resultCache.clear();
this.indexCache.clear();
// Invalidate local cache
this.localIndexManager.rebuildIndex().catch(error => {
logger.warn('Failed to rebuild local index after action', {
action,
error: error instanceof Error ? error.message : String(error)
});
});
// Invalidate GitHub cache
this.githubIndexer.invalidateAfterAction(action);
// Invalidate collection cache
this.collectionIndexCache.clearCache().catch(error => {
logger.warn('Failed to clear collection cache after action', {
action,
error: error instanceof Error ? error.message : String(error)
});
});
// Trigger garbage collection if memory usage is high
this.triggerMemoryCleanup();
}
/**
* Force rebuild of all indexes with performance optimization
*/
async rebuildAll() {
const startTime = Date.now();
logger.info('Rebuilding all portfolio indexes with optimization...');
try {
// Clear all caches
this.resultCache.clear();
this.indexCache.clear();
// Reset performance counters
this.performanceMonitor.reset();
// Rebuild in parallel with memory monitoring
const rebuildPromises = [
this.localIndexManager.rebuildIndex(),
this.githubIndexer.clearCache(),
this.collectionIndexCache.clearCache()
];
await Promise.all(rebuildPromises);
// Trigger cleanup
this.triggerMemoryCleanup();
const duration = Date.now() - startTime;
logger.info('All portfolio indexes rebuilt successfully', {
duration: `${duration}ms`,
memoryUsageMB: process.memoryUsage().heapUsed / (1024 * 1024)
});
}
catch (error) {
ErrorHandler.logError('UnifiedIndexManager.rebuildAll', error);
throw error;
}
}
// =====================================================
// PRIVATE HELPER METHODS
// =====================================================
/**
* Search with fallback strategies for resilient operation
*/
async searchWithFallback(source, query, options) {
const startTime = Date.now();
try {
let results = [];
switch (source) {
case 'local':
results = await this.searchLocal(query, options);
break;
case 'github':
results = await this.searchGitHub(query, options);
break;
case 'collection':
results = await this.searchCollection(query, options);
break;
}
logger.debug(`${source} search completed in ${Date.now() - startTime}ms with ${results.length} results`);
return results;
}
catch (error) {
logger.debug(`${source} search failed, attempting fallback`, {
error: error instanceof Error ? error.message : String(error)
});
// Fallback strategies
return await this.handleSearchFallback(source, query, options, error);
}
}
/**
* Handle search fallback strategies
*/
async handleSearchFallback(source, query, options, originalError) {
try {
switch (source) {
case 'local':
// Try to use stale local index
logger.debug('Attempting to use stale local index');
return await this.searchLocalStale(query, options);
case 'github':
// Try cached GitHub data
logger.debug('Attempting to use cached GitHub data');
return await this.searchGitHubCached(query, options);
case 'collection':
// Try cached collection data
logger.debug('Attempting to use cached collection data');
return await this.searchCollectionCached(query, options);
default:
return [];
}
}
catch (fallbackError) {
logger.warn(`All fallback strategies failed for ${source}`, {
originalError: originalError instanceof Error ? originalError.message : String(originalError),
fallbackError: fallbackError instanceof Error ? fallbackError.message : String(fallbackError)
});
return [];
}
}
/**
* Search local portfolio
*/
async searchLocal(query, options) {
const localOptions = this.convertToLocalOptions(options);
const results = await this.localIndexManager.search(query, localOptions);
return results.map(result => ({
source: 'local',
entry: this.convertLocalEntry(result.entry),
matchType: result.matchType,
score: result.score,
version: result.entry.metadata.version
}));
}
/**
* Search local with stale data fallback
*/
async searchLocalStale(query, options) {
try {
// Try to get any local data, even stale
const localOptions = this.convertToLocalOptions(options);
const results = await this.localIndexManager.search(query, localOptions);
return results.map(result => ({
source: 'local',
entry: this.convertLocalEntry(result.entry),
matchType: result.matchType,
score: result.score * 0.8, // Reduce score for stale data
version: result.entry.metadata.version
}));
}
catch {
return [];
}
}
/**
* Search GitHub portfolio
*/
async searchGitHub(query, options) {
try {
const githubIndex = await this.githubIndexer.getIndex();
const results = [];
const queryLower = query.toLowerCase();
const queryTokens = queryLower.split(/\s+/).filter(token => token.length > 0);
if (queryTokens.length === 0 && query.trim() !== '') {
return results;
}
// Search across all GitHub elements
for (const [elementType, entries] of githubIndex.elements) {
// Filter by element type if specified
if (options.elementType && elementType !== options.elementType) {
continue;
}
for (const entry of entries) {
const score = this.calculateGitHubMatchScore(entry, queryTokens, query);
if (score > 0 || query.trim() === '') {
results.push({
source: 'github',
entry: this.convertGitHubEntry(entry),
matchType: this.determineMatchType(entry, queryTokens),
score: query.trim() === '' ? 1 : score, // Default score for empty query
version: entry.version
});
}
}
}
return results.sort((a, b) => b.score - a.score);
}
catch (error) {
logger.debug('GitHub search failed', {
error: error instanceof Error ? error.message : String(error)
});
throw error; // Re-throw to trigger fallback
}
}
/**
* Search GitHub with cached data fallback
*/
async searchGitHubCached(query, options) {
try {
// Try to use stale GitHub data
const cacheStats = this.githubIndexer.getCacheStats();
if (!cacheStats.isStale) {
return await this.searchGitHub(query, options);
}
// Use stale data with reduced scores
const results = await this.searchGitHub(query, options);
return results.map(result => ({
...result,
score: result.score * 0.7 // Reduce score for stale data
}));
}
catch {
return [];
}
}
/**
* Search collection portfolio
*/
async searchCollection(query, options) {
try {
const collectionIndex = await this.collectionIndexCache.getIndex();
const results = [];
const queryLower = query.toLowerCase();
const queryTokens = queryLower.split(/\s+/).filter(token => token.length > 0);
if (queryTokens.length === 0 && query.trim() !== '') {
return results;
}
// Search across all collection elements
for (const [elementType, entries] of Object.entries(collectionIndex.index)) {
// Filter by element type if specified
if (options.elementType && elementType !== options.elementType.toString()) {
continue;
}
for (const entry of entries) {
const score = this.calculateCollectionMatchScore(entry, queryTokens, query);
if (score > 0 || query.trim() === '') {
results.push({
source: 'collection',
entry: this.convertCollectionEntry(entry, elementType),
matchType: this.determineCollectionMatchType(entry, queryTokens),
score: query.trim() === '' ? 1 : score, // Default score for empty query
version: entry.version
});
}
}
}
return results.sort((a, b) => b.score - a.score);
}
catch (error) {
logger.debug('Collection search failed', {
error: error instanceof Error ? error.message : String(error)
});
throw error; // Re-throw to trigger fallback
}
}
/**
* Search collection with cached data fallback
*/
async searchCollectionCached(query, options) {
try {
// Try to use stale collection data
const cacheStats = this.collectionIndexCache.getCacheStats();
if (cacheStats.isValid) {
return await this.searchCollection(query, options);
}
// Use stale data with reduced scores
const results = await this.searchCollection(query, options);
return results.map(result => ({
...result,
score: result.score * 0.6 // Reduce score for stale collection data
}));
}
catch {
return [];
}
}
/**
* Process search results with advanced features
*/
async processSearchResults(results, options) {
// Apply smart ranking
const rankedResults = this.applySmartRanking(results, options);
// Detect duplicates and version conflicts
const processedResults = await this.detectDuplicatesAndConflicts(rankedResults);
// Apply sorting
const sortedResults = this.applySorting(processedResults, options.sortBy || 'relevance', options.query);
return sortedResults;
}
/**
* Apply smart result ranking
*/
applySmartRanking(results, options) {
return results.map(result => {
let adjustedScore = result.score;
// No location-based scoring - score should be based on relevance only
// Source location doesn't affect the intrinsic value of an element
// Consider version freshness (newer versions get small bonus)
if (result.version && result.version !== 'unknown') {
const versionParts = result.version.split('.');
if (versionParts.length >= 2) {
const major = Number.parseInt(versionParts[0]) || 0;
const minor = Number.parseInt(versionParts[1]) || 0;
adjustedScore += (major * 0.1) + (minor * 0.01);
}
}
// Boost exact matches
if (result.entry.name.toLowerCase() === options.query.toLowerCase()) {
adjustedScore *= 2.0;
}
return {
...result,
score: adjustedScore
};
});
}
/**
* Detect duplicates and version conflicts
*/
async detectDuplicatesAndConflicts(results) {
const nameMap = new Map();
// Group by name and element type
for (const result of results) {
const key = `${result.entry.elementType}:${result.entry.name.toLowerCase()}`;
if (!nameMap.has(key)) {
nameMap.set(key, []);
}
nameMap.get(key).push(result);
}
const processedResults = [];
// Process each group
for (const [, groupResults] of nameMap) {
if (groupResults.length === 1) {
// No duplicates
processedResults.push(groupResults[0]);
}
else {
// Has duplicates - detect version conflicts
const versionConflict = this.detectVersionConflictFromResults(groupResults);
// Mark all results as duplicates and add conflict info
for (const result of groupResults) {
processedResults.push({
...result,
isDuplicate: true,
versionConflict
});
}
}
}
return processedResults;
}
/**
* Apply pagination to results
*/
applyPagination(results, options) {
const page = options.page || 1;
const pageSize = options.pageSize || 20;
const startIndex = (page - 1) * pageSize;
const endIndex = startIndex + pageSize;
return results.slice(startIndex, endIndex);
}
/**
* Apply sorting to results
*/
applySorting(results, sortBy, _query) {
const sorted = [...results];
switch (sortBy) {
case 'name':
sorted.sort((a, b) => a.entry.name.localeCompare(b.entry.name));
break;
case 'source':
sorted.sort((a, b) => {
const sourceOrder = { 'local': 0, 'github': 1, 'collection': 2 };
return sourceOrder[a.source] - sourceOrder[b.source];
});
break;
case 'version':
sorted.sort((a, b) => this.compareVersions(b.version || '0', a.version || '0'));
break;
case 'relevance':
default:
sorted.sort((a, b) => b.score - a.score);
break;
}
return sorted;
}
/**
* Calculate match score for GitHub entries
*/
calculateGitHubMatchScore(entry, queryTokens, query) {
if (queryTokens.length === 0)
return 1; // Default score for empty query
let score = 0;
const name = entry.name.toLowerCase();
const description = (entry.description || '').toLowerCase();
const path = (entry.path || '').toLowerCase();
// Check name matches
for (const token of queryTokens) {
if (name.includes(token)) {
score += name === token ? 10 : (name.startsWith(token) ? 5 : 2);
}
if (description.includes(token)) {
score += 3;
}
if (path.includes(token)) {
score += 1;
}
}
// Exact query match bonus
if (name.includes(query.toLowerCase())) {
score += query.length > 3 ? 15 : 10;
}
return score;
}
/**
* Calculate match score for collection entries
*/
calculateCollectionMatchScore(entry, queryTokens, query) {
if (queryTokens.length === 0)
return 1; // Default score for empty query
let score = 0;
const name = entry.name.toLowerCase();
const description = (entry.description || '').toLowerCase();
const path = (entry.path || '').toLowerCase();
const tags = entry.tags.map(tag => tag.toLowerCase()).join(' ');
// Check matches across all fields
for (const token of queryTokens) {
if (name.includes(token)) {
score += name === token ? 10 : (name.startsWith(token) ? 5 : 2);
}
if (description.includes(token)) {
score += 3;
}
if (path.includes(token)) {
score += 1;
}
if (tags.includes(token)) {
score += 4;
}
}
// Exact query match bonus
if (name.includes(query.toLowerCase())) {
score += query.length > 3 ? 15 : 10;
}
return score;
}
/**
* Get all elements by type across sources
*/
async getAllElementsByType(elementType, options) {
const promises = [];
if (options.includeLocal) {
promises.push(this.getLocalElementsByType(elementType));
}
if (options.includeGitHub) {
promises.push(this.getGitHubElementsByType(elementType));
}
if (options.includeCollection) {
promises.push(this.getCollectionElementsByType(elementType));
}
const results = await Promise.allSettled(promises);
const allResults = [];
results.forEach(result => {
if (result.status === 'fulfilled') {
allResults.push(...result.value);
}
});
return allResults;
}
/**
* Get local elements by type
*/
async getLocalElementsByType(elementType) {
try {
const elements = await this.localIndexManager.getElementsByType(elementType);
return elements.map(entry => ({
source: 'local',
entry: this.convertLocalEntry(entry),
matchType: 'type',
score: 1,
version: entry.metadata.version
}));
}
catch {
return [];
}
}
/**
* Get GitHub elements by type
*/
async getGitHubElementsByType(elementType) {
try {
const githubIndex = await this.githubIndexer.getIndex();
const entries = githubIndex.elements.get(elementType) || [];
return entries.map(entry => ({
source: 'github',
entry: this.convertGitHubEntry(entry),
matchType: 'type',
score: 1,
version: entry.version
}));
}
catch {
return [];
}
}
/**
* Get collection elements by type
*/
async getCollectionElementsByType(elementType) {
try {
const collectionIndex = await this.collectionIndexCache.getIndex();
const entries = collectionIndex.index[elementType.toString()] || [];
return entries.map(entry => ({
source: 'collection',
entry: this.convertCollectionEntry(entry, elementType.toString()),
matchType: 'type',
score: 1,
version: entry.version
}));
}
catch {
return [];
}
}
/**
* Get local portfolio statistics
*/
async getLocalStats() {
return await this.localIndexManager.getStats();
}
/**
* Get GitHub portfolio statistics
*/
async getGitHubStats() {
const cacheStats = this.githubIndexer.getCacheStats();
const githubIndex = await this.githubIndexer.getIndex();
const elementsByType = {};
for (const elementType of Object.values(ElementType)) {
elementsByType[elementType] = (githubIndex.elements.get(elementType) || []).length;
}
return {
totalElements: githubIndex.totalElements,
elementsByType,
lastFetched: cacheStats.lastFetch,
isStale: cacheStats.isStale,
username: githubIndex.username,
repository: githubIndex.repository
};
}
/**
* Get collection portfolio statistics
*/
async getCollectionStats() {
const cacheStats = this.collectionIndexCache.getCacheStats();
const collectionIndex = await this.collectionIndexCache.getIndex();
const elementsByType = {};
for (const [elementType, entries] of Object.entries(collectionIndex.index)) {
elementsByType[elementType] = entries.length;
}
return {
totalElements: collectionIndex.total_elements,
elementsByType,
lastFetched: cacheStats.hasCache ? new Date(Date.now() - cacheStats.age) : null,
isStale: !cacheStats.isValid,
version: collectionIndex.version
};
}
/**
* Calculate duplicates count across all sources
*
* Identifies elements that exist in multiple sources (local, GitHub, collection)
*/
async calculateDuplicatesCount() {
// Track duplicates using a map of element names to sources
const elementSources = new Map();
try {
// Check local index
const localIndex = await this.localIndexManager.getIndex();
if (localIndex?.byName) {
for (const [name] of localIndex.byName) {
if (name) {
if (!elementSources.has(name)) {
elementSources.set(name, new Set());
}
elementSources.get(name).add('local');
}
}
}
// Check GitHub index
const githubIndex = await this.githubIndexer.getIndex();
if (githubIndex?.elements) {
for (const [, entries] of githubIndex.elements) {
for (const entry of entries) {
const name = entry.name;
if (name) {
if (!elementSources.has(name)) {
elementSources.set(name, new Set());
}
elementSources.get(name).add('github');
}
}
}
}
// Check collection index
const collectionIndex = await this.collectionIndexCache.getIndex();
if (collectionIndex?.index) {
for (const elementType in collectionIndex.index) {
const entries = collectionIndex.index[elementType];
for (const entry of entries) {
const name = entry.name;
if (name) {
if (!elementSources.has(name)) {
elementSources.set(name, new Set());
}
elementSources.get(name).add('collection');
}
}
}
}
}
catch (error) {
// Log error but don't fail
logger.debug('Error calculating duplicates count', error);
return 0;
}
// Count elements that appear in more than one source
let duplicateCount = 0;
for (const sources of elementSources.values()) {
if (sources.size > 1) {
duplicateCount++;
}
}
return duplicateCount;
}
/**
* Convert local index entry to unified format
*/
convertLocalEntry(entry) {
return {
name: entry.metadata.name,
description: entry.metadata.description,
version: entry.metadata.version,
author: entry.metadata.author,
elementType: entry.elementType,
lastModified: entry.lastModified,
source: 'local',
localFilePath: entry.filePath,
filename: entry.filename,
tags: entry.metadata.tags,
keywords: entry.metadata.keywords,
triggers: entry.metadata.triggers,
category: entry.metadata.category
};
}
/**
* Convert GitHub index entry to unified format
*/
convertGitHubEntry(entry) {
return {
name: entry.name,
description: entry.description,
version: entry.version,
author: entry.author,
elementType: entry.elementType,
lastModified: entry.lastModified,
source: 'github',
githubPath: entry.path,
githubSha: entry.sha,
githubHtmlUrl: entry.htmlUrl,
githubDownloadUrl: entry.downloadUrl,
githubSize: entry.size
};
}
/**
* Convert collection index entry to unified format
*/
convertCollectionEntry(entry, elementType) {
return {
name: entry.name,
description: entry.description,
version: entry.version,
author: entry.author,
elementType: this.mapStringToElementType(elementType),
lastModified: new Date(entry.created),
source: 'collection',
collectionPath: entry.path,
collectionSha: entry.sha,
collectionTags: entry.tags,
collectionCategory: entry.category,
collectionLicense: entry.license
};
}
/**
* Map string to ElementType enum
*/
mapStringToElementType(elementType) {
// Handle mapping from collection element types to our ElementType enum
switch (elementType.toLowerCase()) {
case 'personas':
return ElementType.PERSONA;
case 'skills':
return ElementType.SKILL;
case 'agents':
return ElementType.AGENT;
case 'prompts':
case 'templates':
return ElementType.TEMPLATE; // Map prompts and templates to TEMPLATE
case 'tools':
return ElementType.SKILL; // Map tools to SKILL as fallback
case 'ensembles':
return ElementType.ENSEMBLE;
case 'memories':
return ElementType.MEMORY;
default:
return ElementType.SKILL; // Default fallback
}
}
/**
* Convert unified search options to local search options
*/
convertToLocalOptions(options) {
return {
elementType: options.elementType,
maxResults: options.pageSize || 20
};
}
/**
* Determine match type for GitHub entries
*/
determineMatchType(entry, queryTokens) {
const name = entry.name.toLowerCase();
const description = (entry.description || '').toLowerCase();
// Check what matched
for (const token of queryTokens) {
if (name.includes(token)) {
return name === token ? 'exact_name' : 'name';
}
if (description.includes(token)) {
return 'description';
}
}
return 'content';
}
/**
* Determine match type for collection entries
*/
determineCollectionMatch