@xiaohui-wang/mcpadvisor
Version:
MCP Advisor & Installation - Find the right MCP server for your needs
336 lines (335 loc) • 13.8 kB
JavaScript
/**
* 增强型离线搜索提供者
* 结合文本匹配和向量搜索，确保即使在向量相似度较低的情况下，也能返回相关的服务器
*/
import { EnhancedMemoryVectorEngine } from '../database/memory/enhancedMemoryVectorEngine.js';
import { getTextEmbedding } from '../../utils/embedding.js';
import logger from '../../utils/logger.js';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
// 获取当前文件的目录
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
/**
* 默认兜底数据路径
*/
const DEFAULT_FALLBACK_DATA_PATH = path.resolve(__dirname, '../../../data/mcp_server_list.json');
/**
* 默认关键词列表
*/
const DEFAULT_KEYWORDS = [
'小红书', 'xiaohongshu', 'rednote', '热点',
'社交媒体', 'social media', '新闻', 'news',
'微博', 'weibo', '知乎', 'zhihu', 'twitter',
'抖音', 'tiktok', '快手', 'kuaishou'
];
/**
* 增强型离线搜索提供者
* 结合文本匹配和向量搜索，确保即使在向量相似度较低的情况下，也能返回相关的服务器
*/
export class EnhancedOfflineSearchProvider {
vectorEngine;
config;
fallbackData = [];
dataLoaded = false;
/**
* 构造函数
* @param config 配置选项
*/
constructor(config = {}) {
this.config = {
minSimilarity: 0.1,
textMatchWeight: 0.5,
vectorSearchWeight: 0.5,
keywords: DEFAULT_KEYWORDS,
...config
};
this.vectorEngine = new EnhancedMemoryVectorEngine(config.fallbackDataPath || DEFAULT_FALLBACK_DATA_PATH);
logger.info('Enhanced offline search provider initialized', {
provider: 'EnhancedOfflineSearchProvider',
config: this.config
});
}
/**
* 加载兜底数据
*/
async loadFallbackData() {
if (this.dataLoaded) {
return;
}
try {
const dataPath = this.config.fallbackDataPath || DEFAULT_FALLBACK_DATA_PATH;
logger.info(`Loading fallback data from: ${dataPath}`);
// 检查文件是否存在
if (!fs.existsSync(dataPath)) {
logger.warn(`Fallback data file not found at: ${dataPath}`);
this.dataLoaded = true;
return;
}
// 读取并解析JSON文件
const rawData = fs.readFileSync(dataPath, 'utf8');
this.fallbackData = JSON.parse(rawData);
logger.info(`Loaded ${this.fallbackData.length} fallback MCP servers`);
this.dataLoaded = true;
}
catch (error) {
const message = error instanceof Error ? error.message : String(error);
logger.error(`Error loading fallback data: ${message}`, { error });
this.dataLoaded = true;
}
}
/**
* 搜索 MCP 服务器
* @param query 搜索查询
* @returns 搜索结果
*/
async search(query) {
try {
logger.info(`Searching for MCP servers with query: ${query}`);
// 确保数据已加载
await this.loadFallbackData();
// 检查查询中是否包含关键词
const containsKeywords = this.checkQueryForKeywords(query);
// 如果查询包含关键词，先进行文本匹配
if (containsKeywords) {
logger.info(`Query contains keywords, prioritizing text matching`);
// 先执行文本匹配
const textMatchResults = await this.textSearch(query);
// 如果文本匹配有结果，则优先返回
if (textMatchResults.length > 0) {
// 再执行向量搜索（不阻塞返回）
this.vectorSearch(query).then(vectorResults => {
logger.debug(`Background vector search found ${vectorResults.length} results`);
}).catch(error => {
logger.error(`Background vector search error: ${error instanceof Error ? error.message : String(error)}`);
});
logger.debug(`Prioritized text match found ${textMatchResults.length} results`);
// 标记结果为离线来源
return textMatchResults.map(result => ({
...result,
source: 'offline'
}));
}
}
// 并行执行向量搜索和文本匹配
const [vectorResults, textMatchResults] = await Promise.all([
this.vectorSearch(query),
this.textSearch(query)
]);
// 合并结果
const mergedResults = this.mergeResults(vectorResults, textMatchResults);
logger.debug(`Found ${mergedResults.length} results from enhanced offline search`);
// 标记结果为离线来源
return mergedResults.map(result => ({
...result,
source: 'offline'
}));
}
catch (error) {
const message = error instanceof Error ? error.message : String(error);
logger.error(`Error in enhanced offline search: ${message}`, {
error,
query,
provider: 'EnhancedOfflineSearchProvider'
});
// 即使出错，也返回空结果而不是抛出异常
return [];
}
}
/**
* 执行向量搜索
* @param query 搜索查询
* @returns 向量搜索结果
*/
async vectorSearch(query) {
try {
// 获取查询的嵌入向量
const queryEmbedding = await getTextEmbedding(query);
// 执行向量搜索
const results = await this.vectorEngine.search(queryEmbedding, 20, // 增加结果数量，以便后续合并
{
minSimilarity: this.config.minSimilarity,
textQuery: query
});
logger.debug(`Found ${results.length} results from vector search`);
return results;
}
catch (error) {
logger.error(`Error in vector search: ${error instanceof Error ? error.message : String(error)}`);
return [];
}
}
/**
* 检查查询中是否包含关键词
* @param query 搜索查询
* @returns 是否包含关键词
*/
checkQueryForKeywords(query) {
const queryLower = query.toLowerCase();
const priorityKeywords = [
'小红书', 'xiaohongshu', 'rednote', '热点', '社交媒体',
'微博', 'weibo', '知乎', 'zhihu'
];
return priorityKeywords.some(keyword => queryLower.includes(keyword.toLowerCase()));
}
/**
* 执行文本匹配搜索
* @param query 搜索查询
* @returns 文本匹配结果
*/
async textSearch(query) {
try {
if (!this.dataLoaded || this.fallbackData.length === 0) {
logger.warn('Fallback data not loaded for text search');
return [];
}
// 准备关键词列表
const keywords = [...(this.config.keywords || [])];
// 从查询中提取额外关键词
const queryWords = query.split(/\s+/).filter(word => word.length > 1);
keywords.push(...queryWords);
// 执行文本匹配
const matchedServers = this.fallbackData.filter(server => {
const searchText = `${server.name} ${server.display_name} ${server.description} ${JSON.stringify(server.categories || [])} ${JSON.stringify(server.tags || [])}`.toLowerCase();
return keywords.some(keyword => searchText.includes(keyword.toLowerCase()));
});
// 转换为 MCPServerResponse 格式
const results = matchedServers.map(server => ({
title: server.display_name || server.name,
description: server.description || '',
github_url: server.repository?.url || server.homepage || '',
categories: server.categories || [],
tags: server.tags || [],
similarity: 0.5 // 默认相似度
}));
// 计算文本相似度分数
for (const result of results) {
// 检查是否包含小红书相关关键词
const isRedNoteRelated = this.isRedNoteRelated(result);
// 计算查询与标题、描述的匹配程度
const titleMatch = this.calculateTextSimilarity(query, result.title);
const descMatch = this.calculateTextSimilarity(query, result.description);
const categoryMatch = Array.isArray(result.categories) ?
Math.max(...result.categories.map(cat => this.calculateTextSimilarity(query, cat))) : 0;
const tagMatch = Array.isArray(result.tags) ?
Math.max(...result.tags.map(tag => this.calculateTextSimilarity(query, tag))) : 0;
// 综合分数 (标题权重高)
let similarity = titleMatch * 0.5 + descMatch * 0.3 + Math.max(categoryMatch, tagMatch) * 0.2;
// 如果是小红书相关，则提高分数
if (isRedNoteRelated) {
similarity = Math.max(similarity, 0.8); // 确保小红书相关结果有足够高的分数
}
result.similarity = Math.max(similarity, result.similarity || 0);
}
// 按相似度排序
const sortedResults = results.sort((a, b) => (b.similarity || 0) - (a.similarity || 0));
logger.debug(`Found ${sortedResults.length} results from text search`);
return sortedResults;
}
catch (error) {
logger.error(`Error in text search: ${error instanceof Error ? error.message : String(error)}`);
return [];
}
}
/**
* 检查是否与小红书相关
*/
isRedNoteRelated(result) {
const redNoteKeywords = [
'小红书', 'xiaohongshu', 'rednote', 'redbook', 'red note'
];
// 检查标题
if (redNoteKeywords.some(keyword => result.title.toLowerCase().includes(keyword.toLowerCase()))) {
return true;
}
// 检查描述
if (redNoteKeywords.some(keyword => result.description.toLowerCase().includes(keyword.toLowerCase()))) {
return true;
}
// 检查分类
if (Array.isArray(result.categories) && result.categories.some(category => redNoteKeywords.some(keyword => category.toLowerCase().includes(keyword.toLowerCase())))) {
return true;
}
// 检查标签
if (Array.isArray(result.tags) && result.tags.some(tag => redNoteKeywords.some(keyword => tag.toLowerCase().includes(keyword.toLowerCase())))) {
return true;
}
return false;
}
/**
* 计算文本相似度
* 简单实现，基于关键词匹配
*/
calculateTextSimilarity(query, text) {
if (!text)
return 0;
const queryLower = query.toLowerCase();
const textLower = text.toLowerCase();
// 完全匹配
if (textLower === queryLower)
return 1.0;
// 包含整个查询
if (textLower.includes(queryLower))
return 0.8;
// 查询词分割匹配
const queryWords = queryLower.split(/\s+/).filter(word => word.length > 1);
let matchCount = 0;
for (const word of queryWords) {
if (textLower.includes(word)) {
matchCount++;
}
}
if (queryWords.length === 0)
return 0;
// 返回匹配词的比例
return matchCount / queryWords.length * 0.6;
}
/**
* 合并向量搜索和文本搜索结果
*/
mergeResults(vectorResults, textResults) {
// 创建结果映射，避免重复
const resultMap = new Map();
// 处理向量结果
vectorResults.forEach(result => {
const key = result.github_url || result.title;
resultMap.set(key, {
...result,
similarity: (result.similarity || 0) * (this.config.vectorSearchWeight || 0.5)
});
});
// 合并文本结果
textResults.forEach(result => {
const key = result.github_url || result.title;
if (resultMap.has(key)) {
// 如果已存在，合并相似度分数
const existing = resultMap.get(key);
resultMap.set(key, {
...existing,
similarity: (existing.similarity || 0) + (result.similarity || 0) * (this.config.textMatchWeight || 0.5)
});
}
else {
// 如果不存在，添加新条目
resultMap.set(key, {
...result,
similarity: (result.similarity || 0) * (this.config.textMatchWeight || 0.5)
});
}
});
// 转换回数组并排序
return Array.from(resultMap.values())
.sort((a, b) => (b.similarity || 0) - (a.similarity || 0));
}
/**
* 设置自定义兜底数据路径
* @param path 数据路径
*/
setFallbackDataPath(path) {
this.config.fallbackDataPath = path;
this.vectorEngine.setFallbackDataPath(path);
this.dataLoaded = false;
logger.info(`Updated fallback data path to: ${path}`);
}
}