UNPKG

@claude-vector/core

Version:

Core vector search engine for code intelligence

876 lines (769 loc) 27.3 kB
/** * Semantic Chunker - AST-based intelligent code chunking * * パラメトリック思考によるAI駆動開発最適化: * - 開発フェーズ適応型チャンキング * - セマンティック境界の保持 * - 複雑度ベースの動的サイズ調整 */ import { parse } from '@babel/parser'; import _traverse from '@babel/traverse'; import * as t from '@babel/types'; import fs from 'fs/promises'; import path from 'path'; // @babel/traverse の ESM import の修正 const traverse = _traverse.default || _traverse; export class SemanticChunker { constructor(options = {}) { // エラー統計の初期化 this.parseErrors = { total: 0, byFileType: {}, byErrorType: {}, failedFiles: [] }; this.options = { // 開発フェーズ別設定 phases: { exploration: { chunkSize: 'large', minLines: 50, maxLines: 200, includeContext: true, preserveClasses: true }, implementation: { chunkSize: 'medium', minLines: 5, maxLines: 300, // 500から削減してトークン制限に対応 maxTokens: 6000, // 新規追加: OpenAI APIの制限を考慮 includeContext: true, preserveFunctions: true }, debugging: { chunkSize: 'small', minLines: 5, maxLines: 30, includeContext: false, preserveBlocks: true }, refactoring: { chunkSize: 'variable', minLines: 10, maxLines: 100, includeContext: true, preserveStructure: true } }, defaultPhase: 'implementation', ...options }; } /** * ファイルをセマンティックチャンクに分割 */ async chunkFile(filePath, content, developmentPhase = 'implementation') { const fileExtension = path.extname(filePath).toLowerCase(); const language = this.detectLanguage(fileExtension); // デバッグ: openweathermap-adapter.ts の処理を追跡 if (filePath.includes('openweathermap-adapter')) { console.log(`🔍 SemanticChunker processing openweathermap-adapter.ts:`); console.log(` Extension: ${fileExtension}, Language: ${language}`); console.log(` Content length: ${content.length} chars`); console.log(` Is code file: ${this.isCodeFile(language)}`); } if (!this.isCodeFile(language)) { return this.chunkNonCodeFile(filePath, content, language); } try { const ast = this.parseCode(content, language); const chunks = this.extractSemanticChunks(ast, content, filePath, developmentPhase); if (filePath.includes('openweathermap-adapter')) { console.log(`📦 Extracted ${chunks.length} semantic chunks from openweathermap-adapter.ts`); } return chunks.map(chunk => ({ ...chunk, metadata: { ...chunk.metadata, file: filePath, language, developmentPhase, chunkingMethod: 'semantic' } })); } catch (error) { // エラー統計の更新 this.updateErrorStats(filePath, language, error); // 改善されたエラーメッセージ const fileType = path.extname(filePath); const errorType = this.categorizeError(error); console.warn(`⚠️ AST parsing failed for ${filePath}, falling back to simple chunking: ${errorType}`); // 詳細なデバッグ情報(Next.js APIルート等の特殊構文対応) if (this.options.verboseErrors || filePath.includes('route.ts')) { console.warn(` File type: ${fileType}, Language: ${language}`); console.warn(` Error details: ${error.message}`); console.warn(` Parser plugins: ${JSON.stringify(this.getParserPlugins(language))}`); if (filePath.includes('/api/') || filePath.includes('route.')) { console.warn(` Note: This appears to be a Next.js API route file`); } } return this.fallbackChunking(filePath, content, language); } } /** * コードをASTに解析 */ parseCode(content, language) { const plugins = this.getParserPlugins(language); // TypeScriptファイルに対する追加設定 const parserOptions = { sourceType: 'module', allowImportExportEverywhere: true, allowReturnOutsideFunction: true, allowUndeclaredExports: true, plugins }; // TypeScriptの場合の特殊対応 if (language.includes('typescript')) { parserOptions.allowSuperOutsideMethod = true; parserOptions.strictMode = false; } return parse(content, parserOptions); } /** * 言語別パーサープラグイン設定 */ getParserPlugins(language) { const basePlugins = [ 'asyncGenerators', 'bigInt', 'classProperties', 'decorators-legacy', 'doExpressions', 'dynamicImport', 'exportDefaultFrom', 'exportNamespaceFrom', 'functionBind', 'functionSent', 'importMeta', 'nullishCoalescingOperator', 'numericSeparator', 'objectRestSpread', 'optionalCatchBinding', 'optionalChaining', 'throwExpressions', 'trailingFunctionCommas', 'importAssertions' ]; switch (language) { case 'typescript': return [...basePlugins, 'typescript', 'decorators-legacy', 'topLevelAwait', 'privateIn', 'classStaticBlock', 'moduleStringNames', 'importAssertions', 'decoratorAutoAccessors', 'explicitResourceManagement']; case 'typescript-jsx': return [...basePlugins, 'typescript', 'jsx', 'decorators-legacy', 'topLevelAwait', 'privateIn', 'classStaticBlock', 'moduleStringNames', 'importAssertions', 'decoratorAutoAccessors', 'explicitResourceManagement']; case 'javascript': return [...basePlugins, 'jsx', 'topLevelAwait']; default: return basePlugins; } } /** * ASTからセマンティックチャンクを抽出 */ extractSemanticChunks(ast, content, filePath, developmentPhase) { const chunks = []; const lines = content.split('\n'); const phaseConfig = this.options.phases[developmentPhase]; if (filePath.includes('openweathermap-adapter')) { console.log(`🔍 Extracting chunks with phase: ${developmentPhase}`); } // トップレベルの要素を分析 const topLevelNodes = []; traverse(ast, { enter(path) { if (path.isProgram()) { path.node.body.forEach(node => { topLevelNodes.push(node); }); } } }); if (filePath.includes('openweathermap-adapter')) { console.log(`📊 Found ${topLevelNodes.length} top-level nodes in openweathermap-adapter.ts`); } // セマンティック単位でチャンクを作成 for (const node of topLevelNodes) { const chunk = this.createChunkFromNode(node, lines, filePath, phaseConfig); if (chunk) { chunks.push(chunk); } } if (filePath.includes('openweathermap-adapter')) { console.log(`📝 Created ${chunks.length} chunks before optimization`); } // 開発フェーズに応じたチャンク最適化 return this.optimizeChunksForPhase(chunks, phaseConfig); } /** * ASTノードからチャンクを作成 */ createChunkFromNode(node, lines, filePath, phaseConfig) { const startLine = node.loc?.start.line || 1; const endLine = node.loc?.end.line || lines.length; const content = lines.slice(startLine - 1, endLine).join('\n'); // セマンティック情報の抽出 const semanticInfo = this.extractSemanticInfo(node); // 複雑度分析 const complexity = this.analyzeComplexity(node); // AI最適化メタデータの構築 const metadata = { startLine, endLine, semanticType: semanticInfo.type, name: semanticInfo.name, complexity, dependencies: this.extractDependencies(node), exports: this.extractExports(node), imports: this.extractImports(node), aiContext: { importance: this.calculateImportance(semanticInfo, complexity), searchability: this.calculateSearchability(semanticInfo), contextRelevance: this.calculateContextRelevance(semanticInfo, phaseConfig) }, tokens: this.estimateTokens(content), size: content.length }; return { content, metadata }; } /** * セマンティック情報の抽出 */ extractSemanticInfo(node) { switch (node.type) { case 'ClassDeclaration': return { type: 'class', name: node.id?.name || 'anonymous', subtype: this.getClassSubtype(node) }; case 'FunctionDeclaration': return { type: 'function', name: node.id?.name || 'anonymous', subtype: this.getFunctionSubtype(node) }; case 'VariableDeclaration': return { type: 'variable', name: this.getVariableNames(node).join(', '), subtype: this.getVariableSubtype(node) }; case 'ExportDefaultDeclaration': case 'ExportNamedDeclaration': return { type: 'export', name: this.getExportName(node), subtype: 'module_export' }; case 'ImportDeclaration': return { type: 'import', name: node.source.value, subtype: 'module_import' }; default: return { type: 'statement', name: node.type, subtype: 'general' }; } } /** * 複雑度分析(安全なAST走査) */ analyzeComplexity(node) { let cyclomaticComplexity = 1; let cognitiveComplexity = 0; let maxNestingLevel = 0; let currentNestingLevel = 0; const complexityNodes = [ 'IfStatement', 'WhileStatement', 'ForStatement', 'ForInStatement', 'ForOfStatement', 'DoWhileStatement', 'SwitchStatement', 'TryStatement', 'ConditionalExpression', 'LogicalExpression' ]; // 再帰的にノードを走査(traverseを使わない安全な方法) const walkNode = (currentNode, nestingLevel = 0) => { if (!currentNode || typeof currentNode !== 'object') return; // 複雑度ノードの検出 if (complexityNodes.includes(currentNode.type)) { cyclomaticComplexity++; cognitiveComplexity += nestingLevel + 1; } // ブロック文の検出(ネストレベル増加) let childNestingLevel = nestingLevel; if (t.isBlockStatement(currentNode) || t.isFunction(currentNode) || t.isArrowFunctionExpression(currentNode) || t.isObjectMethod(currentNode)) { childNestingLevel++; maxNestingLevel = Math.max(maxNestingLevel, childNestingLevel); } // 子ノードを再帰的に処理 for (const key in currentNode) { if (key === 'loc' || key === 'range' || key === 'start' || key === 'end') continue; const value = currentNode[key]; if (Array.isArray(value)) { value.forEach(child => walkNode(child, childNestingLevel)); } else if (value && typeof value === 'object' && value.type) { walkNode(value, childNestingLevel); } } }; walkNode(node); return { cyclomatic: cyclomaticComplexity, cognitive: cognitiveComplexity, nestingLevel: maxNestingLevel, maintainabilityIndex: this.calculateMaintainabilityIndex(cyclomaticComplexity, cognitiveComplexity) }; } /** * 保守性指数の計算 */ calculateMaintainabilityIndex(cyclomatic, cognitive) { // 簡略化された保守性指数計算 const base = 171; const cyclomaticPenalty = 5.2 * Math.log(cyclomatic); const cognitivePenalty = 0.23 * cognitive; return Math.max(0, base - cyclomaticPenalty - cognitivePenalty); } /** * 依存関係の抽出(安全なAST走査) */ extractDependencies(node) { const dependencies = []; // 再帰的にノードを走査(traverseを使わない安全な方法) const walkNode = (currentNode) => { if (!currentNode || typeof currentNode !== 'object') return; // CallExpressionの処理 if (t.isCallExpression(currentNode)) { if (t.isIdentifier(currentNode.callee)) { dependencies.push(currentNode.callee.name); } } // MemberExpressionの処理 if (t.isMemberExpression(currentNode)) { if (t.isIdentifier(currentNode.object)) { dependencies.push(currentNode.object.name); } } // 子ノードを再帰的に処理 for (const key in currentNode) { if (key === 'loc' || key === 'range' || key === 'start' || key === 'end') continue; const value = currentNode[key]; if (Array.isArray(value)) { value.forEach(walkNode); } else if (value && typeof value === 'object' && value.type) { walkNode(value); } } }; walkNode(node); return [...new Set(dependencies)]; } /** * 重要度の計算 */ calculateImportance(semanticInfo, complexity) { let importance = 0.5; // ベーススコア // セマンティックタイプによる重み付け const typeWeights = { class: 0.9, function: 0.8, export: 0.7, import: 0.3, variable: 0.4, statement: 0.2 }; importance += typeWeights[semanticInfo.type] || 0.2; // 複雑度による重み付け importance += Math.min(complexity.cognitive / 100, 0.3); return Math.min(importance, 1.0); } /** * 検索可能性の計算 */ calculateSearchability(semanticInfo) { const searchableTypes = ['class', 'function', 'export']; const baseSearchability = searchableTypes.includes(semanticInfo.type) ? 0.8 : 0.4; // 名前の質による調整 const nameQuality = semanticInfo.name && semanticInfo.name !== 'anonymous' ? 0.2 : 0; return Math.min(baseSearchability + nameQuality, 1.0); } /** * コンテキスト関連性の計算 */ calculateContextRelevance(semanticInfo, phaseConfig) { let relevance = 0.5; // 開発フェーズ別の関連性 if (phaseConfig.preserveClasses && semanticInfo.type === 'class') { relevance += 0.3; } if (phaseConfig.preserveFunctions && semanticInfo.type === 'function') { relevance += 0.3; } if (phaseConfig.preserveBlocks && semanticInfo.type === 'statement') { relevance += 0.2; } return Math.min(relevance, 1.0); } /** * 開発フェーズに応じたチャンク最適化 */ optimizeChunksForPhase(chunks, phaseConfig) { const processedChunks = []; for (const chunk of chunks) { const lineCount = chunk.metadata.endLine - chunk.metadata.startLine + 1; // 小さすぎるチャンクはスキップ if (lineCount < phaseConfig.minLines) { if (chunk.metadata.file && chunk.metadata.file.includes('openweathermap-adapter')) { console.log(`⚠️ Skipping too small chunk from openweathermap-adapter.ts: ${lineCount} lines (min: ${phaseConfig.minLines})`); } continue; } // トークン数をチェック const estimatedTokens = chunk.metadata.tokens || this.estimateTokens(chunk.content); const maxTokens = phaseConfig.maxTokens || 8000; // デフォルトは8000トークン // 大きすぎるチャンクは分割(行数またはトークン数で判定) if (lineCount > phaseConfig.maxLines || estimatedTokens > maxTokens) { if (chunk.metadata.file && chunk.metadata.file.includes('openweathermap-adapter')) { console.log(`🔀 Splitting large chunk from openweathermap-adapter.ts:`); console.log(` Lines: ${lineCount} (max: ${phaseConfig.maxLines})`); console.log(` Tokens: ${estimatedTokens} (max: ${maxTokens})`); console.log(` Chunk type: ${chunk.metadata.semanticType}, name: ${chunk.metadata.name}`); } // トークン数が多すぎる場合は、より小さく分割 const effectiveMaxLines = estimatedTokens > maxTokens ? Math.floor(phaseConfig.maxLines * (maxTokens / estimatedTokens)) : phaseConfig.maxLines; // 大きなチャンクを分割 const splitChunks = this.splitLargeChunk(chunk, effectiveMaxLines, maxTokens); processedChunks.push(...splitChunks); } else { // 適切なサイズのチャンク processedChunks.push({ ...chunk, metadata: { ...chunk.metadata, optimizedForPhase: true } }); } } // デバッグ: 最適化後のチャンク数 if (chunks.length > 0 && chunks[0].metadata.file && chunks[0].metadata.file.includes('openweathermap-adapter')) { console.log(`🔄 Optimization result for openweathermap-adapter.ts: ${chunks.length} chunks → ${processedChunks.length} chunks`); } return processedChunks; } /** * 大きなチャンクを分割 * @param {Object} chunk - 分割するチャンク * @param {number} maxLines - 最大行数 * @param {number} maxTokens - 最大トークン数 */ splitLargeChunk(chunk, maxLines, maxTokens = 6000) { const lines = chunk.content.split('\n'); const splitChunks = []; const overlap = Math.floor(maxLines * 0.1); // 10%のオーバーラップ let currentChunk = []; let currentTokens = 0; let startIdx = 0; for (let i = 0; i < lines.length; i++) { const line = lines[i]; const lineTokens = this.estimateTokens(line); // 現在のチャンクにこの行を追加するとトークン制限を超える場合 if (currentTokens + lineTokens > maxTokens && currentChunk.length > 0) { // 現在のチャンクを保存 const chunkContent = currentChunk.join('\n'); splitChunks.push({ content: chunkContent, metadata: { ...chunk.metadata, startLine: chunk.metadata.startLine + startIdx, endLine: chunk.metadata.startLine + i - 1, split: true, splitIndex: splitChunks.length, originalSize: lines.length, optimizedForPhase: true, size: chunkContent.length, tokens: this.estimateTokens(chunkContent) } }); // 新しいチャンクを開始(オーバーラップを考慮) const overlapStart = Math.max(0, currentChunk.length - overlap); currentChunk = currentChunk.slice(overlapStart); currentChunk.push(line); currentTokens = this.estimateTokens(currentChunk.join('\n')); startIdx = i - (currentChunk.length - 1); } else if (currentChunk.length >= maxLines) { // 行数制限に達した場合 const chunkContent = currentChunk.join('\n'); splitChunks.push({ content: chunkContent, metadata: { ...chunk.metadata, startLine: chunk.metadata.startLine + startIdx, endLine: chunk.metadata.startLine + i - 1, split: true, splitIndex: splitChunks.length, originalSize: lines.length, optimizedForPhase: true, size: chunkContent.length, tokens: this.estimateTokens(chunkContent) } }); // 新しいチャンクを開始 const overlapStart = Math.max(0, currentChunk.length - overlap); currentChunk = currentChunk.slice(overlapStart); currentChunk.push(line); currentTokens = this.estimateTokens(currentChunk.join('\n')); startIdx = i - (currentChunk.length - 1); } else { // 現在のチャンクに追加 currentChunk.push(line); currentTokens += lineTokens; } } // 最後のチャンクを保存 if (currentChunk.length >= 10) { // 非常に小さな断片はスキップ const chunkContent = currentChunk.join('\n'); splitChunks.push({ content: chunkContent, metadata: { ...chunk.metadata, startLine: chunk.metadata.startLine + startIdx, endLine: chunk.metadata.startLine + lines.length - 1, split: true, splitIndex: splitChunks.length, originalSize: lines.length, optimizedForPhase: true, size: chunkContent.length, tokens: this.estimateTokens(chunkContent) } }); } return splitChunks; } /** * 言語検出 */ detectLanguage(extension) { const languageMap = { '.js': 'javascript', '.jsx': 'javascript', '.ts': 'typescript', '.tsx': 'typescript-jsx', '.mjs': 'javascript', '.cjs': 'javascript', '.md': 'markdown', '.txt': 'text', '.json': 'json', '.py': 'python', '.java': 'java', '.cpp': 'cpp', '.c': 'c' }; return languageMap[extension] || 'text'; } /** * コードファイル判定 */ isCodeFile(language) { return ['javascript', 'typescript'].includes(language); } /** * 非コードファイルのチャンキング */ chunkNonCodeFile(filePath, content, language) { const maxChunkSize = 1000; const overlap = 200; const chunks = []; if (content.length <= maxChunkSize) { return [{ content, metadata: { file: filePath, language, startLine: 1, endLine: content.split('\n').length, chunkingMethod: 'simple', size: content.length, tokens: this.estimateTokens(content) } }]; } for (let i = 0; i < content.length; i += (maxChunkSize - overlap)) { const chunk = content.slice(i, i + maxChunkSize); const lines = chunk.split('\n'); // 正しい行番号を計算(バイト位置ではなく実際の行番号) const startLine = content.substring(0, i).split('\n').length; const endLine = content.substring(0, i + chunk.length).split('\n').length; chunks.push({ content: chunk, metadata: { file: filePath, language, startLine: startLine, endLine: endLine, chunkingMethod: 'overlap', size: chunk.length, tokens: this.estimateTokens(chunk) } }); } return chunks; } /** * フォールバックチャンキング */ fallbackChunking(filePath, content, language) { return this.chunkNonCodeFile(filePath, content, language); } /** * トークン数の推定 */ estimateTokens(content) { // 簡略化されたトークン推定 (実際にはtiktokenを使用) return Math.ceil(content.length / 4); } /** * ヘルパーメソッド群 */ getClassSubtype(node) { const hasConstructor = node.body.body.some(member => member.type === 'MethodDefinition' && member.key.name === 'constructor' ); if (node.superClass) return 'extends'; if (hasConstructor) return 'constructor'; return 'basic'; } getFunctionSubtype(node) { if (node.async) return 'async'; if (node.generator) return 'generator'; if (node.params.length === 0) return 'no_params'; return 'regular'; } getVariableNames(node) { return node.declarations.map(decl => decl.id.name || 'anonymous'); } getVariableSubtype(node) { const kind = node.kind; // const, let, var const hasInit = node.declarations.some(decl => decl.init); if (kind === 'const') return 'constant'; if (hasInit) return 'initialized'; return 'declaration'; } getExportName(node) { if (node.declaration) { if (node.declaration.id) { return node.declaration.id.name; } if (node.declaration.type === 'FunctionDeclaration') { return node.declaration.id?.name || 'anonymous'; } } return 'export'; } extractExports(node) { // 簡略化されたエクスポート抽出 return []; } extractImports(node) { // 簡略化されたインポート抽出 return []; } /** * エラー統計の更新 */ updateErrorStats(filePath, language, error) { this.parseErrors.total++; // ファイルタイプ別統計 const fileType = path.extname(filePath); this.parseErrors.byFileType[fileType] = (this.parseErrors.byFileType[fileType] || 0) + 1; // エラータイプ別統計 const errorType = this.categorizeError(error); this.parseErrors.byErrorType[errorType] = (this.parseErrors.byErrorType[errorType] || 0) + 1; // 失敗ファイルの記録 this.parseErrors.failedFiles.push({ file: filePath, language, errorType, message: error.message.split('\n')[0], // 最初の行のみ timestamp: new Date().toISOString() }); } /** * エラーの分類 */ categorizeError(error) { const message = error.message.toLowerCase(); if (message.includes('scope and parentpath')) { return 'traverse_scope_error'; } else if (message.includes('unexpected token') && message.includes('jsx')) { return 'jsx_syntax_error'; } else if (message.includes('unexpected token')) { return 'syntax_error'; } else if (message.includes('parsing error')) { return 'parse_error'; } else if (message.includes('plugin')) { return 'plugin_error'; } else if (message.includes('decorators')) { return 'decorator_error'; } else if (message.includes('import') || message.includes('export')) { return 'import_export_error'; } else if (message.includes('type') && message.includes('only')) { return 'typescript_syntax_error'; } else { // デバッグ情報を含む詳細エラー if (this.options.verboseErrors) { console.warn(` Unknown error details: ${error.message.slice(0, 200)}`); console.warn(` Error stack: ${error.stack?.slice(0, 300)}`); } return `unknown_error: ${error.message.slice(0, 50)}`; } } /** * エラー統計の取得 */ getErrorStats() { return { ...this.parseErrors, errorRate: this.parseErrors.total > 0 ? (this.parseErrors.total / (this.parseErrors.total + this.getSuccessCount())) * 100 : 0 }; } /** * 成功数の取得(近似値) */ getSuccessCount() { // これは近似値です。実際の実装では正確な成功カウントを追跡する必要があります return Math.max(0, 100 - this.parseErrors.total); } /** * エラー統計のリセット */ resetErrorStats() { this.parseErrors = { total: 0, byFileType: {}, byErrorType: {}, failedFiles: [] }; } }