@claude-vector/core
Version:
Core vector search engine for code intelligence
876 lines (769 loc) • 27.3 kB
JavaScript
/**
* Semantic Chunker - AST-based intelligent code chunking
*
* パラメトリック思考によるAI駆動開発最適化:
* - 開発フェーズ適応型チャンキング
* - セマンティック境界の保持
* - 複雑度ベースの動的サイズ調整
*/
import { parse } from '@babel/parser';
import _traverse from '@babel/traverse';
import * as t from '@babel/types';
import fs from 'fs/promises';
import path from 'path';
// @babel/traverse の ESM import の修正
const traverse = _traverse.default || _traverse;
export class SemanticChunker {
constructor(options = {}) {
// エラー統計の初期化
this.parseErrors = {
total: 0,
byFileType: {},
byErrorType: {},
failedFiles: []
};
this.options = {
// 開発フェーズ別設定
phases: {
exploration: {
chunkSize: 'large',
minLines: 50,
maxLines: 200,
includeContext: true,
preserveClasses: true
},
implementation: {
chunkSize: 'medium',
minLines: 5,
maxLines: 300, // 500から削減してトークン制限に対応
maxTokens: 6000, // 新規追加: OpenAI APIの制限を考慮
includeContext: true,
preserveFunctions: true
},
debugging: {
chunkSize: 'small',
minLines: 5,
maxLines: 30,
includeContext: false,
preserveBlocks: true
},
refactoring: {
chunkSize: 'variable',
minLines: 10,
maxLines: 100,
includeContext: true,
preserveStructure: true
}
},
defaultPhase: 'implementation',
...options
};
}
/**
* ファイルをセマンティックチャンクに分割
*/
async chunkFile(filePath, content, developmentPhase = 'implementation') {
const fileExtension = path.extname(filePath).toLowerCase();
const language = this.detectLanguage(fileExtension);
// デバッグ: openweathermap-adapter.ts の処理を追跡
if (filePath.includes('openweathermap-adapter')) {
console.log(`🔍 SemanticChunker processing openweathermap-adapter.ts:`);
console.log(` Extension: ${fileExtension}, Language: ${language}`);
console.log(` Content length: ${content.length} chars`);
console.log(` Is code file: ${this.isCodeFile(language)}`);
}
if (!this.isCodeFile(language)) {
return this.chunkNonCodeFile(filePath, content, language);
}
try {
const ast = this.parseCode(content, language);
const chunks = this.extractSemanticChunks(ast, content, filePath, developmentPhase);
if (filePath.includes('openweathermap-adapter')) {
console.log(`📦 Extracted ${chunks.length} semantic chunks from openweathermap-adapter.ts`);
}
return chunks.map(chunk => ({
...chunk,
metadata: {
...chunk.metadata,
file: filePath,
language,
developmentPhase,
chunkingMethod: 'semantic'
}
}));
} catch (error) {
// エラー統計の更新
this.updateErrorStats(filePath, language, error);
// 改善されたエラーメッセージ
const fileType = path.extname(filePath);
const errorType = this.categorizeError(error);
console.warn(`⚠️ AST parsing failed for ${filePath}, falling back to simple chunking: ${errorType}`);
// 詳細なデバッグ情報(Next.js APIルート等の特殊構文対応)
if (this.options.verboseErrors || filePath.includes('route.ts')) {
console.warn(` File type: ${fileType}, Language: ${language}`);
console.warn(` Error details: ${error.message}`);
console.warn(` Parser plugins: ${JSON.stringify(this.getParserPlugins(language))}`);
if (filePath.includes('/api/') || filePath.includes('route.')) {
console.warn(` Note: This appears to be a Next.js API route file`);
}
}
return this.fallbackChunking(filePath, content, language);
}
}
/**
* コードをASTに解析
*/
parseCode(content, language) {
const plugins = this.getParserPlugins(language);
// TypeScriptファイルに対する追加設定
const parserOptions = {
sourceType: 'module',
allowImportExportEverywhere: true,
allowReturnOutsideFunction: true,
allowUndeclaredExports: true,
plugins
};
// TypeScriptの場合の特殊対応
if (language.includes('typescript')) {
parserOptions.allowSuperOutsideMethod = true;
parserOptions.strictMode = false;
}
return parse(content, parserOptions);
}
/**
* 言語別パーサープラグイン設定
*/
getParserPlugins(language) {
const basePlugins = [
'asyncGenerators',
'bigInt',
'classProperties',
'decorators-legacy',
'doExpressions',
'dynamicImport',
'exportDefaultFrom',
'exportNamespaceFrom',
'functionBind',
'functionSent',
'importMeta',
'nullishCoalescingOperator',
'numericSeparator',
'objectRestSpread',
'optionalCatchBinding',
'optionalChaining',
'throwExpressions',
'trailingFunctionCommas',
'importAssertions'
];
switch (language) {
case 'typescript':
return [...basePlugins, 'typescript', 'decorators-legacy', 'topLevelAwait',
'privateIn', 'classStaticBlock', 'moduleStringNames', 'importAssertions',
'decoratorAutoAccessors', 'explicitResourceManagement'];
case 'typescript-jsx':
return [...basePlugins, 'typescript', 'jsx', 'decorators-legacy', 'topLevelAwait',
'privateIn', 'classStaticBlock', 'moduleStringNames', 'importAssertions',
'decoratorAutoAccessors', 'explicitResourceManagement'];
case 'javascript':
return [...basePlugins, 'jsx', 'topLevelAwait'];
default:
return basePlugins;
}
}
/**
* ASTからセマンティックチャンクを抽出
*/
extractSemanticChunks(ast, content, filePath, developmentPhase) {
const chunks = [];
const lines = content.split('\n');
const phaseConfig = this.options.phases[developmentPhase];
if (filePath.includes('openweathermap-adapter')) {
console.log(`🔍 Extracting chunks with phase: ${developmentPhase}`);
}
// トップレベルの要素を分析
const topLevelNodes = [];
traverse(ast, {
enter(path) {
if (path.isProgram()) {
path.node.body.forEach(node => {
topLevelNodes.push(node);
});
}
}
});
if (filePath.includes('openweathermap-adapter')) {
console.log(`📊 Found ${topLevelNodes.length} top-level nodes in openweathermap-adapter.ts`);
}
// セマンティック単位でチャンクを作成
for (const node of topLevelNodes) {
const chunk = this.createChunkFromNode(node, lines, filePath, phaseConfig);
if (chunk) {
chunks.push(chunk);
}
}
if (filePath.includes('openweathermap-adapter')) {
console.log(`📝 Created ${chunks.length} chunks before optimization`);
}
// 開発フェーズに応じたチャンク最適化
return this.optimizeChunksForPhase(chunks, phaseConfig);
}
/**
* ASTノードからチャンクを作成
*/
createChunkFromNode(node, lines, filePath, phaseConfig) {
const startLine = node.loc?.start.line || 1;
const endLine = node.loc?.end.line || lines.length;
const content = lines.slice(startLine - 1, endLine).join('\n');
// セマンティック情報の抽出
const semanticInfo = this.extractSemanticInfo(node);
// 複雑度分析
const complexity = this.analyzeComplexity(node);
// AI最適化メタデータの構築
const metadata = {
startLine,
endLine,
semanticType: semanticInfo.type,
name: semanticInfo.name,
complexity,
dependencies: this.extractDependencies(node),
exports: this.extractExports(node),
imports: this.extractImports(node),
aiContext: {
importance: this.calculateImportance(semanticInfo, complexity),
searchability: this.calculateSearchability(semanticInfo),
contextRelevance: this.calculateContextRelevance(semanticInfo, phaseConfig)
},
tokens: this.estimateTokens(content),
size: content.length
};
return {
content,
metadata
};
}
/**
* セマンティック情報の抽出
*/
extractSemanticInfo(node) {
switch (node.type) {
case 'ClassDeclaration':
return {
type: 'class',
name: node.id?.name || 'anonymous',
subtype: this.getClassSubtype(node)
};
case 'FunctionDeclaration':
return {
type: 'function',
name: node.id?.name || 'anonymous',
subtype: this.getFunctionSubtype(node)
};
case 'VariableDeclaration':
return {
type: 'variable',
name: this.getVariableNames(node).join(', '),
subtype: this.getVariableSubtype(node)
};
case 'ExportDefaultDeclaration':
case 'ExportNamedDeclaration':
return {
type: 'export',
name: this.getExportName(node),
subtype: 'module_export'
};
case 'ImportDeclaration':
return {
type: 'import',
name: node.source.value,
subtype: 'module_import'
};
default:
return {
type: 'statement',
name: node.type,
subtype: 'general'
};
}
}
/**
* 複雑度分析(安全なAST走査)
*/
analyzeComplexity(node) {
let cyclomaticComplexity = 1;
let cognitiveComplexity = 0;
let maxNestingLevel = 0;
let currentNestingLevel = 0;
const complexityNodes = [
'IfStatement', 'WhileStatement', 'ForStatement', 'ForInStatement',
'ForOfStatement', 'DoWhileStatement', 'SwitchStatement', 'TryStatement',
'ConditionalExpression', 'LogicalExpression'
];
// 再帰的にノードを走査(traverseを使わない安全な方法)
const walkNode = (currentNode, nestingLevel = 0) => {
if (!currentNode || typeof currentNode !== 'object') return;
// 複雑度ノードの検出
if (complexityNodes.includes(currentNode.type)) {
cyclomaticComplexity++;
cognitiveComplexity += nestingLevel + 1;
}
// ブロック文の検出(ネストレベル増加)
let childNestingLevel = nestingLevel;
if (t.isBlockStatement(currentNode) || t.isFunction(currentNode) ||
t.isArrowFunctionExpression(currentNode) || t.isObjectMethod(currentNode)) {
childNestingLevel++;
maxNestingLevel = Math.max(maxNestingLevel, childNestingLevel);
}
// 子ノードを再帰的に処理
for (const key in currentNode) {
if (key === 'loc' || key === 'range' || key === 'start' || key === 'end') continue;
const value = currentNode[key];
if (Array.isArray(value)) {
value.forEach(child => walkNode(child, childNestingLevel));
} else if (value && typeof value === 'object' && value.type) {
walkNode(value, childNestingLevel);
}
}
};
walkNode(node);
return {
cyclomatic: cyclomaticComplexity,
cognitive: cognitiveComplexity,
nestingLevel: maxNestingLevel,
maintainabilityIndex: this.calculateMaintainabilityIndex(cyclomaticComplexity, cognitiveComplexity)
};
}
/**
* 保守性指数の計算
*/
calculateMaintainabilityIndex(cyclomatic, cognitive) {
// 簡略化された保守性指数計算
const base = 171;
const cyclomaticPenalty = 5.2 * Math.log(cyclomatic);
const cognitivePenalty = 0.23 * cognitive;
return Math.max(0, base - cyclomaticPenalty - cognitivePenalty);
}
/**
* 依存関係の抽出(安全なAST走査)
*/
extractDependencies(node) {
const dependencies = [];
// 再帰的にノードを走査(traverseを使わない安全な方法)
const walkNode = (currentNode) => {
if (!currentNode || typeof currentNode !== 'object') return;
// CallExpressionの処理
if (t.isCallExpression(currentNode)) {
if (t.isIdentifier(currentNode.callee)) {
dependencies.push(currentNode.callee.name);
}
}
// MemberExpressionの処理
if (t.isMemberExpression(currentNode)) {
if (t.isIdentifier(currentNode.object)) {
dependencies.push(currentNode.object.name);
}
}
// 子ノードを再帰的に処理
for (const key in currentNode) {
if (key === 'loc' || key === 'range' || key === 'start' || key === 'end') continue;
const value = currentNode[key];
if (Array.isArray(value)) {
value.forEach(walkNode);
} else if (value && typeof value === 'object' && value.type) {
walkNode(value);
}
}
};
walkNode(node);
return [...new Set(dependencies)];
}
/**
* 重要度の計算
*/
calculateImportance(semanticInfo, complexity) {
let importance = 0.5; // ベーススコア
// セマンティックタイプによる重み付け
const typeWeights = {
class: 0.9,
function: 0.8,
export: 0.7,
import: 0.3,
variable: 0.4,
statement: 0.2
};
importance += typeWeights[semanticInfo.type] || 0.2;
// 複雑度による重み付け
importance += Math.min(complexity.cognitive / 100, 0.3);
return Math.min(importance, 1.0);
}
/**
* 検索可能性の計算
*/
calculateSearchability(semanticInfo) {
const searchableTypes = ['class', 'function', 'export'];
const baseSearchability = searchableTypes.includes(semanticInfo.type) ? 0.8 : 0.4;
// 名前の質による調整
const nameQuality = semanticInfo.name && semanticInfo.name !== 'anonymous' ? 0.2 : 0;
return Math.min(baseSearchability + nameQuality, 1.0);
}
/**
* コンテキスト関連性の計算
*/
calculateContextRelevance(semanticInfo, phaseConfig) {
let relevance = 0.5;
// 開発フェーズ別の関連性
if (phaseConfig.preserveClasses && semanticInfo.type === 'class') {
relevance += 0.3;
}
if (phaseConfig.preserveFunctions && semanticInfo.type === 'function') {
relevance += 0.3;
}
if (phaseConfig.preserveBlocks && semanticInfo.type === 'statement') {
relevance += 0.2;
}
return Math.min(relevance, 1.0);
}
/**
* 開発フェーズに応じたチャンク最適化
*/
optimizeChunksForPhase(chunks, phaseConfig) {
const processedChunks = [];
for (const chunk of chunks) {
const lineCount = chunk.metadata.endLine - chunk.metadata.startLine + 1;
// 小さすぎるチャンクはスキップ
if (lineCount < phaseConfig.minLines) {
if (chunk.metadata.file && chunk.metadata.file.includes('openweathermap-adapter')) {
console.log(`⚠️ Skipping too small chunk from openweathermap-adapter.ts: ${lineCount} lines (min: ${phaseConfig.minLines})`);
}
continue;
}
// トークン数をチェック
const estimatedTokens = chunk.metadata.tokens || this.estimateTokens(chunk.content);
const maxTokens = phaseConfig.maxTokens || 8000; // デフォルトは8000トークン
// 大きすぎるチャンクは分割(行数またはトークン数で判定)
if (lineCount > phaseConfig.maxLines || estimatedTokens > maxTokens) {
if (chunk.metadata.file && chunk.metadata.file.includes('openweathermap-adapter')) {
console.log(`🔀 Splitting large chunk from openweathermap-adapter.ts:`);
console.log(` Lines: ${lineCount} (max: ${phaseConfig.maxLines})`);
console.log(` Tokens: ${estimatedTokens} (max: ${maxTokens})`);
console.log(` Chunk type: ${chunk.metadata.semanticType}, name: ${chunk.metadata.name}`);
}
// トークン数が多すぎる場合は、より小さく分割
const effectiveMaxLines = estimatedTokens > maxTokens
? Math.floor(phaseConfig.maxLines * (maxTokens / estimatedTokens))
: phaseConfig.maxLines;
// 大きなチャンクを分割
const splitChunks = this.splitLargeChunk(chunk, effectiveMaxLines, maxTokens);
processedChunks.push(...splitChunks);
} else {
// 適切なサイズのチャンク
processedChunks.push({
...chunk,
metadata: {
...chunk.metadata,
optimizedForPhase: true
}
});
}
}
// デバッグ: 最適化後のチャンク数
if (chunks.length > 0 && chunks[0].metadata.file && chunks[0].metadata.file.includes('openweathermap-adapter')) {
console.log(`🔄 Optimization result for openweathermap-adapter.ts: ${chunks.length} chunks → ${processedChunks.length} chunks`);
}
return processedChunks;
}
/**
* 大きなチャンクを分割
* @param {Object} chunk - 分割するチャンク
* @param {number} maxLines - 最大行数
* @param {number} maxTokens - 最大トークン数
*/
splitLargeChunk(chunk, maxLines, maxTokens = 6000) {
const lines = chunk.content.split('\n');
const splitChunks = [];
const overlap = Math.floor(maxLines * 0.1); // 10%のオーバーラップ
let currentChunk = [];
let currentTokens = 0;
let startIdx = 0;
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const lineTokens = this.estimateTokens(line);
// 現在のチャンクにこの行を追加するとトークン制限を超える場合
if (currentTokens + lineTokens > maxTokens && currentChunk.length > 0) {
// 現在のチャンクを保存
const chunkContent = currentChunk.join('\n');
splitChunks.push({
content: chunkContent,
metadata: {
...chunk.metadata,
startLine: chunk.metadata.startLine + startIdx,
endLine: chunk.metadata.startLine + i - 1,
split: true,
splitIndex: splitChunks.length,
originalSize: lines.length,
optimizedForPhase: true,
size: chunkContent.length,
tokens: this.estimateTokens(chunkContent)
}
});
// 新しいチャンクを開始(オーバーラップを考慮)
const overlapStart = Math.max(0, currentChunk.length - overlap);
currentChunk = currentChunk.slice(overlapStart);
currentChunk.push(line);
currentTokens = this.estimateTokens(currentChunk.join('\n'));
startIdx = i - (currentChunk.length - 1);
} else if (currentChunk.length >= maxLines) {
// 行数制限に達した場合
const chunkContent = currentChunk.join('\n');
splitChunks.push({
content: chunkContent,
metadata: {
...chunk.metadata,
startLine: chunk.metadata.startLine + startIdx,
endLine: chunk.metadata.startLine + i - 1,
split: true,
splitIndex: splitChunks.length,
originalSize: lines.length,
optimizedForPhase: true,
size: chunkContent.length,
tokens: this.estimateTokens(chunkContent)
}
});
// 新しいチャンクを開始
const overlapStart = Math.max(0, currentChunk.length - overlap);
currentChunk = currentChunk.slice(overlapStart);
currentChunk.push(line);
currentTokens = this.estimateTokens(currentChunk.join('\n'));
startIdx = i - (currentChunk.length - 1);
} else {
// 現在のチャンクに追加
currentChunk.push(line);
currentTokens += lineTokens;
}
}
// 最後のチャンクを保存
if (currentChunk.length >= 10) { // 非常に小さな断片はスキップ
const chunkContent = currentChunk.join('\n');
splitChunks.push({
content: chunkContent,
metadata: {
...chunk.metadata,
startLine: chunk.metadata.startLine + startIdx,
endLine: chunk.metadata.startLine + lines.length - 1,
split: true,
splitIndex: splitChunks.length,
originalSize: lines.length,
optimizedForPhase: true,
size: chunkContent.length,
tokens: this.estimateTokens(chunkContent)
}
});
}
return splitChunks;
}
/**
* 言語検出
*/
detectLanguage(extension) {
const languageMap = {
'.js': 'javascript',
'.jsx': 'javascript',
'.ts': 'typescript',
'.tsx': 'typescript-jsx',
'.mjs': 'javascript',
'.cjs': 'javascript',
'.md': 'markdown',
'.txt': 'text',
'.json': 'json',
'.py': 'python',
'.java': 'java',
'.cpp': 'cpp',
'.c': 'c'
};
return languageMap[extension] || 'text';
}
/**
* コードファイル判定
*/
isCodeFile(language) {
return ['javascript', 'typescript'].includes(language);
}
/**
* 非コードファイルのチャンキング
*/
chunkNonCodeFile(filePath, content, language) {
const maxChunkSize = 1000;
const overlap = 200;
const chunks = [];
if (content.length <= maxChunkSize) {
return [{
content,
metadata: {
file: filePath,
language,
startLine: 1,
endLine: content.split('\n').length,
chunkingMethod: 'simple',
size: content.length,
tokens: this.estimateTokens(content)
}
}];
}
for (let i = 0; i < content.length; i += (maxChunkSize - overlap)) {
const chunk = content.slice(i, i + maxChunkSize);
const lines = chunk.split('\n');
// 正しい行番号を計算(バイト位置ではなく実際の行番号)
const startLine = content.substring(0, i).split('\n').length;
const endLine = content.substring(0, i + chunk.length).split('\n').length;
chunks.push({
content: chunk,
metadata: {
file: filePath,
language,
startLine: startLine,
endLine: endLine,
chunkingMethod: 'overlap',
size: chunk.length,
tokens: this.estimateTokens(chunk)
}
});
}
return chunks;
}
/**
* フォールバックチャンキング
*/
fallbackChunking(filePath, content, language) {
return this.chunkNonCodeFile(filePath, content, language);
}
/**
* トークン数の推定
*/
estimateTokens(content) {
// 簡略化されたトークン推定 (実際にはtiktokenを使用)
return Math.ceil(content.length / 4);
}
/**
* ヘルパーメソッド群
*/
getClassSubtype(node) {
const hasConstructor = node.body.body.some(member =>
member.type === 'MethodDefinition' && member.key.name === 'constructor'
);
if (node.superClass) return 'extends';
if (hasConstructor) return 'constructor';
return 'basic';
}
getFunctionSubtype(node) {
if (node.async) return 'async';
if (node.generator) return 'generator';
if (node.params.length === 0) return 'no_params';
return 'regular';
}
getVariableNames(node) {
return node.declarations.map(decl => decl.id.name || 'anonymous');
}
getVariableSubtype(node) {
const kind = node.kind; // const, let, var
const hasInit = node.declarations.some(decl => decl.init);
if (kind === 'const') return 'constant';
if (hasInit) return 'initialized';
return 'declaration';
}
getExportName(node) {
if (node.declaration) {
if (node.declaration.id) {
return node.declaration.id.name;
}
if (node.declaration.type === 'FunctionDeclaration') {
return node.declaration.id?.name || 'anonymous';
}
}
return 'export';
}
extractExports(node) {
// 簡略化されたエクスポート抽出
return [];
}
extractImports(node) {
// 簡略化されたインポート抽出
return [];
}
/**
* エラー統計の更新
*/
updateErrorStats(filePath, language, error) {
this.parseErrors.total++;
// ファイルタイプ別統計
const fileType = path.extname(filePath);
this.parseErrors.byFileType[fileType] = (this.parseErrors.byFileType[fileType] || 0) + 1;
// エラータイプ別統計
const errorType = this.categorizeError(error);
this.parseErrors.byErrorType[errorType] = (this.parseErrors.byErrorType[errorType] || 0) + 1;
// 失敗ファイルの記録
this.parseErrors.failedFiles.push({
file: filePath,
language,
errorType,
message: error.message.split('\n')[0], // 最初の行のみ
timestamp: new Date().toISOString()
});
}
/**
* エラーの分類
*/
categorizeError(error) {
const message = error.message.toLowerCase();
if (message.includes('scope and parentpath')) {
return 'traverse_scope_error';
} else if (message.includes('unexpected token') && message.includes('jsx')) {
return 'jsx_syntax_error';
} else if (message.includes('unexpected token')) {
return 'syntax_error';
} else if (message.includes('parsing error')) {
return 'parse_error';
} else if (message.includes('plugin')) {
return 'plugin_error';
} else if (message.includes('decorators')) {
return 'decorator_error';
} else if (message.includes('import') || message.includes('export')) {
return 'import_export_error';
} else if (message.includes('type') && message.includes('only')) {
return 'typescript_syntax_error';
} else {
// デバッグ情報を含む詳細エラー
if (this.options.verboseErrors) {
console.warn(` Unknown error details: ${error.message.slice(0, 200)}`);
console.warn(` Error stack: ${error.stack?.slice(0, 300)}`);
}
return `unknown_error: ${error.message.slice(0, 50)}`;
}
}
/**
* エラー統計の取得
*/
getErrorStats() {
return {
...this.parseErrors,
errorRate: this.parseErrors.total > 0 ?
(this.parseErrors.total / (this.parseErrors.total + this.getSuccessCount())) * 100 : 0
};
}
/**
* 成功数の取得(近似値)
*/
getSuccessCount() {
// これは近似値です。実際の実装では正確な成功カウントを追跡する必要があります
return Math.max(0, 100 - this.parseErrors.total);
}
/**
* エラー統計のリセット
*/
resetErrorStats() {
this.parseErrors = {
total: 0,
byFileType: {},
byErrorType: {},
failedFiles: []
};
}
}