@claude-vector/core/src/chunk-processor.js

Core vector search engine for code intelligence

/**
 * Chunk processor for splitting documents into searchable chunks
 */

export class ChunkProcessor {
  constructor(config = {}) {
    this.config = {
      maxSize: config.maxSize || 1000,
      minSize: config.minSize || 100,
      overlap: config.overlap || 200,
      splitByParagraph: config.splitByParagraph ?? true,
      preserveCodeBlocks: config.preserveCodeBlocks ?? true,
      ...config
    };
  }

  /**
   * Process a document into chunks
   */
  async processDocument(content, metadata = {}) {
    const chunks = [];
    
    // Split into initial segments
    const segments = this.splitIntoSegments(content);
    
    for (const segment of segments) {
      const processedChunks = await this.processSegment(segment, metadata);
      chunks.push(...processedChunks);
    }
    
    return chunks;
  }

  /**
   * Split content into segments (paragraphs, code blocks, etc.)
   */
  splitIntoSegments(content) {
    if (this.config.preserveCodeBlocks) {
      // Extract code blocks and process separately
      const codeBlockRegex = /```[\s\S]*?```/g;
      const codeBlocks = content.match(codeBlockRegex) || [];
      let remaining = content;
      
      for (const block of codeBlocks) {
        remaining = remaining.replace(block, '\n[CODE_BLOCK]\n');
      }
      
      const segments = [];
      const parts = remaining.split('[CODE_BLOCK]');
      
      for (let i = 0; i < parts.length; i++) {
        if (parts[i].trim()) {
          segments.push({ type: 'text', content: parts[i].trim() });
        }
        if (i < codeBlocks.length) {
          segments.push({ type: 'code', content: codeBlocks[i] });
        }
      }
      
      return segments;
    }
    
    // Simple paragraph split
    return content.split(/\n\n+/).map(p => ({ type: 'text', content: p.trim() })).filter(s => s.content);
  }

  /**
   * Process a segment into chunks
   */
  async processSegment(segment, metadata) {
    const chunks = [];
    const tokens = this.tokenize(segment.content);
    
    if (tokens.length <= this.config.maxSize) {
      // Segment fits in one chunk
      chunks.push({
        content: segment.content,
        type: segment.type,
        tokens: tokens.length,
        ...metadata
      });
    } else {
      // Split into multiple chunks with overlap
      let start = 0;
      while (start < tokens.length) {
        const end = Math.min(start + this.config.maxSize, tokens.length);
        const chunkTokens = tokens.slice(start, end);
        
        chunks.push({
          content: this.tokensToText(chunkTokens),
          type: segment.type,
          tokens: chunkTokens.length,
          ...metadata
        });
        
        start = end - this.config.overlap;
      }
    }
    
    return chunks.filter(chunk => chunk.tokens >= this.config.minSize);
  }

  /**
   * Tokenize text (simplified - in real implementation use tiktoken)
   */
  tokenize(text) {
    // Simplified tokenization - approximately 4 characters per token
    // In production, use tiktoken for accurate token counting
    const words = text.split(/\s+/);
    const tokens = [];
    
    for (const word of words) {
      // Rough approximation: ~1 token per 4 characters
      const tokenCount = Math.ceil(word.length / 4);
      for (let i = 0; i < tokenCount; i++) {
        tokens.push(word.slice(i * 4, (i + 1) * 4));
      }
    }
    
    return tokens;
  }

  /**
   * Convert tokens back to text
   */
  tokensToText(tokens) {
    // Since we're using a simplified tokenization, just join
    return tokens.join('');
  }
}