@hpbyte/h-codex-core/dist/src/ingestion/chunker/cst-chunker.d.ts

Core indexing and search functionality for h-codex

import type { ChunkParams } from '../../types';
export declare class CSTChunker {
    private filePath;
    private fileContent;
    private projectId;
    private language;
    chunk({ filePath, projectId, language }: ChunkParams): Promise<{
        projectId: string;
        content: string;
        filePath: string;
        startLine: number;
        endLine: number;
        nodeType: string;
        hash: string;
        size: number;
        id?: string | undefined;
        createdAt?: Date | null | undefined;
        updatedAt?: Date | null | undefined;
        language?: string | null | undefined;
    }[]>;
    /**
     * AST-based chunking
     * - each nodes are traversed and greedily bundled together
     * - if a node is too large, recursively chunk its children
     * - else, concatenate the current chunk to the node
     */
    private chunkNode;
    /**
     * try to combine very small chunks (under the coalesce threshold) with adjacent chunks
     * but if combing will exceed the max chunk size, keep as is
     */
    private processChunks;
    private formatAsCodeChunk;
}
export declare const cstChunker: CSTChunker;